# SMALLSTUDIO ROBOTS.TXT

# ============================
# Allow Googlebots
# ============================
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

# ============================
# Global rules for all crawlers
# ============================
User-agent: *
Crawl-delay: 30

# Allow important assets
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.avif$
Allow: /wp-content/uploads/

# Block sensitive or duplicate paths
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /?s=
Disallow: /*?
Disallow: /favorites/
Disallow: /my-account/
Disallow: /cart/
Disallow: /checkout/
Disallow: /?orderby=
Disallow: /?gad_source=
Disallow: /?wc-ajax=get_refreshed_fragments
Disallow: /?action=pys_get_pbid
Disallow: /*add_to_wishlist=
Disallow: /*add-to-cart=

# ============================
# Block known scrapers / bots
# ============================
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: CCBot
User-agent: MJ12bot
User-agent: MegaIndex.ru
User-agent: Yandex
User-agent: Baiduspider
User-agent: YoudaoBot
User-agent: BLEXBot
User-agent: TwengaBot
User-agent: 008
User-agent: AhrefsBot
User-agent: WotBox
User-agent: Sosospider
User-agent: ZumBot
User-agent: dotbot
User-agent: Linguee
User-agent: barkrowler
User-agent: crawl
Disallow: /