# General crawlers
User-agent: *
Disallow:
Crawl-delay: 10

  User-agent: *
  Allow: /

  # Aggressive AI/SEO bots - slow them down significantly
  User-agent: BLEXBot
  Crawl-delay: 10

  User-agent: ChatGPT-User
  Crawl-delay: 10

  User-agent: GPTBot
  Crawl-delay: 10

  User-agent: ClaudeBot
  Crawl-delay: 10

  User-agent: Amazonbot
  Crawl-delay: 10

  User-agent: anthropic-ai
  Crawl-delay: 10

  User-agent: cohere-ai
  Crawl-delay: 10

  User-agent: PerplexityBot
  Crawl-delay: 10

  User-agent: Omgilibot
  Crawl-delay: 10

  # SEO/Rank tracking bots (moderate slow down)
  User-agent: SemrushBot
  Crawl-delay: 5

  User-agent: AhrefsBot
  Crawl-delay: 5

  User-agent: DotBot
  Crawl-delay: 5

  User-agent: serpstatbot
  Crawl-delay: 5

  User-agent: BLP_bbot
  Crawl-delay: 5

  User-agent: DataForSeoBot
  Crawl-delay: 5

  # Allow major search engines normal access
  User-agent: Googlebot
  Crawl-delay: 1

  User-agent: Bingbot
  Crawl-delay: 2

  User-agent: Slurp
  Crawl-delay: 2

  User-agent: DuckDuckBot
  Crawl-delay: 2

  User-agent: Baiduspider
  Crawl-delay: 2

  # Facebook/Social crawlers - normal rate
  User-agent: facebookexternalhit
  Crawl-delay: 1

  User-agent: Twitterbot
  Crawl-delay: 1

  # Disallow problematic paths for all bots
  Disallow: /wp-admin/
  Allow: /wp-admin/admin-ajax.php
  Disallow: /wp-login.php
  Disallow: /wp-json/
  Disallow: /xmlrpc.php
  Disallow: /wp-includes/
  Disallow: /wp-content/plugins/
  Disallow: /wp-content/cache/
  Disallow: /*?s=
  Disallow: /*?p=
  Disallow: /*?replytocom
  Disallow: /tag/
  Disallow: /author/