User-agent: *
Disallow: /cexplorer/api/
Disallow: /admin/
Disallow: /inc/
Allow: /

# Ad and AI crawlers (known legit)
User-agent: ClaudeBot
Allow: /

User-agent: GPTBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: PerplexityBot
Allow: /

# Ad verification bots (IAS, DoubleVerify, etc.)
User-agent: ias-or
Allow: /
User-agent: ias-va
Allow: /
User-agent: ias-ie
Allow: /
User-agent: DoubleVerify
Allow: /
User-agent: Moatbot
Allow: /
User-agent: ComscoreBot
Allow: /

# Bad or heavy scrapers — ban politely
User-agent: MJ12bot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: Yandex
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: SemrushBot
Disallow: /

Sitemap: https://www.cryptyolo.com/sitemap.xml