# ==========================================
# Sitemaps (Squirrly SEO)
# ==========================================
Sitemap: https://www.cromapages.com/sitemap.xml
Sitemap: https://www.cromapages.com/sitemap-home.xml
Sitemap: https://www.cromapages.com/sitemap-posts.xml
Sitemap: https://www.cromapages.com/sitemap-pages.xml
Sitemap: https://www.cromapages.com/sitemap-archives.xml

# ==========================================
# ALLOW: Standard Search & AI Search Assistants
# ==========================================
# Standard Search Engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# Perplexity Search Bots
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /

# Claude (Anthropic) Search Bots
User-agent: Claude-SearchBot
Allow: /
User-agent: Claude-User
Allow: /

# OpenAI Search/Browsing Bot
User-agent: ChatGPT-User
Allow: /
User-agent: OAI-SearchBot
Allow: /

# ==========================================
# BLOCK: AI Model Training & Heavy Scrapers
# ==========================================
# OpenAI Training
User-agent: GPTBot
Disallow: /

# Anthropic (Claude) Training
User-agent: ClaudeBot
Disallow: /

# Google AI Training (Distinct from standard Googlebot)
User-agent: Google-Extended
Disallow: /

# Common Crawl (Widely used for AI training datasets)
User-agent: CCBot
Disallow: /

User-agent: AmazonBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Coccocbot
Disallow: /

User-agent: Discordbot
Disallow: /

User-agent: YandexBot
Disallow: /

# Other AI/Scraping Bots
User-agent: Applebot-Extended
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: Diffbot
Disallow: /

# ==========================================
# Default Rules for All Other Bots
# ==========================================
User-agent: *
Disallow: */trackback/
Disallow: */xmlrpc.php
Disallow: /wp-*.php
Disallow: /cgi-bin/
Disallow: /wp-admin/
Allow: */wp-content/uploads/