# --- THE VIP LIST (AI Training Bots) ---
# These are the bots Cloudflare monetizes. We want them to index EVERYTHING.

User-agent: GPTBot
User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Bytespider
Allow: /

User-agent: FacebookBot
Allow: /

# --- THE "MAYBE" LIST ---
# Apple's bot. Good to have.
User-agent: Applebot
Allow: /

# --- THE JUNK ---
# SEO scrapers that just waste bandwidth and don't pay.
# You can remove these blocks if you want pure volume, 
# but they won't help your Cloudflare application.
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /

# --- GENERAL RULES ---
# Fallback for everyone else. 
User-agent: *
Allow: /
# Add any private admin folders here if needed:
Disallow: /admin/
Disallow: /login/

# --- THE SECRET WEAPON ---
# Point them to your map so they find all 100 years of uniforms
Sitemap: https://mlbcollectors.com/sitemap.xml