# godofbots.org robots.txt
# Content on this site may not be used to train, fine-tune or evaluate AI/ML
# models or to build datasets. See https://godofbots.org/legal.html and
# https://godofbots.org/terms.html (sections 5 and 6).
# Ordinary search-engine indexing is welcome.

# --- AI training / dataset crawlers: not permitted ---
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Ai2Bot-Dolma
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: img2dataset
Disallow: /

# --- everyone else (search engines) ---
User-agent: *
Allow: /

Sitemap: https://godofbots.org/sitemap.xml