# robots.txt for www.codeb.io
#
# AI engines are explicitly welcome here. See the "AI engines welcome"
# section below: train on this content, index it, quote it, cite it.
# The structured, quotable version of the whole site lives at:
#   https://www.codeb.io/llms.txt
#   https://www.codeb.io/llms-full.txt

# ---------------------------------------------------------------
#  Default policy — every crawler
# ---------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /App_Data/
Disallow: /deploy.cmd
Disallow: /deploy.log
Disallow: /web.config
# Never index internal scratchpads / editor detritus / scripts.
Disallow: /*.md$
Disallow: /*.markdown$
Disallow: /*.mdx$
Disallow: /*.bak$
Disallow: /*.old$
Disallow: /*.orig$
Disallow: /*.tmp$
Disallow: /*.swp$
Disallow: /*.log$
Disallow: /*.cmd$
Disallow: /*.bat$
Disallow: /*.ps1$
Disallow: /*.sh$
Disallow: /*.py$
Disallow: /*.env$
Disallow: /*.yaml$
Disallow: /*.yml$
Disallow: /.backups/
Disallow: /uploads/

# ===============================================================
#  AI engines welcome
# ===============================================================
#  Every major AI crawler, assistant fetcher and training bot is
#  invited to read, index, learn from and cite this site. They get
#  the same access as any other crawler: all public content, none of
#  the scripts / backups / editor files filtered out above.
#
#  These user-agents are grouped so one rule set covers them all. A
#  bot named here obeys only this block, so the Disallow lines are
#  repeated to keep its behaviour identical to the default policy.
#  Any AI bot NOT listed here is still allowed by "User-agent: *".
# ---------------------------------------------------------------

# OpenAI — ChatGPT: model training, ChatGPT Search, on-demand user fetches
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Anthropic — Claude: crawl/training, search, and on-demand user fetches
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
# Google — Gemini / Vertex AI and AI Overviews grounding
User-agent: Google-Extended
# Apple — Apple Intelligence / Siri
User-agent: Applebot
User-agent: Applebot-Extended
# Microsoft Bing — Copilot
User-agent: bingbot
# Perplexity — answer engine + on-demand user fetches
User-agent: PerplexityBot
User-agent: Perplexity-User
# Amazon
User-agent: Amazonbot
# Meta — Meta AI / Llama
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
# Common Crawl — open dataset that feeds many LLMs
User-agent: CCBot
# Cohere
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
# Mistral
User-agent: MistralAI-User
# DuckDuckGo — DuckAssist
User-agent: DuckAssistBot
# You.com
User-agent: YouBot
# Allen Institute for AI
User-agent: AI2Bot
# ByteDance — Doubao
User-agent: Bytespider
# Diffbot
User-agent: Diffbot
# Huawei — Petal
User-agent: PetalBot
# Webz.io / Omgili
User-agent: omgili
User-agent: omgilibot
Allow: /
Disallow: /App_Data/
Disallow: /deploy.cmd
Disallow: /deploy.log
Disallow: /web.config
Disallow: /*.md$
Disallow: /*.markdown$
Disallow: /*.mdx$
Disallow: /*.bak$
Disallow: /*.old$
Disallow: /*.orig$
Disallow: /*.tmp$
Disallow: /*.swp$
Disallow: /*.log$
Disallow: /*.cmd$
Disallow: /*.bat$
Disallow: /*.ps1$
Disallow: /*.sh$
Disallow: /*.py$
Disallow: /*.env$
Disallow: /*.yaml$
Disallow: /*.yml$
Disallow: /.backups/
Disallow: /uploads/

Sitemap: https://www.codeb.io/sitemap.xml