# CREBISO robots.txt — Optimized for Search Engines & AI Crawlers
# Updated: July 2026
#
# All crawlers are welcome — including AI / LLM crawlers:
#   GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, anthropic-ai, Claude-Web,
#   Google-Extended, PerplexityBot, CCBot, Amazonbot, Applebot, Applebot-Extended,
#   Meta-ExternalAgent, cohere-ai, Bytespider, DuckDuckBot.
#
# Note: a single "*" group is used so the Disallow rules below apply to ALL crawlers
# (named per-bot groups would exempt those bots from the Disallow rules per the robots spec).

User-agent: *
Allow: /
Disallow: /OLD/
Disallow: /docs/
Disallow: /.superpowers/
Crawl-delay: 1

# Sitemap
Sitemap: https://crebiso.com/sitemap.xml

# Machine-readable files for AI / LLM discovery:
#   https://crebiso.com/llms.txt
#   https://crebiso.com/llms-full.txt
#   https://crebiso.com/.well-known/ai-plugin.json
#   https://crebiso.com/ai.html

Host: https://crebiso.com