# OverSkill robots.txt
# Updated May 2026 — explicit AI bot rules + agent discovery
# See https://www.robotstxt.org/robotstxt.html for documentation.
#
# We explicitly invite AI agents to read marketing/SEO content (templates,
# integrations, /build/* landing pages) and our agent skill files. The
# editor (/account/*), API surface (/api/*), and OAuth flows are blocked
# from training crawlers — those endpoints are for live agent use, not
# for being scraped into a training corpus.

# ============================================================================
# All crawlers (default policy)
# ============================================================================
User-agent: *
Disallow: /account/
Disallow: /admin/
Disallow: /api/
Disallow: /oauth/
Disallow: /auth/
Disallow: /webhooks/
Disallow: /users/
Disallow: /cable
Disallow: /lo122025
Disallow: /a/
Allow: /

# ============================================================================
# AI training/inference crawlers — granular rules
# ============================================================================
# We allow crawlers to read public marketing/SEO pages and the agent skill
# files (SKILL.md, llms.txt, llms-full.txt) so end-user agents can discover
# our capabilities. Mirror of the default policy above for backstops.

User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: FacebookBot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: CCBot
Disallow: /account/
Disallow: /admin/
Disallow: /api/
Disallow: /oauth/
Disallow: /auth/
Disallow: /webhooks/
Disallow: /users/
Disallow: /cable
Allow: /SKILL.md
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/
Allow: /

# ============================================================================
# Cloudflare Content Signals (2025 standard)
# ============================================================================
# Tells crawlers what they may do with our content. We allow search indexing
# and AI agent input (live use), but disallow training. See:
# https://blog.cloudflare.com/content-signals/

Content-Signal: search=yes, ai-input=yes, ai-train=no

# ============================================================================
# Sitemaps
# ============================================================================
Sitemap: https://overskill.com/sitemap.xml