# UGC Copilot — robots.txt
# Strategy: allow indexing, AI citation, AND selective training opt-in.
# The major three training crawlers (GPTBot, Google-Extended, Applebot-Extended)
# are opted IN on all paths except /compare-* — comparison pages stay out of
# training data to protect competitive positioning. Long-tail training scrapers
# remain blocked. Policy decision logged 2026-05-01;
# see docs/distribution-strategy-priorities.md for rationale.

# -----------------------------------------------------------------------------
# Default rules — apply to all crawlers including search engines (Googlebot,
# Bingbot, DuckDuckBot, etc.) and AI crawlers without an explicit block below.
# -----------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /auth/
Disallow: /oauth-consent
Disallow: /oauth-consent/
Disallow: /__/

# -----------------------------------------------------------------------------
# AI training crawlers — major three opted IN (with /compare protection).
# These power ChatGPT (GPTBot), Gemini and AI Overviews (Google-Extended), and
# Apple Intelligence (Applebot-Extended) — the highest-leverage surfaces for
# durable brand recall in future model snapshots. /compare-* is disallowed to
# keep competitive comparison pages out of training data.
# -----------------------------------------------------------------------------
User-agent: GPTBot
Allow: /
Disallow: /compare
Disallow: /compare/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /auth/
Disallow: /oauth-consent
Disallow: /oauth-consent/
Disallow: /__/

User-agent: Google-Extended
Allow: /
Disallow: /compare
Disallow: /compare/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /auth/
Disallow: /oauth-consent
Disallow: /oauth-consent/
Disallow: /__/

User-agent: Applebot-Extended
Allow: /
Disallow: /compare
Disallow: /compare/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /auth/
Disallow: /oauth-consent
Disallow: /oauth-consent/
Disallow: /__/

# -----------------------------------------------------------------------------
# AI training crawlers — long tail still blocked. Lower brand-recall value,
# less transparent training-set practices, and several are content scrapers
# (Common Crawl resale, Diffbot data products) more than first-party model
# trainers. Revisit individually as practices evolve.
# -----------------------------------------------------------------------------
User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Claude-Web
Disallow: /

# -----------------------------------------------------------------------------
# AI search / citation crawlers — explicitly ALLOWED so we appear in
# answer engines (ChatGPT search, Perplexity, Claude search, Bing Copilot).
# -----------------------------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /auth/
Disallow: /oauth-consent
Disallow: /oauth-consent/
Disallow: /__/

User-agent: ChatGPT-User
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /auth/
Disallow: /oauth-consent
Disallow: /oauth-consent/
Disallow: /__/

User-agent: PerplexityBot
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /auth/
Disallow: /oauth-consent
Disallow: /oauth-consent/
Disallow: /__/

User-agent: Perplexity-User
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /auth/
Disallow: /oauth-consent
Disallow: /oauth-consent/
Disallow: /__/

User-agent: ClaudeBot
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /auth/
Disallow: /oauth-consent
Disallow: /oauth-consent/
Disallow: /__/

User-agent: Claude-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /auth/
Disallow: /oauth-consent
Disallow: /oauth-consent/
Disallow: /__/

User-agent: Applebot
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /auth/
Disallow: /oauth-consent
Disallow: /oauth-consent/
Disallow: /__/

# -----------------------------------------------------------------------------
# Forward-looking signal (IETF draft draft-romm-aipref-contentsignals).
# Not yet honored by major crawlers — explicit User-agent rules above are the
# enforcement mechanism — but harmless to include for future compatibility.
# -----------------------------------------------------------------------------
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# -----------------------------------------------------------------------------
# Sitemap and AI documentation pointers
# -----------------------------------------------------------------------------
Sitemap: https://ugccopilot.ai/sitemap.xml

# llms.txt        — structured product summary for AI crawlers
# llms-full.txt   — comprehensive product documentation
# Tracking parameter URLs (utm_*, ref=*) intentionally NOT disallowed —
# blocking them triggers "Indexed, though blocked by robots.txt" warnings in
# Search Console; canonical tags handle dedup instead.