# robots.txt for Quilter — https://www.quilter.ai
# Last reviewed: 2026-05
#
# Companion files:
#   - https://www.quilter.ai/sitemap.xml   (search engine sitemap)
#   - https://www.quilter.ai/llms.txt      (AI/LLM context index, llmstxt.org)

# ============================================================
# Default — applies to all crawlers not explicitly matched below
# ============================================================
User-agent: *
Allow: /

# Low-value / transactional pages — keep out of search results
Disallow: /401
Disallow: /thank-you

# ============================================================
# AI crawlers
# ============================================================
# Quilter wants to be discoverable by AI-powered search and coding
# assistants (ChatGPT, Claude, Perplexity, Google AI Mode, etc.),
# so all major AI bots are ALLOWED below.
#
# To opt OUT of AI model training while staying discoverable in
# AI-powered search, change "Allow: /" to "Disallow: /" under the
# bots labeled (training). Leave bots labeled (search/RAG) and
# (live browsing) allowed — those are how citations and references
# of Quilter content get generated in chat answers.

# --- OpenAI ---
User-agent: GPTBot
Allow: /
# (training)

User-agent: OAI-SearchBot
Allow: /
# (search/RAG)

User-agent: ChatGPT-User
Allow: /
# (live browsing on a user's behalf)

# --- Anthropic ---
User-agent: ClaudeBot
Allow: /
# (training)

User-agent: anthropic-ai
Allow: /
# (legacy)

User-agent: Claude-Web
Allow: /
# (live browsing on a user's behalf)

# --- Google AI ---
# Note: Googlebot (regular search indexing) is covered by User-agent: *
# Google-Extended controls Gemini and Vertex AI training only.
User-agent: Google-Extended
Allow: /
# (Gemini/Vertex AI training)

# --- Perplexity ---
User-agent: PerplexityBot
Allow: /
# (search/RAG)

User-agent: Perplexity-User
Allow: /
# (live browsing on a user's behalf)

# --- Apple ---
User-agent: Applebot-Extended
Allow: /
# (Apple Intelligence training)

# --- Meta ---
User-agent: Meta-ExternalAgent
Allow: /
# (training)

# --- Mistral ---
User-agent: MistralAI-User
Allow: /

# --- Common Crawl (used by many AI labs to bootstrap datasets) ---
User-agent: CCBot
Allow: /

# ============================================================
# Aggressive / noncompliant scrapers
# ============================================================
# ByteDance/TikTok — widely reported as aggressive and noncompliant
# with crawl-delay. Negligible discovery cost to block.
User-agent: Bytespider
Disallow: /

# Huawei Petal — high-volume, low-traffic-return crawler.
User-agent: PetalBot
Disallow: /

# Majestic — SEO backlink crawler. Block unless you actively use Majestic.
User-agent: MJ12bot
Disallow: /

# DotBot / Moz — SEO crawler. Block unless you actively use Moz.
User-agent: DotBot
Disallow: /

# ============================================================
# Sitemaps & companion indexes
# ============================================================
Sitemap: https://www.quilter.ai/sitemap.xml

Sitemap: https://www.quilter.ai/sitemap.xml