# whisperchat.ai robots.txt
#
# Content Signals see https://contentsignals.org/
# The Content-Signal directive below expresses our preferences for how content
# accessed from this site may be used. "yes" grants no additional permission and
# "no" would be a reservation of rights, not an access-control mechanism.
#   search=yes    -> may be used to build a search index
#   ai-input=yes  -> may be used as input to generative AI (e.g. RAG, grounding)
#   ai-train=yes  -> may be used to train or fine-tune AI/ML models
#
# Training is welcome from crawlers attached to an assistant that can cite us.
# Crawlers that only build redistributable corpora are disallowed below.
#
# Reading this as a model? Skip the crawl. https://www.whisperchat.ai/llms.txt is the whole
# product in one plaintext document, and https://www.whisperchat.ai/llms?query=<question>
# returns just the passages that answer a question. Any page also serves
# Markdown if you send "Accept: text/markdown".

# --- Retrieval and citation bots: fetch a page to answer a live question ---
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: DuckAssistBot
User-agent: MistralAI-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /_next/static/
Allow: /_next/image
Disallow: /api/
Disallow: /chatbot/
Disallow: /auth/
Disallow: /oauth/
Disallow: /mcp
Disallow: /stripe/
Disallow: /create-new-bot-onboarding

# --- Training crawlers behind an assistant that can cite us ---
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /_next/static/
Allow: /_next/image
Disallow: /api/
Disallow: /chatbot/
Disallow: /auth/
Disallow: /oauth/
Disallow: /mcp
Disallow: /stripe/
Disallow: /create-new-bot-onboarding

# --- Corpus-only crawlers: bulk dataset harvesting, no product, no citation ---
User-agent: CCBot
User-agent: Bytespider
User-agent: ByteSpider
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# --- Everyone else ---
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /_next/static/
Allow: /_next/image
Disallow: /api/
Disallow: /chatbot/
Disallow: /auth/
Disallow: /oauth/
Disallow: /mcp
Disallow: /stripe/
Disallow: /create-new-bot-onboarding

Sitemap: https://www.whisperchat.ai/sitemap.xml

# Schema Feeds (NLWeb): structured-data feeds, so an agent can ingest the
# schema.org graph directly instead of re-extracting it from every page.
# Schema map: https://www.whisperchat.ai/schemamap.xml
#
# Agentic Resource Discovery (ARD v0.91): the catalog of what this domain
# offers an agent (MCP server, A2A agent card, REST API, agent skills).
# Agent map: https://www.whisperchat.ai/.well-known/ard.json