# StoreLister Marketing Site - robots.txt
#
# Posture: maximize indexing and citation surface across classic search
# (Google, Bing, Apple) AND generative answer engines (AI Overviews, AI Mode,
# ChatGPT Search, Perplexity, Claude with web, Microsoft Copilot, Gemini,
# Meta AI, Brave AI, DuckDuckGo Assist).
#
# Note: blocking a training crawler (GPTBot, ClaudeBot, Google-Extended,
# Applebot-Extended, Meta-ExternalAgent) does NOT block the corresponding
# AI search/citation crawler. Search inclusion and training are separate
# pipelines per vendor docs. Recovery posture: allow all named crawlers.

User-agent: *
Allow: /

Sitemap: https://storelister.com/sitemap.xml

# ─── Classic search engines ───
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# ─── OpenAI ───
# GPTBot: training. OAI-SearchBot: ChatGPT Search index. ChatGPT-User:
# user-initiated browse fetches (ignores robots.txt per OpenAI policy
# but listed for completeness and explicit intent).
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-AdsBot
Allow: /

# ─── Anthropic ───
# ClaudeBot: training. Claude-SearchBot: search index. Claude-User:
# user-initiated fetches (Anthropic honors robots.txt for Claude-User
# unlike OpenAI/Perplexity).
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

# ─── Perplexity ───
# PerplexityBot: search index. Perplexity-User: user-initiated fetches
# (ignores robots.txt per published policy; listed for explicit intent).
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ─── Google AI training / grounding ───
# Google-Extended controls Gemini Apps grounding and AI training only.
# It does NOT affect Google AI Overviews or AI Mode (those use Googlebot).
User-agent: Google-Extended
Allow: /

# ─── Apple AI training ───
# Applebot-Extended controls re-use of Applebot-crawled content for
# Apple Intelligence training. Does NOT affect Siri / Spotlight inclusion.
User-agent: Applebot-Extended
Allow: /

# ─── Meta AI ───
User-agent: Meta-WebIndexer
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: facebookexternalhit
Allow: /

# ─── Brave Search / AI ───
User-agent: Bravebot
Allow: /

# ─── ByteDance (TikTok/Doubao) ───
User-agent: Bytespider
Allow: /

# ─── Common Crawl (feeds many open models) ───
User-agent: CCBot
Allow: /