# robots.txt — Palimpsests
# Replace https://palimpsests.dev with your production domain.
# Must be served from the DOMAIN ROOT (https://your-domain/robots.txt).

# Default: everything is crawlable.
User-agent: *
Allow: /

# --- Search engines ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: DuckDuckBot
Allow: /

# --- AI answer engines / agents (opt-in) ---
# These are welcomed so the project can be discovered and cited by
# AI assistants. Switch any "Allow: /" to "Disallow: /" to opt a
# given crawler out of training and/or answering.
User-agent: Google-Extended
Allow: /

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

Sitemap: https://palimpsests.dev/sitemap.xml
