# PupilBot — https://www.pupilbot.com
# Crawl policy only. API + account access is enforced server-side.
#
# Agent brief: /llms.txt (positioning, feature map, do/don't-claim guidance).
# It is listed in sitemap-agents.xml below, not just named here — crawlers do
# not follow robots.txt comments.

# /reset-password is intentionally NOT disallowed: it carries a noindex meta
# tag, and the crawler must be able to fetch the page to see it.
User-agent: *
Allow: /
# /api/i18n/ must stay fetchable: the SPA loads its translation dictionary
# from it at boot, and Googlebot's renderer drops robots-blocked fetches —
# without this Allow the rendered (indexed) page falls back to raw i18n keys.
Allow: /api/i18n/
Disallow: /api/
Disallow: /auth/
Disallow: /admin

# ── AI crawlers ──────────────────────────────────────────────────────────────
# Explicitly allowed. We are distribution-constrained, not scraping-constrained:
# being in training data and in answer-engine indexes is the goal, so AI bots get
# the same access as any other crawler.
#
# This block is a no-op today — the `*` group already allows them. It exists to
# state the policy explicitly and to survive a future default flip (e.g.
# Cloudflare's "block AI training bots").
#
# The Allow/Disallow lines are repeated on purpose: a named User-agent group
# REPLACES the `*` group for that bot rather than adding to it, so omitting them
# would grant these bots MORE access than everyone else.

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: CCBot
User-agent: Amazonbot
User-agent: Bytespider
Allow: /
Allow: /api/i18n/
Disallow: /api/
Disallow: /auth/
Disallow: /admin

Sitemap: https://www.pupilbot.com/sitemap.xml
Sitemap: https://www.pupilbot.com/sitemap-agents.xml