# zapit.work — open to crawlers, including AI training and answer-engine bots.
#
# ONE rule group shared by every agent listed below (RFC 9309 multi-agent
# group). Do not split bots back into their own `User-agent: X / Allow: /`
# stubs: a named group REPLACES `*` for that bot, so a stub would silently
# exempt it from the Disallows (that's how Googlebot ended up crawling
# /oauth/google/login and reporting it in Search Console).

User-agent: *

# OpenAI
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User

# Anthropic
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai

# Google
User-agent: Googlebot
User-agent: Google-Extended
User-agent: GoogleOther

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User

# Common Crawl (training data for many LLMs)
User-agent: CCBot

# Apple
User-agent: Applebot
User-agent: Applebot-Extended

# Microsoft / Bing
User-agent: Bingbot

# DuckDuckGo
User-agent: DuckAssistBot
User-agent: DuckDuckBot

# Meta
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher

# Other AI / answer engines
User-agent: cohere-ai
User-agent: Mistral-User
User-agent: YouBot
User-agent: Amazonbot
User-agent: Diffbot
User-agent: Bytespider
User-agent: PetalBot

# App/auth plumbing — crawling it only burns budget and pollutes GSC:
# /oauth/ 307s to Google's consent screen, /login and /auth/ are the sign-in
# UI, /api/ is JSON endpoints. All also send X-Robots-Tag: noindex via the
# nginx catch-all.
Allow: /
Disallow: /chat
Disallow: /oauth/
Disallow: /login
Disallow: /auth/
Disallow: /api/

Sitemap: https://zapit.work/sitemap.xml