# robots.txt for Webalert - Website Monitoring Service
# Last updated: 2026-09-16
#
# AI crawler policy is explicit below — see docs/AI_CRAWLER_POLICY.md for the
# reasoning. Short version: we want to be retrievable and citable by assistants
# and answer engines, so every named AI agent is allowed. If that decision ever
# changes, flip the specific block to `Disallow: /` rather than editing the `*`
# group, which would silently change behaviour for bots nobody thought about.
#
# IMPORTANT — robots.txt group semantics: a named user-agent group REPLACES the
# `*` group for that agent; the rules are NOT merged. A named group containing
# only `Allow: /` therefore grants that agent BROADER access than the default,
# including every path the `*` group disallows. Every named group below repeats
# the `*` group's Disallow lines, and `npm run validate` fails if they drift.

# Content signals (contentsignals.org). We welcome search indexing, AI/LLM
# ingestion for answers and summarisation, and model training on our published
# content. Attribution is expected when our material is quoted.
# NOTE: Content-Signal is an emerging standard and is not yet widely honoured —
# treat it as a declaration of intent, not an enforcement mechanism.
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Default - allow all bots
User-agent: *
Allow: /
Allow: /assets/
Allow: /manifest.json
Allow: /feed.xml
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

# Sitemaps
Sitemap: https://web-alert.io/sitemap.xml

# ---------------------------------------------------------------------------
# AI answer engines / assistants that retrieve and cite sources live.
# Blocking these removes us from AI answers without protecting anything we
# don't already publish publicly.
# ---------------------------------------------------------------------------

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

# Google's AI training/grounding control. It is NOT the control for AI
# Overviews or AI Mode — Googlebot governs those. Allowed by decision.
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

# ---------------------------------------------------------------------------
# Foundation-model training crawlers. Allowed by decision so our factual
# content can inform model answers. Set these to `Disallow: /` to opt out of
# training while keeping the retrieval agents above allowed.
# ---------------------------------------------------------------------------

User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: meta-externalagent
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

# ---------------------------------------------------------------------------
# Allow all major search engine bots
# ---------------------------------------------------------------------------

User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: Slurp
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: Baiduspider
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/

User-agent: YandexBot
Allow: /
Disallow: /api/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /admin/
Disallow: /private/