# Crawl policy (2026-08-20, previews added 2026-10-03): search engines, Anthropic,
# OpenAI and link-preview fetchers are welcome;
# every other bot is disallowed. Named groups below override the * group.

# --- Search engines ---
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Applebot
# --- Anthropic ---
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: anthropic-ai
# --- OpenAI ---
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# --- Link previews (2026-10-03): without these, a shared link shows no card ---
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Slackbot
User-agent: Discordbot
User-agent: TelegramBot
User-agent: WhatsApp
Allow: /
# Don't hammer the API endpoints
Disallow: /api/
# Heavy mirrored media: crawl the reader pages instead (/r/<slug> serves the
# full OCR text of every document). Thumbnails stay crawlable for previews.
Disallow: /data/pdfs/
Disallow: /data/videos/
Disallow: /data/audios/
Disallow: /data/extracted/
Disallow: /data/images/
Disallow: /data/prior/
Allow: /data/thumbs/
Allow: /data/extracted_thumbs/

# --- Everyone else: no crawling ---
User-agent: *
Disallow: /

# Static on purpose (not generated from inc/site.php): if a rewrite rule were
# ever missing, a 404 here would read as "crawl everything". Update this line
# with SITE_URL on a rename.
Sitemap: https://declassifieduap.org/sitemap.xml