# www.sjorestaurant.no - robots.txt
# Allow-by-default block-list. AI, answer-engine, search and social-preview
# crawlers are STACKED into groups so they share one set of Disallow rules.
# Naming them signals intent (we want AI discovery) and, per RFC 9309, is
# required for them to inherit the blocks at all: a named group does NOT fall
# through to `*`. Agent lists live in discoverability/crawlers.py.

# 1. Bulk crawlers and AI indexers. Paced at 5s -- they fetch broadly.
#    The Content-Signal line (contentsignals.org) states permitted use per
#    axis; ai-train tracks the same setting as the policy files below.
User-agent: *
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Google-CloudVertexBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: CCBot
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: Bytespider
User-agent: QwenBot
User-agent: DuckAssistBot
User-agent: TavilyBot
User-agent: Bravebot
User-agent: Diffbot
User-agent: cohere-ai
User-agent: Cohere-Training-Data-Crawler
User-agent: xAI-Bot
User-agent: GrokBot
User-agent: xAI-Grok
User-agent: YouBot
User-agent: LinerBot
User-agent: omgili
User-agent: Omgilibot
User-agent: AI2Bot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: PetalBot
User-agent: DotBot
User-agent: MJ12bot
User-agent: SemrushBot
User-agent: SiteAuditBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /admin
Disallow: /django-admin
Disallow: /signed-s3-uploads/
Disallow: /inbound/
Disallow: /documents/
Disallow: /product-details/
Disallow: /load-more-products/
Crawl-delay: 5

# 2. On-demand fetchers and social previews: user-triggered single-page
#    fetches for a chat answer or a link unfurl. No Crawl-delay, or previews
#    break.
User-agent: ChatGPT-User
User-agent: OAI-AdsBot
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: Amzn-User
User-agent: Google-Agent
User-agent: Gemini-Notebook
User-agent: Google-NotebookLM
User-agent: facebookexternalhit
User-agent: meta-externalfetcher
User-agent: LinkedInBot
User-agent: Twitterbot
User-agent: Discordbot
User-agent: TelegramBot
User-agent: WhatsApp
User-agent: Pinterestbot
User-agent: Mastodon
User-agent: Slackbot-LinkExpanding
Disallow: /admin
Disallow: /django-admin
Disallow: /signed-s3-uploads/
Disallow: /inbound/
Disallow: /documents/
Disallow: /product-details/
Disallow: /load-more-products/

# 3. Classical search engines, paced fastest. Bing honours Crawl-delay;
#    Google ignores it but is named to carry the same Disallow set.
User-agent: Bingbot
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Googlebot-News
Disallow: /admin
Disallow: /django-admin
Disallow: /signed-s3-uploads/
Disallow: /inbound/
Disallow: /documents/
Disallow: /product-details/
Disallow: /load-more-products/
Crawl-delay: 1

Sitemap: https://www.sjorestaurant.no/sitemap.xml

# Formal AI usage policy (permissions, attribution, licensing contact):
#   https://www.sjorestaurant.no/.well-known/ai-policy.json
# Text-and-data-mining reservation (W3C TDMRep / EU DSM Art. 4):
#   https://www.sjorestaurant.no/.well-known/tdmrep.json

# AI/LLM context map: https://www.sjorestaurant.no/llms.txt
# Full content dump (Markdown): https://www.sjorestaurant.no/llms-full.txt
# Every page has a Markdown twin: strip the canonical URL's trailing slash and
# append .md (/meny/ -> /meny.md). A locale root keeps the slash (/ -> /.md,
# /en/ -> /en/.md). Every page also answers `Accept: text/markdown` on its
# canonical URL.