# Funbit AS - robots.txt
# Allow-by-default block-list. Known AI / answer-engine, classical
# search, and social-preview crawlers are STACKED into groups so they
# all share the same private-path Disallow rules - naming them signals
# intent (we want AI discovery) and, per RFC 9309, is required for them
# to inherit the blocks at all (a named group does NOT fall through to
# ``*``). The agent lists live in views.py (ROBOTS_*_AGENTS).

# 1. Bulk crawlers & AI scrapers - wildcard fallback + heavy
#    training/index bots. Standard Disallows, 5s crawl-delay.
User-agent: *
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Google-CloudVertexBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: CCBot
User-agent: Amazonbot
User-agent: Bytespider
User-agent: DuckAssistBot
User-agent: TavilyBot
User-agent: Bravebot
User-agent: Diffbot
User-agent: cohere-ai
User-agent: Cohere-Training-Data-Crawler
User-agent: xAI-Bot
User-agent: YouBot
User-agent: omgili
User-agent: AI2Bot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: AdsBot-Google-Mobile
User-agent: DotBot
User-agent: FunbitDashboardCrawler
User-agent: MJ12bot
User-agent: SemrushBot
User-agent: SiteAuditBot
User-agent: Uptime-Kuma
Disallow: /admin
Disallow: /django-admin
Disallow: /api/
Disallow: /signed-s3-uploads/
Disallow: /inbound/
Disallow: /qr
Disallow: /documents/
Crawl-delay: 5

# 2. On-demand fetchers & social previews - user-triggered single-page
#    fetches for chat answers / link unfurls. No crawl-delay so previews
#    don't break.
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: Google-Agent
User-agent: Google-NotebookLM
User-agent: facebookexternalhit
User-agent: meta-externalfetcher
User-agent: LinkedInBot
User-agent: Twitterbot
User-agent: Discordbot
User-agent: TelegramBot
User-agent: WhatsApp
User-agent: Pinterestbot
User-agent: Mastodon
User-agent: Slackbot-LinkExpanding
Disallow: /admin
Disallow: /django-admin
Disallow: /api/
Disallow: /signed-s3-uploads/
Disallow: /inbound/
Disallow: /qr
Disallow: /documents/

# 3. Classical search engines - paced fastest (1s). Bing honours
#    Crawl-delay strictly; Google ignores it but is named to keep the
#    same Disallow set + clear intent in Search Console.
User-agent: Bingbot
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Googlebot-News
Disallow: /admin
Disallow: /django-admin
Disallow: /api/
Disallow: /signed-s3-uploads/
Disallow: /inbound/
Disallow: /qr
Disallow: /documents/
Crawl-delay: 1

Sitemap: https://funbit.no/sitemap.xml
Sitemap: https://funbit.no/news-sitemap.xml

# AI/LLM context map: https://funbit.no/llms.txt
# Full content dump (Markdown): https://funbit.no/llms-full.txt
# Structured fact-sheet (JSON-LD ItemList of Claim nodes):
#   https://funbit.no/funbit-facts.json
# Markdown facts (same data as the JSON above):
#   https://funbit.no/llms-claims.txt
# Formal AI usage policy (permissions, attribution):
#   https://funbit.no/.well-known/ai-policy.json
# TDM Reservation Protocol (EU DSM Art. 4 mining reservation):
#   https://funbit.no/.well-known/tdmrep.json