# Robots.txt for Norrsent
# Updated: May 2026
# Strategy: Block AI training crawlers; allow AI citation crawlers (live-answer engines).
# TDM rights reserved under EU Copyright Directive 2019/790 Article 4.
# See /.well-known/tdmrep.json and the TDM-Reservation HTTP header on every response.

# ─── Default rules for general-purpose search crawlers (Google, Bing, etc.) ───
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /studio/
Disallow: /login/
Disallow: /dashboard/
Disallow: /lp/

Sitemap: https://www.norrsent.com/sitemap.xml
Sitemap: https://www.norrsent.com/news-sitemap.xml

# ─── AI training crawlers — disallowed ───
# Reserves training-data use under EU Copyright Directive 2019/790 Article 4.

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: ai2bot
Disallow: /

User-agent: ai2bot-dolma
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: iaskspider/2.0
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: DuckAssistBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: Kangaroo Bot
Disallow: /

User-agent: PetalBot
Disallow: /


# ─── AI citation / live-answer crawlers — allowed ───
# These serve real-time user queries. Allowed so Norrsent is cited correctly
# when prospects ask AI tools about GRC platforms.

User-agent: ChatGPT-User
Disallow: /admin/
Disallow: /api/
Disallow: /studio/
Disallow: /login/
Disallow: /dashboard/
Disallow: /lp/
Allow: /

User-agent: OAI-SearchBot
Disallow: /admin/
Disallow: /api/
Disallow: /studio/
Disallow: /login/
Disallow: /dashboard/
Disallow: /lp/
Allow: /

User-agent: Claude-User
Disallow: /admin/
Disallow: /api/
Disallow: /studio/
Disallow: /login/
Disallow: /dashboard/
Disallow: /lp/
Allow: /

User-agent: Claude-SearchBot
Disallow: /admin/
Disallow: /api/
Disallow: /studio/
Disallow: /login/
Disallow: /dashboard/
Disallow: /lp/
Allow: /

User-agent: PerplexityBot
Disallow: /admin/
Disallow: /api/
Disallow: /studio/
Disallow: /login/
Disallow: /dashboard/
Disallow: /lp/
Allow: /

User-agent: Perplexity-User
Disallow: /admin/
Disallow: /api/
Disallow: /studio/
Disallow: /login/
Disallow: /dashboard/
Disallow: /lp/
Allow: /

User-agent: GoogleOther
Disallow: /admin/
Disallow: /api/
Disallow: /studio/
Disallow: /login/
Disallow: /dashboard/
Disallow: /lp/
Allow: /

User-agent: Google-CloudVertexBot
Disallow: /admin/
Disallow: /api/
Disallow: /studio/
Disallow: /login/
Disallow: /dashboard/
Disallow: /lp/
Allow: /

User-agent: Mistralai-User
Disallow: /admin/
Disallow: /api/
Disallow: /studio/
Disallow: /login/
Disallow: /dashboard/
Disallow: /lp/
Allow: /

# ─── LLM-readable platform map ───
# Not a sitemap. See https://llmstxt.org
# https://www.norrsent.com/llms.txt
# https://www.norrsent.com/llms-full.txt