User-agent: *
Allow: /
# Content Signal policy for AI crawlers:
# search=yes -> classic search indexing is allowed
# ai-input=yes -> retrieval/grounding in AI answers is allowed
# ai-train=no -> model training on site content is not permitted
Content-Signal: search=yes, ai-input=yes, ai-train=no

# Language choice is stored client-side; canonical URLs stay query-free
Disallow: /api/
Disallow: /admin/
Disallow: /.well-known/
Disallow: /test/
Disallow: /debug/
Disallow: /chat$
Disallow: /chat/
Disallow: /search

# Allow important legal content paths
Allow: /use-cases/
Allow: /mietrecht-beratung
Allow: /arbeitsrecht-ki
Allow: /ki-rechtsassistent
Allow: /ki-fuer-rechtsfragen
Allow: /ki-fuer-anwaelte
Allow: /legal-tech-deutschland
Allow: /chatgpt-alternative-recht
Allow: /ki-alternative-zu-chatgpt-recht
Allow: /plattform/quellen
Allow: /specter-vs-chatgpt
Allow: /mietrecht-aenderungen-2026

# Explicitly allow AI search and answer engines to discover public content
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Allow model-grounding crawlers so public SpecterAI pages can be discovered,
# learned as an entity, and cited by AI answer engines.
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

# Block common unwanted crawlers and scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MegaIndex
Disallow: /

# Sitemap location
Sitemap: https://www.specterlaw.ai/sitemap.xml
Sitemap: https://www.specterlaw.ai/news-sitemap.xml

# LLMs.txt location for AI knowledge discovery
# Summary: https://www.specterlaw.ai/llms.txt
# Full: https://www.specterlaw.ai/llms-full.txt