User-agent: *
Allow: /
# Private / non-indexable paths (admin, user dashboard, controlled User Guide, API, embeddable widget)
# NB: /demo.html is intentionally NOT disallowed — it is publicly linked (menu/footer) but
#     carries a `noindex, nofollow` meta tag. Crawlers must be allowed to FETCH it to SEE the
#     noindex; a robots.txt Disallow would block the crawl, hide the noindex, and risk the URL
#     being indexed bare (no snippet) purely from its inbound links. Keep it crawlable + noindex.
Disallow: /admin.html
Disallow: /dashboard.html
Disallow: /guide.html
Disallow: /api/
Disallow: /.netlify/
Disallow: /widget.html
Disallow: /widget.js

# AI crawlers — explicitly allowed everywhere EXCEPT the controlled User Guide
# (each bot uses its own group, so the Disallow under User-agent:* above does not
#  apply to them — it must be repeated here. /guide.html is more specific than
#  Allow:/, so it wins for that path while everything else stays allowed.)
User-agent: ClaudeBot
Allow: /
Disallow: /guide.html

User-agent: anthropic-ai
Allow: /
Disallow: /guide.html

User-agent: GPTBot
Allow: /
Disallow: /guide.html

User-agent: ChatGPT-User
Allow: /
Disallow: /guide.html

User-agent: PerplexityBot
Allow: /
Disallow: /guide.html

User-agent: Google-Extended
Allow: /
Disallow: /guide.html

User-agent: Applebot-Extended
Allow: /
Disallow: /guide.html

User-agent: cohere-ai
Allow: /
Disallow: /guide.html

# Sitemaps
Sitemap: https://auditspark.io/sitemap.xml