# robots.txt — cloudarq.net
#
# Allow crawlers on every public page; block authenticated /
# administrative surfaces. Sitemap points to the canonical list of
# pre-rendered routes.
#
# Explicit Allow blocks for major AI crawlers below: belt-and-
# suspenders so future maintainers can't accidentally tighten the
# wildcard rule and re-block the AI bots we deliberately allow.
# Compliant AI assistants (Claude, ChatGPT, Gemini, Perplexity,
# Apple Intelligence, etc.) check user-agent-specific blocks first
# before falling back to the wildcard.

User-agent: *
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /admin/
Disallow: /api
Disallow: /api/
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success
Disallow: /verify-email
Disallow: /reset-password
Disallow: /forgot-password

# ── AI crawlers (explicit Allow) ──
# Same Disallow set as the wildcard above; explicitly allowing the
# rest. Cloudflare's "Manage AI bots" feature has been disabled at
# the zone level so the managed Disallow block is no longer
# injected. If anyone re-enables it, this section will be
# overridden — re-disable in Cloudflare → Bots → AI bots.

# OpenAI
User-agent: GPTBot
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success
Disallow: /verify-email
Disallow: /reset-password
Disallow: /forgot-password

User-agent: ChatGPT-User
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success

User-agent: OAI-SearchBot
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success

# Anthropic
User-agent: ClaudeBot
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success

User-agent: anthropic-ai
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success

User-agent: Claude-Web
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success

# Google AI (separate user-agent from Googlebot — this controls
# whether Google may use the content for Gemini training / grounding)
User-agent: Google-Extended
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success

User-agent: Perplexity-User
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success

# Common Crawl (open dataset, used for many model training pipelines)
User-agent: CCBot
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success

# DuckDuckGo / DuckAssist
User-agent: DuckAssistBot
Allow: /
Disallow: /app
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /w/
Disallow: /connections
Disallow: /audits
Disallow: /findings
Disallow: /billing
Disallow: /settings
Disallow: /inbox
Disallow: /api-docs
Disallow: /support
Disallow: /onboarding
Disallow: /checkout
Disallow: /setup/
Disallow: /change-password
Disallow: /signup-success

# ── End AI crawler block ──

# Crawlers occasionally re-query with querystrings that create
# duplicate content; the canonical link in each page resolves that
# at the search-engine side.

Sitemap: https://cloudarq.net/sitemap.xml