# robots.txt for bzzip.com
#
# ONE GROUP, NAMING EVERY CRAWLER, AND THE REASON IT MUST STAY THAT WAY.
#
# This file used to end with eight little groups like this one:
#
#     User-agent: GPTBot
#     Allow: /
#
# written to say "AI crawlers are welcome". They were welcome, and they were
# also handed the entire private surface of the application. A crawler obeys
# EXACTLY ONE group — the most specific one whose user-agent matches — and it
# does NOT inherit from `User-agent: *`. So naming GPTBot in a group of its own
# did not add a permission to the shared rules; it replaced them. Every AI
# crawler was reading a rule set whose only line was `Allow: /`, which let it
# walk /admin12345, /dashboard, /billing and /invoices.
#
# The fix is that there is now one group and every agent is listed on it.
# Multiple `User-agent:` lines before a single rule block is the standard way to
# say "these all share these rules" (RFC 9309 §2.2.1). The AI crawlers are still
# welcome — nothing here disallows the public pages — but "welcome" now means
# the same thing for them as for Googlebot.
#
# ## If you add a crawler
#
# Add its name to the list below. Do NOT give it a group of its own, however
# tempting the symmetry looks: a group of its own is exactly the bug above.
#
# ## If you add a private route
#
# Add BOTH lines. `Disallow: /foo` does not match `/es/foo`, and `/*/foo` does
# not match `/foo` — the router mounts almost everything inside
# `scope "(:locale)"`, so a private path has five spellings and the wildcard
# covers four of them.

User-agent: *
User-agent: Googlebot
User-agent: Bingbot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Google-Extended
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: CCBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Bytespider

# Public pages are allowed: everything not named below.
Allow: /

# --- Operator surface -------------------------------------------------------
Disallow: /admin12345
Disallow: /admin/
Disallow: /*/admin/

# --- Signed-in product ------------------------------------------------------
Disallow: /dashboard
Disallow: /*/dashboard
Disallow: /workspaces
Disallow: /*/workspaces
Disallow: /conversations
Disallow: /*/conversations
Disallow: /notifications
Disallow: /*/notifications
Disallow: /search
Disallow: /*/search
Disallow: /settings
Disallow: /*/settings
Disallow: /team
Disallow: /*/team
Disallow: /agency
Disallow: /*/agency
Disallow: /profile
Disallow: /*/profile
Disallow: /notification_preferences
Disallow: /*/notification_preferences
Disallow: /quick_replies
Disallow: /*/quick_replies
Disallow: /message_templates
Disallow: /*/message_templates
Disallow: /sla_settings
Disallow: /*/sla_settings
Disallow: /business_hours
Disallow: /*/business_hours
Disallow: /analytics/
Disallow: /*/analytics/

# --- AI agent configuration -------------------------------------------------
Disallow: /suggestions
Disallow: /*/suggestions
Disallow: /knowledge_suggestions
Disallow: /*/knowledge_suggestions
Disallow: /ai_credential
Disallow: /*/ai_credential
Disallow: /ai_agent_setup
Disallow: /*/ai_agent_setup

# --- Billing ----------------------------------------------------------------
Disallow: /billing
Disallow: /*/billing
Disallow: /subscription
Disallow: /*/subscription
Disallow: /invoices
Disallow: /*/invoices
Disallow: /payment_methods
Disallow: /*/payment_methods
Disallow: /checkout/
Disallow: /*/checkout/

# --- Authentication and onboarding ------------------------------------------
Disallow: /session
Disallow: /*/session
Disallow: /sign_in
Disallow: /*/sign_in
Disallow: /sign_up
Disallow: /*/sign_up
Disallow: /passwords
Disallow: /*/passwords
Disallow: /confirmations
Disallow: /*/confirmations
Disallow: /change-email
Disallow: /*/change-email
Disallow: /invitations
Disallow: /*/invitations
Disallow: /onboarding
Disallow: /*/onboarding
Disallow: /welcome
Disallow: /*/welcome
Disallow: /first-workspace
Disallow: /*/first-workspace

# --- Form endpoints (POST targets with no readable GET) ----------------------
Disallow: /waitlist
Disallow: /*/waitlist
Disallow: /concierge_requests
Disallow: /*/concierge_requests
Disallow: /support_tickets
Disallow: /*/support_tickets
Disallow: /platform_waitlist

# --- Framework and machine endpoints ----------------------------------------
Disallow: /oauth/
Disallow: /webhooks/
Disallow: /rails/
Disallow: /up

Sitemap: https://bzzip.com/sitemap.xml