# =========================
# AI Crawlers (AEO optimized)
# =========================

User-agent: GPTBot
Allow: /
Allow: /browse
Allow: /pricing
Allow: /about
Allow: /tools
Disallow: /admin/
Disallow: /auth/
Disallow: /login
Disallow: /signup
Disallow: /saved/
Disallow: /plan-billing
Disallow: /profile
Disallow: /payment/
Disallow: /invoices/
Disallow: /brand/*/
Disallow: /affiliate

User-agent: OAI-SearchBot
Allow: /
Allow: /browse
Allow: /pricing
Allow: /about
Allow: /tools
Disallow: /admin/
Disallow: /auth/
Disallow: /login
Disallow: /signup
Disallow: /saved/
Disallow: /plan-billing
Disallow: /profile
Disallow: /payment/
Disallow: /invoices/
Disallow: /brand/*/
Disallow: /affiliate

User-agent: ChatGPT-User
Allow: /
Allow: /browse
Allow: /pricing
Allow: /about
Allow: /tools
Disallow: /admin/
Disallow: /auth/
Disallow: /login
Disallow: /signup
Disallow: /saved/
Disallow: /plan-billing
Disallow: /profile
Disallow: /payment/
Disallow: /invoices/
Disallow: /brand/*/
Disallow: /affiliate

User-agent: ClaudeBot
Allow: /
Allow: /browse
Allow: /pricing
Allow: /about
Allow: /tools
Disallow: /admin/
Disallow: /auth/
Disallow: /login
Disallow: /signup
Disallow: /saved/
Disallow: /plan-billing
Disallow: /profile
Disallow: /payment/
Disallow: /invoices/
Disallow: /brand/*/
Disallow: /affiliate

User-agent: anthropic-ai
Allow: /
Allow: /browse
Allow: /pricing
Allow: /about
Allow: /tools
Disallow: /admin/
Disallow: /auth/
Disallow: /login
Disallow: /signup
Disallow: /saved/
Disallow: /plan-billing
Disallow: /profile
Disallow: /payment/
Disallow: /invoices/
Disallow: /brand/*/
Disallow: /affiliate

User-agent: PerplexityBot
Allow: /
Allow: /browse
Allow: /pricing
Allow: /about
Allow: /tools
Disallow: /admin/
Disallow: /auth/
Disallow: /login
Disallow: /signup
Disallow: /saved/
Disallow: /plan-billing
Disallow: /profile
Disallow: /payment/
Disallow: /invoices/
Disallow: /brand/*/
Disallow: /affiliate

User-agent: Applebot-Extended
Allow: /
Allow: /browse
Allow: /pricing
Allow: /about
Allow: /tools
Disallow: /admin/
Disallow: /auth/
Disallow: /login
Disallow: /signup
Disallow: /saved/
Disallow: /plan-billing
Disallow: /profile
Disallow: /payment/
Disallow: /invoices/
Disallow: /brand/*/
Disallow: /affiliate

User-agent: Meta-ExternalAgent
Allow: /
Allow: /browse
Allow: /pricing
Allow: /about
Allow: /tools
Disallow: /admin/
Disallow: /auth/
Disallow: /login
Disallow: /signup
Disallow: /saved/
Disallow: /plan-billing
Disallow: /profile
Disallow: /payment/
Disallow: /invoices/
Disallow: /brand/*/
Disallow: /affiliate

User-agent: Meta-ExternalFetcher
Allow: /
Allow: /browse
Allow: /pricing
Allow: /about
Allow: /tools
Disallow: /admin/
Disallow: /auth/
Disallow: /login
Disallow: /signup
Disallow: /saved/
Disallow: /plan-billing
Disallow: /profile
Disallow: /payment/
Disallow: /invoices/
Disallow: /brand/*/
Disallow: /affiliate


# =========================
# AI Training (optional control)
# =========================

User-agent: Google-Extended
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: CCBot
Disallow: /


# =========================
# Traditional SEO Crawlers
# =========================

User-agent: Googlebot
Allow: /
Allow: /browse
Allow: /pricing
Allow: /about
Allow: /tools
Disallow: /admin/
Disallow: /auth/
Disallow: /login
Disallow: /signup
Disallow: /saved/
Disallow: /plan-billing
Disallow: /profile
Disallow: /payment/
Disallow: /invoices/
Disallow: /brand/*/
Disallow: /affiliate

User-agent: Bingbot
Allow: /
Allow: /browse
Allow: /pricing
Allow: /about
Allow: /tools
Disallow: /admin/
Disallow: /auth/
Disallow: /login
Disallow: /signup
Disallow: /saved/
Disallow: /plan-billing
Disallow: /profile
Disallow: /payment/
Disallow: /invoices/
Disallow: /brand/*/
Disallow: /affiliate


# =========================
# Default fallback
# =========================

User-agent: *
Allow: /
Allow: /browse
Allow: /pricing
Allow: /about
Allow: /tools
Disallow: /admin/
Disallow: /auth/
Disallow: /login
Disallow: /signup
Disallow: /saved/
Disallow: /plan-billing
Disallow: /profile
Disallow: /payment/
Disallow: /invoices/
Disallow: /brand/*/
Disallow: /affiliate


# =========================
# Sitemap
# =========================

Sitemap: https://bencma.com/sitemap.xml