# ConsentEase - GDPR/CCPA Compliance Platform
# https://consent-ease.com
# Last updated: 2025-01-11

# Default crawler rules
User-agent: *
Allow: /
Allow: /features
Allow: /pricing
Allow: /faq
Allow: /demo
Allow: /plan-recommendation
Allow: /documentation
Allow: /help-center
Allow: /contact
Allow: /status
Allow: /cmp-partnership

# Allow crawling of regulation pages
Allow: /regulations/

# Allow crawling of alternative comparison pages (high SEO value)
Allow: /alternatives/

# Disallow admin and private areas
Disallow: /admin
Disallow: /dashboard
Disallow: /settings
Disallow: /websites
Disallow: /banner-designer
Disallow: /analytics
Disallow: /add-ons
Disallow: /cookie-policies/
Disallow: /feedback

# Disallow API endpoints
Disallow: /api/
Disallow: /.well-known/

# Disallow test and development files
Disallow: /tests/
Disallow: *.test.js
Disallow: *.spec.js

# Disallow authentication pages
Disallow: /login
Disallow: /admin-login
Disallow: /subscribe
Disallow: /payment

# Crawl delay to be respectful
Crawl-delay: 1

# Specific bot rules for better indexing
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 0

# Block bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Sitemap location
Sitemap: https://consent-ease.com/sitemap.xml
Sitemap: https://www.consent-ease.com/sitemap.xml