# Block sensitive pages
User-agent: *
Disallow: /get-started
Disallow: /signup
Disallow: /verify-mail
Disallow: /recover-password
Disallow: /recover-password-mail
Disallow: /create-new-password
Disallow: /select-country
Disallow: /add-assets
Disallow: /add-wallet
Disallow: /portfolio
Disallow: /profile
Disallow: /transaction
Disallow: /wallet
Disallow: /performance
Disallow: /tax
Disallow: /onboarding
Disallow: /payment-processing
Disallow: /payment-cancelled
Disallow: /login
Disallow: /404
Allow: /public/
Allow: /static/


# Allow all major search engines
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: SemrushBot
Allow: /

# AI crawlers
User-agent: GPTBot
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: Anthropic-Bot
User-agent: CoPilotBot
Allow: /

# Social media bots
User-agent: Facebot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Pinterestbot
Allow: /

Sitemap: https://www.blockstats.app/sitemap_index.xml