User-agent: *
Allow: /

# Block admin and private areas
Disallow: /admin/
Disallow: /ejk4302ejk4302/
Disallow: /api/
Disallow: /static/flask_session/
Disallow: /instance/
Disallow: /*.log$
Disallow: /flask_session/
Disallow: /background_search_cache/

# Block duplicate/faceted search results
Disallow: /*?

# Allow important static assets
Allow: /static/css/
Allow: /static/js/
Allow: /static/img/
Allow: /static/lightning_search.css
Allow: /static/lightning_search_frontend.js

# Block known scrapers and bad bots
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Allow social media bots for link previews
User-agent: facebookexternalhit/
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Sitemap location (point to lean sitemap or sitemap index)
Sitemap: https://carchase.ai/cars-1.xml