# Framerate - Movie and TV Show Reviews
# https://framerate.site

# Block SEO data-harvesting bots (no SEO value, waste server resources)
User-agent: MJ12bot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SemrushBot-SI
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: SemrushBot-CT
Disallow: /

User-agent: SemrushBot-BM
Disallow: /

User-agent: SplitSignalBot
Disallow: /

User-agent: SemrushBot-COUB
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: AhrefsSiteAudit
Disallow: /

User-agent: Dataprovider
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: *

# Allow all public pages
Allow: /
Allow: /movie/
Allow: /tv/
Allow: /person/
Allow: /company/
Allow: /network/
Allow: /profile/
Allow: /discover
Allow: /search

# Public user profiles (/profile/<username>) are crawlable — needed for social
# cards (X/Twitter, Facebook obey robots.txt) and SEO. Only the bare /profile
# (own profile, auth-gated) is blocked, via the $ end-anchor.
Disallow: /profile$
# Other user-specific pages (require authentication, no SEO value)
Disallow: /settings
Disallow: /watchlist
Disallow: /saved
Disallow: /loved
Disallow: /tracking
Disallow: /ratings
Disallow: /reviews
Disallow: /feed
Disallow: /lists
Disallow: /notifications

# Disallow authentication pages
Disallow: /login
Disallow: /register
Disallow: /reset-password
Disallow: /auth/
Disallow: /email-verification

# Disallow API endpoints
Disallow: /api/

# Disallow admin/internal pages
Disallow: /admin
Disallow: /debug

# Crawl-delay for polite crawling (optional)
Crawl-delay: 1

# Sitemap location
Sitemap: https://framerate.site/sitemap.xml