# syfert.com robots policy. Updated 2026-09-06: AI TRAINING crawlers admitted
# (Graham: open up to Meta + training crawlers now that CPU is not the constraint).
# gate.php verifies the ones with published IP lists (shared/verified_bots.php);
# Common Crawl (CCBot) and Cohere are admitted on User-Agent at the gate. SEO /
# link-analysis scrapers, security scanners, ByteDance (Bytespider) and foreign
# search engines stay blocked. Thin/duplicate URLs (search results, digest
# facets) are Disallow'd for everyone; case, statute and code pages are open.

# ── Allowed: search engines + AI index & training ──
User-agent: Googlebot
User-agent: Google-Extended
User-agent: Bingbot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: DuckDuckBot
User-agent: DuckAssistBot
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: Amzn-User
User-agent: meta-externalagent
User-agent: meta-webindexer
User-agent: Meta-ExternalFetcher
User-agent: facebookexternalhit
User-agent: CCBot
User-agent: cohere-ai
Allow: /
Disallow: /caselaw/?
Disallow: /*?q=
Disallow: /florida/case-law-digest/*?
Crawl-delay: 5

# ── Blocked: SEO / link-analysis scrapers (pure extraction, no upside) ──
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: BLEXBot
User-agent: linkdexbot
User-agent: MajesticBot
User-agent: SEOkicks
User-agent: serpstatbot
User-agent: Barkrowler
Disallow: /

# ── Blocked: ByteDance (ignores robots, abusive), foreign search, social scrapers ──
User-agent: Bytespider
User-agent: PetalBot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: Sogou
User-agent: 360Spider
User-agent: YouBot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Pinterest
Disallow: /

# ── Blocked: security scanners / site mirrors ──
User-agent: HTTrack
User-agent: Nuclei
User-agent: Censys
User-agent: masscan
User-agent: SleepBot
Disallow: /

# ── Everyone else ──
User-agent: *
Content-Signal: search=yes,ai-train=yes
Allow: /
Disallow: /caselaw/?
Disallow: /*?q=
Disallow: /florida/case-law-digest/*?
Crawl-delay: 10

Sitemap: https://syfert.com/sitemap_index.xml