User-agent: *
Disallow: /cgi-bin/
Disallow: /css/
Disallow: /casefiles/wp-admin/
Disallow: /casefiles/wp-includes/
Disallow: /casefiles/wp-content/plugins/
Disallow: /casefiles/wp-content/cache/
Disallow: /casefiles/wp-content/themes/
Disallow: /documentarchive/wp-admin/
Disallow: /documentarchive/wp-includes/
Disallow: /documentarchive/wp-content/plugins/
Disallow: /documentarchive/wp-content/cache/
Disallow: /documentarchive/wp-content/themes/
Disallow: /community/wp-admin/
Disallow: /community/wp-includes/
Disallow: /community/wp-content/plugins/
Disallow: /community/wp-content/cache/
Disallow: /community/wp-content/themes/
Disallow: */feed/
Disallow: */trackback/
Disallow: /*?
Disallow: /xmlrpc.php

# Explicitly allow uploads (images, PDFs — crucial for your docs)
Allow: /wp-content/uploads/
Allow: /documentarchive/wp-content/uploads/
Allow: /casefiles/wp-content/uploads/

# Allow legitimate AI/research bots for visibility & citations in AI answers
# These often source/link back, aligning with your transparency mission
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: cohere-ai
Allow: /

# If you see traffic from these variants (less common but reported)
User-agent: Claude-Web
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Sitemaps (keep all — excellent for discovery of your archives)
Sitemap: https://www.theblackvault.com/documentarchive/sitemap_index.xml
Sitemap: https://www.theblackvault.com/casefiles/sitemap_index.xml
Sitemap: https://www.theblackvault.com/sitemap-www-documents.txt
Sitemap: https://documents.theblackvault.com/sitemap3.txt
Sitemap: https://documents.theblackvault.com/sitemap2.txt
Sitemap: https://documents.theblackvault.com/sitemap.txt
Sitemap: https://documents2.theblackvault.com/sitemap.txt