# robots.txt - 911 Restoration Website
# This file allows search engines to crawl and index the website

# Allow all search engines to crawl the entire site
User-agent: *
Allow: /

# Block access to admin and private areas
Disallow: /admin/
Disallow: /api/
Disallow: /storage/
Disallow: /vendor/
Disallow: /.env
Disallow: /config/

# Block common non-essential files and directories
Disallow: /*.log
Disallow: /*.tmp
Disallow: /temp/
Disallow: /cache/

# Allow social media bots for better sharing
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# AI Training Bots - Configure based on your preference
# ALLOW: If you want your content used for AI training (potential exposure/marketing)
# DISALLOW: If you want to protect your content from AI training

# OpenAI (ChatGPT) crawler
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /

# Google AI (Gemini/Bard) - uses Googlebot, already allowed above
User-agent: Google-Extended
Allow: /

# Anthropic (Claude) crawler
User-agent: ClaudeBot
Allow: /

# Other AI crawlers
User-agent: CCBot
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: AI2Bot
Allow: /

# Reasonable crawl delay to prevent server overload
# Crawl-delay: 1

# Sitemap location
Sitemap: https://www.911restoration.com/sitemap.xml