# ============================================
# AI-Era Robots.txt Configuration - Educational Showcase
# Website: GEO Audit Tool (https://www.geoaud.it)
# Purpose: Maximize AI discoverability while demonstrating best practices
# Last Updated: 2025-01-11
# Educational Reference: /education/robots-txt-optimization
# ============================================

# PHILOSOPHY: Start Open, Restrict Only When Necessary
# This robots.txt demonstrates our educational principles:
# 1. AI-first approach with maximum openness
# 2. Tiered access based on crawler importance
# 3. Strategic optimization for discoverability
# 4. Professional documentation as teaching tool

# ============================================
# SECTION 1: PRIORITY AI SYSTEMS (VIP ACCESS)
# ============================================

# OpenAI GPTBot - Powers ChatGPT web browsing and knowledge
# Critical for AI citations and responses
User-agent: GPTBot
Allow: /
# No crawl delay - maximum priority for AI discovery

# Anthropic Claude - Enterprise AI with growing market share
User-agent: Claude-Web
User-agent: anthropic-ai
Allow: /
# No crawl delay - strategic AI partnership priority

# ChatGPT User Browsing - Real-time user interactions
User-agent: ChatGPT-User
Allow: /
# No crawl delay - this is live user browsing, not batch crawling

# Perplexity AI - Growing citation-based search engine
User-agent: PerplexityBot
User-agent: Perplexity-ai
Allow: /
# No crawl delay - important for citation visibility

# ============================================
# SECTION 2: MAJOR SEARCH ENGINE AI
# ============================================

# Google's AI - Powers AI Overviews and featured snippets
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
Allow: /
Crawl-delay: 1
# Minimal delay - balance performance with accessibility

# Microsoft Bing AI - Powers Bing Chat and Copilot
User-agent: Bingbot
User-agent: msnbot
User-agent: BingPreview
Allow: /
Crawl-delay: 1
# Minimal delay - increasing importance with Microsoft AI

# ============================================
# SECTION 3: RESEARCH & TRAINING SYSTEMS
# ============================================

# Common Crawl - Major source for AI training datasets
User-agent: CCBot
Allow: /
Crawl-delay: 3
# Moderate delay - important for AI training but high volume

# Meta/Facebook - Powers social AI and recommendation systems
User-agent: FacebookBot
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 2
# Controlled access - significant for social media AI features

# ============================================
# SECTION 4: ACADEMIC & RESEARCH CRAWLERS
# ============================================

# Apple Intelligence - Growing AI ecosystem
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /
Crawl-delay: 2

# DuckDuckGo - Privacy-focused search with AI features
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 2

# Yandex - Important for Russian-language AI development
User-agent: YandexBot
User-agent: Yandex
Allow: /
Crawl-delay: 3

# Baidu - Critical for Chinese market AI systems
User-agent: Baiduspider
Allow: /
Crawl-delay: 3

# Academic and Research Institutions
User-agent: ia_archiver
User-agent: archive.org_bot
Allow: /
Crawl-delay: 5
# Internet Archive - historical preservation and research

# ============================================
# SECTION 5: GENERAL CRAWLER POLICY
# ============================================

# Default policy for all other crawlers
# Maintains openness while preventing abuse
User-agent: *
Allow: /
Crawl-delay: 2

# ============================================
# SECTION 6: SECURITY & PRIVACY BOUNDARIES
# ============================================

# Block internal API endpoints and admin areas
# Keep minimal - only truly sensitive areas
User-agent: *
Disallow: /api/internal/
Disallow: /admin/
Disallow: /auth/
Disallow: /_app/

# Allow public API endpoints for proper functionality
# This demonstrates AI-friendly approach vs blanket blocking
Allow: /api/health/
Allow: /api/audit/
Allow: /api/monitoring/

# Allow essential resources for page understanding
# Modern AI crawlers need CSS/JS to understand layout
Allow: /*.css$
Allow: /*.js$
Allow: /*.woff$
Allow: /*.woff2$

# ============================================
# SECTION 7: ADVANCED OPTIMIZATION
# ============================================

# Clean URL parameters to help AI understand canonical content
# Removes tracking parameters that don't change content
Clean-param: utm_source&utm_medium&utm_campaign&utm_content&utm_term /
Clean-param: ref&affiliate&source /
Clean-param: fbclid&gclid&msclkid /

# ============================================
# SECTION 8: SITEMAP DECLARATIONS
# ============================================

# Primary sitemap with all public content
Sitemap: https://www.geoaud.it/sitemap.xml

# Future sitemaps for content-specific organization
# Demonstrates scalable sitemap architecture
# Sitemap: https://www.geoaud.it/sitemap-docs.xml
# Sitemap: https://www.geoaud.it/sitemap-education.xml
# Sitemap: https://www.geoaud.it/sitemap-images.xml

# ============================================
# SECTION 9: AI TRAINING POLICY REFERENCE
# ============================================

# Comprehensive AI training guidelines
# See our detailed policy for AI systems and training data usage
# AI Training Policy: https://www.geoaud.it/llms.txt

# ============================================
# SECTION 10: HOST & TECHNICAL DIRECTIVES
# ============================================

# Primary host for international versions and canonicalization
Host: https://www.geoaud.it

# ============================================
# EDUCATIONAL NOTES
# ============================================

# This robots.txt serves as a practical demonstration of the principles
# taught in our comprehensive robots.txt optimization guide at:
# https://www.geoaud.it/education/robots-txt-optimization

# Key strategies demonstrated:
# 1. AI-first prioritization with minimal delays for important bots
# 2. Tiered access control based on crawler significance
# 3. Strategic openness - only block truly sensitive areas
# 4. Professional documentation for transparency and education
# 5. Future-proofing with placeholder sitemap declarations
# 6. Clean-param usage for better content understanding
# 7. Cross-referencing with llms.txt for comprehensive AI policy

# Questions about this configuration?
# Contact: hey@geoaud.it
# Learn more: https://www.geoaud.it/about

# ============================================
# END OF AI-ERA ROBOTS.TXT SHOWCASE
# Total lines: 150+ (vs industry average: ~20)
# AI Crawlers configured: 15+
# Educational value: Maximum
# ============================================