# Global rules for all crawlers
User-agent: *
Disallow: /api/
Disallow: /_next/
Disallow: /static/temp/
Disallow: /node_modules/
Disallow: /private/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /uploads/

# Allow public content
Allow: /
Allow: /terms-conditions/
Allow: /thanks/
Allow: /static/
Allow: /favicon.ico
Allow: /robots.txt
Allow: /sitemap.xml

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 5

User-agent: Bingbot
Allow: /
Crawl-delay: 5

User-agent: Slurp
Allow: /
Crawl-delay: 5

User-agent: Gigabot
Allow: /
Crawl-delay: 5

User-agent: Robozilla
Allow: /
Crawl-delay: 5

User-agent: Nutch
Allow: /
Crawl-delay: 5

User-agent: ia_archiver
Allow: /
Crawl-delay: 5

User-agent: baiduspider
Allow: /
Crawl-delay: 5

User-agent: naverbot
Allow: /
Crawl-delay: 5

User-agent: yeti
Allow: /
Crawl-delay: 5

User-agent: yahoo-mmcrawler
Allow: /
Crawl-delay: 5

User-agent: psbot
Allow: /
Crawl-delay: 5

User-agent: yahoo-blogs/v3.9
Allow: /
Crawl-delay: 5

# AI-specific crawlers (LLM / content understanding)
User-agent: GPTBot
Allow: /
Allow: /terms-conditions/
Crawl-delay: 5

User-agent: Google-Extended
Allow: /
Allow: /terms-conditions/
Crawl-delay: 5

User-agent: CCBot
Allow: /
Allow: /terms-conditions/
Crawl-delay: 5

User-agent: anthropic-ai
Allow: /
Allow: /terms-conditions/
Crawl-delay: 5

User-agent: Claude-Web
Allow: /
Allow: /terms-conditions/
Crawl-delay: 5

# File type guidance
Allow: /*.css
Allow: /*.js
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.svg
Allow: /*.webp
Disallow: /*.pdf
Disallow: /*.doc
Disallow: /*.zip

# Sitemap location
Sitemap: https://aifilms.ai/sitemap.xml