# ================================
# MASL WORLD - LLM CRAWLER RULES
# ================================

# Allow AI agents to crawl the public website
User-agent: *
Allow: /

# Disallow system-level folders
Disallow: /admin/
Disallow: /config/
Disallow: /database/
Disallow: /phpmailer/
Disallow: /assets/uploads/
Disallow: /backup/
Disallow: /private/
Disallow: /temp/
Disallow: /vendor/

# Disallow crawling of scripts and backend files
Disallow: /*.php
Disallow: /*.sql
Disallow: /*.json
Disallow: /*.env

# Allow public HTML pages
Allow: /*.html

# ================================
# BLOCK AI TRAINING (Optional)
# ================================
Allow-LLM: display
Disallow-LLM: train

# Explanation:
# display → AI is allowed to show website content in search results
# train   → AI is NOT allowed to use your content for model training

# ================================
# Specific AI Crawlers
# ================================
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow-LLM: train

User-agent: Google-Extended
Allow: /
Disallow-LLM: train

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Allow: /
Disallow: /admin/
Disallow-LLM: train

User-agent: Meta-LLM
Allow: /
Disallow: /admin/
Disallow-LLM: train

User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow-LLM: train

# ================================
# Crawl rate limits (optional)
# ================================
Crawl-delay: 5

# ================================
# Contact
# ================================
Contact: info@maslworld.com