User-agent: *
Allow: /
Disallow: /admin/
Disallow: /saved/
Disallow: /compare/

# AI crawlers — uncomment to opt out individually
# User-agent: GPTBot
# Disallow: /
# User-agent: Google-Extended
# Disallow: /
# User-agent: ClaudeBot
# Disallow: /
# User-agent: CCBot
# Disallow: /

Sitemap: https://arabguide.com/sitemap-index.xml
Sitemap: https://arabguide.com/sitemap-news.xml

# llms.txt for AI crawlers — site overview + key URLs
# https://llmstxt.org/