# 1. GLOBAL CRAWLER RULES (Save Crawl Budget by blocking system/junk files)
User-agent: *
Allow: /
Disallow: /cgi-bin/
Disallow: /cdn-cgi/
Disallow: /login/
Disallow: /xmlrpc.php
Disallow: /trash.txt
Disallow: /trackback/
Disallow: /*.inc$
Disallow: /*.gz$
Disallow: /*.zip$
Disallow: /*.rar$
Disallow: /*.tar$
Disallow: /*.exe$

# 2. EXPLICITLY ALLOW AI & SEARCH BOTS (AEO / GEO Optimization)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

# 3. SITEMAP
Sitemap: https://www.idatam.com/sitemap.xml
Sitemap: https://www.idatam.com/llms.txt