# robots.txt for Aremko Spa - Puerto Varas, Chile
# Optimized for search engines and AI crawlers

# === DEFAULT RULES FOR ALL BOTS ===
User-agent: *

# Allow public pages
Allow: /
Allow: /categoria/
Allow: /static/
Allow: /media/

# Block admin and private areas
Disallow: /admin/
Disallow: /accounts/
Disallow: /ventas/admin-dashboard/
Disallow: /ventas/admin/

# Block checkout and payment processes
Disallow: /ventas/checkout/
Disallow: /ventas/complete-checkout/
Disallow: /ventas/cart/
Disallow: /payment/

# Block API endpoints (not meant for crawling)
Disallow: /ventas/api/
Disallow: /api/

# Block internal tools and reports
Disallow: /ventas/caja-diaria/
Disallow: /ventas/auditoria-movimientos/
Disallow: /ventas/servicios-vendidos/
Disallow: /ventas/productos-vendidos/
Disallow: /ventas/reportes/
Disallow: /ventas/importar-clientes/
Disallow: /ventas/exportar-clientes/
Disallow: /ventas/clientes/

# Block booking process (privacy)
Disallow: /ventas/get-available-hours/
Disallow: /ventas/check-availability/
Disallow: /ventas/add-to-cart/
Disallow: /ventas/remove-from-cart/

# === SPECIFIC BOT RULES ===

# NOTA: Googlebot y Bingbot NO tienen grupo propio a propósito. Por el RFC 9309
# un crawler obedece solo su grupo más específico e ignora el de "*"; si les
# damos un grupo aquí, dejan de leer todos los Disallow del bloque "*" de arriba
# (páginas internas, checkout, reportes) y quedan rastreables. Sin grupo propio,
# ambos caen al "*", que ya bloquea todo lo privado.

# AI Crawlers - allow with guidance
User-agent: GPTBot
Allow: /
Allow: /ai.txt
Allow: /llm.txt
Disallow: /admin/
Disallow: /api/
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Allow: /ai.txt
Allow: /llm.txt
Disallow: /admin/
Disallow: /api/

User-agent: CCBot
Allow: /
Allow: /ai.txt
Allow: /llm.txt
Disallow: /admin/
Disallow: /api/
Crawl-delay: 2

User-agent: anthropic-ai
Allow: /
Allow: /ai.txt
Allow: /llm.txt
Disallow: /admin/
Disallow: /api/
Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Allow: /ai.txt
Allow: /llm.txt
Disallow: /admin/
Disallow: /api/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /ai.txt
Allow: /llm.txt
Disallow: /admin/
Disallow: /api/
Crawl-delay: 2

# Block aggressive or unwanted bots
User-agent: SemrushBot
Crawl-delay: 10

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: SeznamBot
Crawl-delay: 5

# === SITEMAPS ===
Sitemap: https://www.aremko.cl/sitemap.xml

# === ADDITIONAL RESOURCES FOR AI ===
# AI crawlers should check these files for structured information:
# - /ai.txt - Business information for AI systems
# - /llm.txt - Comprehensive LLM guidance
# - /sitemap.xml - Complete site structure