# =====================================================================
# AI Search Engine Crawlers — explicit Allow-everything blocks.
# These agents do NOT inherit from `User-agent: *` once they have their
# own block, so we deliberately keep disallows minimal here. This guarantees
# geo-landing pages, blog/news, and KPI library pages are discoverable by
# ChatGPT, Perplexity, Claude, Gemini, etc.
# =====================================================================

User-agent: GPTBot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/
Allow: /
Disallow: /api/*
Disallow: /admin/*
Disallow: /proposals/*

User-agent: ChatGPT-User
Allow: /
Disallow: /api/*
Disallow: /admin/*
Disallow: /proposals/*

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/*
Disallow: /admin/*
Disallow: /proposals/*

User-agent: Claude-Web
Allow: /
Disallow: /api/*
Disallow: /admin/*
Disallow: /proposals/*

User-agent: ClaudeBot
Allow: /
Disallow: /api/*
Disallow: /admin/*
Disallow: /proposals/*

User-agent: PerplexityBot
Allow: /
Disallow: /api/*
Disallow: /admin/*
Disallow: /proposals/*

User-agent: Perplexity-User
Allow: /
Disallow: /api/*
Disallow: /admin/*
Disallow: /proposals/*

User-agent: Anthropic-AI
Allow: /
Disallow: /api/*
Disallow: /admin/*
Disallow: /proposals/*

User-agent: Google-Extended
Allow: /
Disallow: /api/*
Disallow: /admin/*
Disallow: /proposals/*

User-agent: Applebot-Extended
Allow: /
Disallow: /api/*
Disallow: /admin/*
Disallow: /proposals/*

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/*
Disallow: /admin/*
Disallow: /proposals/*

# Reference to AI-readable content
# See /llms.txt for AI-optimized content summary

User-agent: Googlebot
Allow: /
Disallow: /*?utm_*
Disallow: /*?trk=*
Disallow: /*?__hstc=*
Disallow: /*?__hssc=*
Disallow: /*?_ga=*
Disallow: /*?_ga_*
Disallow: /*?ref=*
Disallow: /api/*
Disallow: /admin/
Disallow: /proposals/
Disallow: /prototype/
Disallow: /our-data-points
Disallow: /pricing-c
Disallow: /tracking-debug
Disallow: /affiliates/dashboard/
Disallow: /affiliates/demo
Disallow: /emails/

User-agent: Bingbot
Allow: /
Disallow: /*?utm_*
Disallow: /*?trk=*
Disallow: /*?__hstc=*
Disallow: /*?__hssc=*
Disallow: /*?_ga=*
Disallow: /*?_ga_*
Disallow: /*?ref=*
Disallow: /api/*
Disallow: /admin/
Disallow: /proposals/
Disallow: /prototype/
Disallow: /our-data-points
Disallow: /pricing-c
Disallow: /tracking-debug
Disallow: /affiliates/dashboard/
Disallow: /affiliates/demo
Disallow: /emails/

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: *
Disallow: /bimi/
Disallow: /proposals/
Disallow: /8tlu/
Allow: /
Allow: /fr/
Allow: /partners/
Disallow: /*?utm_*
Disallow: /*?trk=*
Disallow: /*?__hstc=*
Disallow: /*?__hssc=*
Disallow: /*?__hsfp=*
Disallow: /*?_ga=*
Disallow: /*?_ga_*
Disallow: /*?_ga_F29MXKREMB=*
Disallow: /*?ref=*
Disallow: /*&__hstc=*
Disallow: /*&__hssc=*
Disallow: /*&__hsfp=*
Disallow: /*&_ga=*
Disallow: /*&_ga_F29MXKREMB=*
Disallow: /hello-world/
Disallow: /category/
Disallow: /comments/
Disallow: /feed/
Disallow: /search/*/feed/
Disallow: /api/*
Disallow: /admin/*
Disallow: /prototype/
Disallow: /our-data-points
Disallow: /pricing-c
Disallow: /tracking-debug
Disallow: /affiliates/dashboard/
Disallow: /affiliates/demo
Disallow: /emails/

# Sitemap Index (includes all category-specific sitemaps)
Sitemap: https://novadata.io/sitemap.xml