# Preinfuse — crawler policy
# Reviewed 2026-09-01.
#
# Every crawler Preinfuse has an opinion about is named explicitly, so the
# effective policy is unambiguous. Anything unnamed inherits User-agent: *.

User-agent: *
Allow: /
# The comparison tool builds ad-hoc URLs from the compare tray. The curated
# comparison pages are the canonical, indexable version of the same content.
Disallow: /compare/?
Disallow: /*?ids=

# --- Search and AI retrieval ---------------------------------------------
# Search indexing, answer-engine retrieval and citation, and user-initiated
# fetches. These decide whether Preinfuse can be found or cited at all.
User-agent: Googlebot
Allow: /
User-agent: Bingbot
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: Applebot
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: Claude-SearchBot
Allow: /
User-agent: Claude-User
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /
User-agent: Meta-WebIndexer
Allow: /
User-agent: Amzn-SearchBot
Allow: /
User-agent: Amzn-User
Allow: /

# --- Commercial / mixed-purpose crawlers ---------------------------------
# Retrieval and training are bundled under one user agent and cannot be
# separated. Allowed: the distribution value outweighs the training use.
User-agent: Google-Extended
Allow: /
User-agent: Amazonbot
Allow: /
User-agent: meta-externalagent
Allow: /

# --- Training-only crawlers ----------------------------------------------
# Disallowed: none of these carries a search or citation surface Preinfuse
# depends on. To reverse, change Disallow to Allow. Nothing else depends on it.
User-agent: GPTBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Bytespider
Disallow: /

Sitemap: https://preinfuse.com/sitemap.xml