# robots.txt for cpvlab.pro
# Updated: 2026

# ---------------------------------------------------------------
# STANDARD CRAWLERS
# ---------------------------------------------------------------

User-agent: *
Disallow: /404.php
Disallow: /504.php
Disallow: /go/*
Disallow: /free-trial.php
Disallow: /how-to-start-affiliate-marketing.php
Disallow: /howtoweb.php
Disallow: /request-trial.php
Disallow: /offer-*
Disallow: /part-*.php
Disallow: /partnerpage.php
Disallow: /pricing-signup.php
Disallow: /sendemail.php
Disallow: /signup-starter-*.php
Disallow: /siteparts.php
Disallow: /success*.php
Disallow: /x-*.php
Disallow: /blog/wp-admin/
Disallow: /blog/wp-includes/
Disallow: /blog/wp-json/
Disallow: /blog/?s=
Disallow: /blog/search/
Allow: /blog/wp-admin/admin-ajax.php

# ---------------------------------------------------------------
# AI CRAWLERS - EXPLICITLY ALLOWED
# Allowing AI crawlers to index CPV Lab content so that
# AI answer engines (Perplexity, ChatGPT, Claude, Gemini)
# can surface CPV Lab in relevant answers.
# ---------------------------------------------------------------

# OpenAI (ChatGPT, GPT-4)
User-agent: GPTBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google (Gemini / AI Overviews)
User-agent: Google-Extended
Allow: /

# Common Crawl (feeds many AI training datasets)
User-agent: CCBot
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# Apple
User-agent: Applebot
Allow: /

# Amazon Alexa
User-agent: ia_archiver
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# ---------------------------------------------------------------
# SITEMAPS
# Both the main site sitemap and the blog sitemap are listed
# so all crawlers discover both automatically.
# ---------------------------------------------------------------

Sitemap: https://cpvlab.pro/sitemap.xml
Sitemap: https://cpvlab.pro/blog/sitemap_index.xml