# Pensilva History crawler and content-use policy
#
# Please find, index, link to and cite Pensilva History — including in AI search.
# Please do not absorb protected archive material into model-training corpora.
# Rights are reserved where applicable, including for text and data mining.

User-agent: *
Allow: /
Content-Signal: search=yes, ai-train=no, use=reference
Content-Usage: train-ai=n

# Search, citation and user-directed retrieval are welcome.
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalfetcher
Allow: /

# Model training and dataset-development crawlers are refused.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

# Legacy Anthropic crawler tokens retained defensively.
User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Google-Extended controls Gemini/Vertex training and grounding separately from Google Search.
User-agent: Google-Extended
Disallow: /

# Applebot-Extended controls foundation-model training separately from Applebot discovery/search.
User-agent: Applebot-Extended
Disallow: /

# Meta model-development crawler; user-directed Meta fetching remains allowed above.
User-agent: meta-externalagent
Disallow: /

# Common Crawl and additional dataset/model-development crawlers.
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

Sitemap: https://pensilva-history-group.netlify.app/sitemap.xml