# My Guitar Notes — crawler policy
# Generated by tools/atlas/pipeline/crawl.js. Edit the generator, not this file.
#
# Position: reputable search and archival crawlers are welcome. The site is
# static, there is nothing that needs hiding from a general crawler, and being
# indexed is the point. Only the two pipeline bookkeeping files are excluded.
#
# Everything here is published under the terms on /licenses.html: original
# content is all rights reserved, and open-dataset content carries the
# attribution obligations named for each source.

User-agent: *
Allow: /
Disallow: /deploy-plan.json
Disallow: /deploy-upload.txt

# --- named search engines (explicit, so intent survives a future * change)
User-agent: Googlebot
Allow: /
User-agent: Bingbot
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: Applebot
Allow: /
User-agent: Slurp
Allow: /
User-agent: YandexBot
Allow: /
User-agent: Baiduspider
Allow: /

# --- named archival crawlers
User-agent: ia_archiver
Allow: /
User-agent: archive.org_bot
Allow: /

# ---------------------------------------------------------------------------
# AI-TRAINING CRAWLERS — OWNER SWITCH
# Current policy: ALLOWED (a one-line switch in tools/atlas/pipeline/crawl.js)
#
# The site's original content is all rights reserved and its open-dataset
# content carries attribution terms (see /licenses.html). Whether bulk
# collection for model training is welcome is an owner decision, so it is a
# one-line switch rather than a judgement call buried in a list.
#
# DEFAULT — to refuse these eight agents, uncomment the eight Disallow lines below
# (delete the leading "# " from each), or switch the constant in crawl.js.
# User-agent: GPTBot
# Disallow: /
# User-agent: Google-Extended
# Disallow: /
# User-agent: ClaudeBot
# Disallow: /
# User-agent: CCBot
# Disallow: /
# User-agent: PerplexityBot
# Disallow: /
# User-agent: Applebot-Extended
# Disallow: /
# User-agent: Bytespider
# Disallow: /
# User-agent: meta-externalagent
# Disallow: /
# ---------------------------------------------------------------------------

Sitemap: https://www.myguitarnotes.com/sitemap.xml