# Lucky Earth - robots.txt
# Rewritten 2026-07-27, note corrected 2026-07-28. Changed 2026-10-27: Google-Extended, see below.
# Changed 2026-11-10: the MCP note below (comments only, no rule changed).
#
# AI assistants and agents: Lucky Earth answers travel questions over MCP too, free and read-only,
# at https://luckyearth.org/mcp. Server card: https://luckyearth.org/.well-known/mcp.json.
# More for language models: https://luckyearth.org/llms.txt
#
# READ THIS BEFORE EDITING. Since 27.10.2026 this file is the whole robots.txt the world receives.
# Cloudflare's managed robots.txt (Settings > Bot traffic > Configure AI bot policies >
# Enable Bot Preference Sync) is switched OFF: it cannot leave one crawler out, and it kept
# Google-Extended blocked. Its notice on content signals and its whole list of training crawlers
# are copied into this file, so for every other agent nothing changed. Blocking at the edge
# (AI Crawl Control, Training = Block) is a separate setting and stays on.
#
# The policy: assistants that fetch a page in order to answer a question are welcome;
# crawlers that harvest for model training are not.
#
# One exception, decided 27.10.2026: Google-Extended. It is not a crawler. It is Google's single
# switch for two things at once: training Gemini, and grounding Gemini's answers on our pages.
# While it was blocked, Gemini could not read luckyearth.org and described Lucky Earth Ltd from
# another site's footer. We accept the training half in order to be read and cited by Gemini.
# The content signal below stays ai-train=no for everyone; the Google-Extended group further
# down is our explicit, narrower permission to Google. It does not affect Google Search.

# As a condition of accessing this website, you agree to abide by the following
# content signals:

# (a)  If a Content-Signal = yes, you may collect content for the corresponding
#      use.
# (b)  If a Content-Signal = no, you may not collect content for the
#      corresponding use.
# (c)  If the website operator does not include a Content-Signal for a
#      corresponding use, the website operator neither grants nor restricts
#      permission via Content-Signal with respect to the corresponding use.

# The content signals and their meanings are:

# search:   building a search index and providing search results (e.g., returning
#           hyperlinks and short excerpts from your website's contents). Search does not
#           include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of content for
#           generative AI search answers).
# ai-train: training or fine-tuning AI models.
# use:      how AI systems may consume the content (immediate, reference, or full).

# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

# Content signals, per the emerging convention.
User-agent: *
Content-Signal: search=yes, ai-train=no, use=reference
Allow: /
Disallow: /data/
Disallow: /city/data/
Disallow: /admin/
Disallow: /administrator/
Disallow: /plugins/
Disallow: /city-pulse-vote.php
Disallow: /city-photo-submit.php
Disallow: /stripe-add-code.php
Disallow: /submit-ad.php
Disallow: /approve-ad.php
Disallow: /funnel.php
Disallow: /funnel-report.php
Disallow: /map/?le_tool=
Disallow: /activity.php

# Search engines — every public page, but not our data files, forms and counters.
# 2026-10-11: a crawler reads ONLY the most specific group that names it. While these groups said just
# "Allow: /", Googlebot, bingbot and Applebot ignored every Disallow under "User-agent: *" above, and Search
# Console showed /data/*.json, /funnel.php and 243 tool links being crawled. The list below is the same as
# the one under "*"; change both together. Not blocked on purpose: /map/og/ (share images: Twitter/X reads
# robots before fetching one) and JSON in general (the map is drawn from /map/data/*.json).
User-agent: Googlebot
Allow: /
Disallow: /data/
Disallow: /city/data/
Disallow: /admin/
Disallow: /administrator/
Disallow: /plugins/
Disallow: /city-pulse-vote.php
Disallow: /city-photo-submit.php
Disallow: /stripe-add-code.php
Disallow: /submit-ad.php
Disallow: /approve-ad.php
Disallow: /funnel.php
Disallow: /funnel-report.php
Disallow: /map/?le_tool=
Disallow: /activity.php

User-agent: Googlebot-Image
Allow: /

User-agent: bingbot
Allow: /
Disallow: /data/
Disallow: /city/data/
Disallow: /admin/
Disallow: /administrator/
Disallow: /plugins/
Disallow: /city-pulse-vote.php
Disallow: /city-photo-submit.php
Disallow: /stripe-add-code.php
Disallow: /submit-ad.php
Disallow: /approve-ad.php
Disallow: /funnel.php
Disallow: /funnel-report.php
Disallow: /map/?le_tool=
Disallow: /activity.php

User-agent: Applebot
Allow: /
Disallow: /data/
Disallow: /city/data/
Disallow: /admin/
Disallow: /administrator/
Disallow: /plugins/
Disallow: /city-pulse-vote.php
Disallow: /city-photo-submit.php
Disallow: /stripe-add-code.php
Disallow: /submit-ad.php
Disallow: /approve-ad.php
Disallow: /funnel.php
Disallow: /funnel-report.php
Disallow: /map/?le_tool=
Disallow: /activity.php

# Assistants that fetch a page in order to answer a live question. These are how a traveller
# reaches us through an AI tool, so they get the same access as a search engine.
User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /
Crawl-delay: 5

# Gemini: grounding answers on our pages (and, with it, Gemini training). See the note at the top.
User-agent: Google-Extended
Allow: /
Disallow: /data/
Disallow: /city/data/
Disallow: /admin/
Disallow: /administrator/
Disallow: /plugins/
Disallow: /city-pulse-vote.php
Disallow: /city-photo-submit.php
Disallow: /stripe-add-code.php
Disallow: /submit-ad.php
Disallow: /approve-ad.php
Disallow: /funnel.php
Disallow: /funnel-report.php
Disallow: /map/?le_tool=
Disallow: /activity.php

# Training crawlers — not permitted. Our content is verified by hand and dated; we would
# rather be cited than absorbed.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Copied from Cloudflare's managed list on 27.10.2026, when that list was switched off.
User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: MistralAI-Training
Disallow: /

User-agent: GoogleOther
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: AwarioSmartBot
User-agent: AwarioRssBot
Disallow: /

User-agent: QualifiedBot
Disallow: /

User-agent: Cotoyogi
Disallow: /

User-agent: ICC-Crawler
Disallow: /

User-agent: atlassian-bot
Disallow: /

User-agent: FishBot
Disallow: /

User-agent: BorderxBot
Disallow: /

User-agent: NavuBot
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: WARDBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: KimiBot
Disallow: /

User-agent: CitibotSiteCrawler
Disallow: /

# Social preview fetchers — needed for link cards when a page is shared.
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

# SEO tooling — allowed, rate-limited.
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

User-agent: SemrushBot
Allow: /
Crawl-delay: 10

# Aggressive or low-value crawlers.
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: SecurityAudit
Disallow: /

Sitemap: https://luckyearth.org/sitemap.xml