# robots.txt, based on the Sep 25 2026 access.log analysis
#
# Note: a file in the document root replaces WordPress's generated robots.txt
# for every mapped domain in the multisite (mit.whoi.edu, biogeoscapes.org,
# oceaniron.org, …). So there are no per-site Sitemap lines here; see the notes.
#
# Crawlers use only the most specific User-agent group that matches them,
# so the Disallow rules are repeated in each group.

# ---------------------------------------------------------------------
# Heavy crawlers: same rules plus a crawl delay.
# (Amazonbot 21.4k, ClaudeBot 13.1k, SERanking 1.9k requests in ~8 hours.)
# Crawl-delay is honored by ClaudeBot, Bing, Yandex, PetalBot, SERanking and others.
# Googlebot and Amazonbot ignore it but do follow Disallow.
# ---------------------------------------------------------------------
User-agent: Amazonbot
User-agent: ClaudeBot
User-agent: PetalBot
User-agent: SERankingBacklinksBot
User-agent: serpstatbot
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: bingbot
Crawl-delay: 5
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /dm-sso-endpoint/
Disallow: /*?*tribe-bar-date=
Disallow: /*?*posts_per_page=
Disallow: /*?*eventDisplay=
Disallow: /*?*ical=
Disallow: /*?*outlook-ical=
Disallow: /*?*tribe_
Disallow: /events/month/
Disallow: /events/week/
Disallow: /events/day/
Disallow: /events/photo/
Disallow: /events/*/day/
Disallow: /events/*/month/
Disallow: /events/*/week/
Disallow: /events/*/photo/
Disallow: /*?*eventDate=
Disallow: /*?*_rt=
Disallow: /events/list/page/
Disallow: /events/*/list/page/
Disallow: /events/1
Disallow: /events/2
Disallow: /wp-content/uploads/*/$
Disallow: /*?s=
Disallow: /search/

# ---------------------------------------------------------------------
# AI crawlers that fetch pages for datasets or knowledge bases.
# The Bedrock crawler is also blocked in Apache. This entry is for
# well-behaved agents that check robots.txt.
# ---------------------------------------------------------------------
User-agent: bedrockbot
User-agent: amazon-bedrock-knowledgebase
Disallow: /

# ---------------------------------------------------------------------
# Everyone else
# ---------------------------------------------------------------------
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /dm-sso-endpoint/
Disallow: /*?*tribe-bar-date=
Disallow: /*?*posts_per_page=
Disallow: /*?*eventDisplay=
Disallow: /*?*ical=
Disallow: /*?*outlook-ical=
Disallow: /*?*tribe_
Disallow: /events/month/
Disallow: /events/week/
Disallow: /events/day/
Disallow: /events/photo/
Disallow: /events/*/day/
Disallow: /events/*/month/
Disallow: /events/*/week/
Disallow: /events/*/photo/
Disallow: /*?*eventDate=
Disallow: /*?*_rt=
Disallow: /events/list/page/
Disallow: /events/*/list/page/
Disallow: /events/1
Disallow: /events/2
Disallow: /wp-content/uploads/*/$
Disallow: /*?s=
Disallow: /search/