# [SONNET: P1 SEO cloudflare/code fixes]
# SGSchoolKaki - robots.txt
#
# ⚠️  MAINTENANCE NOTE (2026-08-14): robots.txt groups do NOT inherit from one another.
# A crawler obeys ONLY the single most specific User-agent group that matches it and
# ignores every other group — including "User-agent: *".
# A previous version declared "User-agent: Googlebot" with Allow lines but no Disallow
# lines, which silently granted Googlebot access to /admin/, /api/, /profile/,
# /community/user/ and the whole legacy-WordPress block. If you add a named group below,
# you MUST repeat the full Disallow list inside it.

# ---------------------------------------------------------------------------
User-agent: *
Allow: /
Allow: /_next/static/

# Block admin and user-specific pages
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /profile/
Disallow: /bookmarks/
Disallow: /_next/image
Disallow: /cdn-cgi/

# Community.
#
# ⚠️  DO NOT re-add "Disallow: /community/user/" or "Disallow: /community/new".
# Verified in Google on 2026-08-14: ~30-40 /community/user/ URLs are in the
# index, all sharing one title, plus /community/new. Every one of those pages
# now serves "noindex, follow" (app/community/layout.tsx + per-route overrides).
# Google has to CRAWL a page to READ its noindex, so a Disallow here would lock
# the noindex out and freeze those entries in the index permanently. Crawling
# must stay open until the entries drop out.
#
# Re-check with:  site:sgschoolkaki.com/community/user
# When the count reaches 0 (expect several weeks), a Disallow may be restored —
# but it is not needed. The noindex alone keeps them out for good.
Allow: /community
Disallow: /community/search
Disallow: /community/admin

# Block demo/dummy blog posts from being indexed
Disallow: /blog/demo-*

# Block old WordPress/BuddyPress URLs from previous site
Disallow: /events/
Disallow: /members/
Disallow: /bp-groups/
Disallow: /item/
Disallow: /shop/
Disallow: /docs/
Disallow: /login/
Disallow: /*?replytocom=
Disallow: /*?tribe-bar-date=
Disallow: /*?ical=
Disallow: /*?outlook-ical=
Disallow: /*?has-attachment=
Disallow: /*?acpage=
Disallow: /*?offset_lower=
Disallow: /*?eventDisplay=

# ---------------------------------------------------------------------------
# AI crawlers — allowed for content discovery and summarisation.
# Each group repeats the Disallow list because groups do not inherit (see note above).
# ---------------------------------------------------------------------------
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Google-Extended
User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /profile/
Disallow: /bookmarks/
Disallow: /cdn-cgi/
Disallow: /_next/image
Disallow: /blog/demo-*

# Deliberate difference from the "*" group: these two STAY blocked here.
# The "*" group must allow /community/user/ and /community/new so GOOGLEBOT can
# crawl them and read their noindex (see the note at the top of this file).
# None of the agents below is Google's web-search crawler — Google-Extended
# governs AI training and grounding, not Search indexing — so blocking them here
# does not slow the index removal, and there is no reason to hand member profile
# pages to AI scrapers.
Disallow: /community/user/
Disallow: /community/new
Disallow: /community/search
Disallow: /community/admin

# Legacy WordPress/BuddyPress paths — repeated here because groups do not inherit
Disallow: /events/
Disallow: /members/
Disallow: /bp-groups/
Disallow: /item/
Disallow: /shop/
Disallow: /docs/
Disallow: /login/
Disallow: /*?replytocom=
Disallow: /*?tribe-bar-date=
Disallow: /*?ical=
Disallow: /*?outlook-ical=
Disallow: /*?has-attachment=
Disallow: /*?acpage=
Disallow: /*?offset_lower=
Disallow: /*?eventDisplay=

# Sitemaps
Sitemap: https://sgschoolkaki.com/sitemap.xml
# Sitemap: https://sgschoolkaki.com/sitemap-schools.xml  # Disabled - school detail pages not yet created
Sitemap: https://sgschoolkaki.com/sitemap-blog.xml
Sitemap: https://sgschoolkaki.com/sitemap-community.xml