From f73fe179d00e131cac48d24fdb7916339443b9ae Mon Sep 17 00:00:00 2001 From: sudacode Date: Sun, 16 Aug 2026 01:44:27 -0700 Subject: [PATCH] fix(docs): keep versioned pages out of search indexes - Add self-canonical noindex signals and headers for archived docs - Restore sitemap lastmod dates from the tracked checkout --- changes/docs-site-index-hygiene.md | 5 +++ docs-site/.vitepress/config.ts | 63 +++++++++++++++++++++------ docs-site/README.md | 4 +- docs-site/seo.test.ts | 70 +++++++++++++++++++++--------- scripts/build-versioned-docs.ts | 13 ++++++ 5 files changed, 119 insertions(+), 36 deletions(-) create mode 100644 changes/docs-site-index-hygiene.md diff --git a/changes/docs-site-index-hygiene.md b/changes/docs-site-index-hygiene.md new file mode 100644 index 00000000..d988658f --- /dev/null +++ b/changes/docs-site-index-hygiene.md @@ -0,0 +1,5 @@ +type: internal +area: docs + +- Excluded the `/main/` and `/v//` docs trees from search indexing with a self-referential canonical, `noindex,follow`, and a matching `X-Robots-Tag` header, so crawlers spend their budget on the current docs instead of ~30 archived copies of every page. +- Restored `` dates in the docs sitemap, which were silently dropped because production builds render from an untracked release snapshot. diff --git a/docs-site/.vitepress/config.ts b/docs-site/.vitepress/config.ts index 42641f32..122da831 100644 --- a/docs-site/.vitepress/config.ts +++ b/docs-site/.vitepress/config.ts @@ -1,3 +1,4 @@ +import { spawnSync } from 'node:child_process'; import { existsSync, readFileSync, statSync } from 'node:fs'; import { extname, join, posix, resolve, sep } from 'node:path'; import type { DefaultTheme, HeadConfig, TransformContext, UserConfig } from 'vitepress'; @@ -26,6 +27,9 @@ function optionalEnv(value: string | undefined): string | undefined { const base = normalizeBase(optionalEnv(process.env.SUBMINER_DOCS_BASE) ?? '/'); const outDir = optionalEnv(process.env.SUBMINER_DOCS_OUT_DIR); const docsSourceDir = optionalEnv(process.env.SUBMINER_DOCS_SOURCE_DIR) ?? process.cwd(); +// The tracked `docs-site/` checkout, which stays a git working tree even when +// `docsSourceDir` points at an untracked release snapshot. Used for git lookups only. +const repoDocsDir = optionalEnv(process.env.SUBMINER_DOCS_REPO_DIR) ?? process.cwd(); const channel = normalizeChannel(optionalEnv(process.env.SUBMINER_DOCS_CHANNEL)); const docsVersion = optionalEnv(process.env.SUBMINER_DOCS_VERSION); const latestStable = optionalEnv(process.env.SUBMINER_DOCS_LATEST_STABLE) ?? 'v0.18.0'; @@ -82,15 +86,18 @@ function pageToRoute(page: string): string | null { return route ? `/${route}` : '/'; } +// Only the root channel is indexable. `main` and every /v// archive are +// near-verbatim copies of it, so they own their URL via a self-referential canonical +// and are excluded from the index instead of being consolidated onto root. Uniform +// self-canonical plus noindex avoids mixing noindex with a cross-page canonical, +// which Google treats as a conflicting signal. +const isIndexableChannel = channel === 'stable-root'; + function pageToCanonicalHref(page: string): string | null { const route = pageToRoute(page); if (!route) return null; - if (channel === 'main') { - return `${DOCS_HOSTNAME}${canonicalRouteWithBase(route)}`; - } - - if (channel === 'stable-archive' && docsVersion !== latestStable) { + if (!isIndexableChannel) { return `${DOCS_HOSTNAME}${canonicalRouteWithBase(route)}`; } @@ -106,7 +113,9 @@ function transformPageHead({ page }: TransformContext): HeadConfig[] { const href = pageToCanonicalHref(page); const head: HeadConfig[] = href ? [['link', { rel: 'canonical', href }]] : []; - if (channel === 'main') { + // Crawlable so links still pass through, but out of the index: ~30 archived copies + // of every page otherwise soak up the crawl budget the current docs need. + if (!isIndexableChannel) { head.push(['meta', { name: 'robots', content: 'noindex,follow' }]); } @@ -287,6 +296,39 @@ const versionItems = [ })), ]; +function sitemapUrlToPage(url: string): string { + const route = url.replace(/\.html$/, '').replace(/^\/+|\/+$/g, ''); + return route ? `${route}.md` : 'index.md'; +} + +// VitePress derives by running `git log` inside its source dir. Production +// builds point that at an untracked snapshot of the release tag, so the lookup comes +// back empty and the sitemap ships with no dates at all. Resolve it from the tracked +// checkout at the ref being built instead. +function lastModifiedFor(url: string): string | undefined { + const ref = docsVersion && docsVersion !== 'main' ? docsVersion : 'HEAD'; + const result = spawnSync('git', ['log', '-1', '--format=%cI', ref, '--', sitemapUrlToPage(url)], { + cwd: repoDocsDir, + encoding: 'utf8', + }); + + return (result.status === 0 && result.stdout.trim()) || undefined; +} + +// Only the root channel publishes a sitemap. Archived and `main` builds would emit +// their own copies listing the same canonical URLs, which just advertises the +// duplicate trees we are trying to keep out of the index. +const sitemap: UserConfig['sitemap'] = isIndexableChannel + ? { + hostname: DOCS_HOSTNAME, + transformItems(items) { + return items + .filter((item) => item.url !== 'README' && item.url !== `${DOCS_HOSTNAME}/README`) + .map((item) => ({ ...item, lastmod: item.lastmod ?? lastModifiedFor(item.url) })); + }, + } + : undefined; + const nav: DefaultTheme.NavItem[] = [ { text: 'Home', link: '/' }, { text: 'Get Started', link: '/installation' }, @@ -419,14 +461,7 @@ const config: UserConfig = { appearance: 'dark', cleanUrls: true, metaChunk: true, - sitemap: { - hostname: DOCS_HOSTNAME, - transformItems(items) { - return items.filter( - (item) => item.url !== 'README' && item.url !== `${DOCS_HOSTNAME}/README`, - ); - }, - }, + sitemap, transformHead: transformPageHead, lastUpdated: true, srcExclude: ['subagents/**', 'README.md'], diff --git a/docs-site/README.md b/docs-site/README.md index 8386e3b2..16d10ef4 100644 --- a/docs-site/README.md +++ b/docs-site/README.md @@ -38,8 +38,10 @@ bun run docs:dev The public docs root is stable-only: - `/` serves the latest stable release docs. -- `/main/` serves development docs from `main` and is marked `noindex,follow`. +- `/main/` serves development docs from `main`. - `/v//` serves stable release archives. - Prerelease tags do not update the docs site. +Only `/` is indexable. `/main/` and every `/v//` page carries a self-referential canonical plus `noindex,follow`, and the generated `_headers` file repeats that as an `X-Robots-Tag`. They stay crawlable so their links still resolve, but ~30 archived copies of every page would otherwise consume the crawl budget the current docs need. Only the root build emits `sitemap.xml`, and its `` dates come from `git log` against the tracked checkout at the released tag, because the build renders from an untracked snapshot that VitePress cannot date itself. + Keep Cloudflare Git auto-deploy disabled. The production deploy is `.github/workflows/docs-pages.yml`, which uploads `.tmp/docs-versioned-site` with `--branch main` so tag-triggered runs update Production instead of creating preview deployments. diff --git a/docs-site/seo.test.ts b/docs-site/seo.test.ts index 0ff98e27..99980516 100644 --- a/docs-site/seo.test.ts +++ b/docs-site/seo.test.ts @@ -56,34 +56,43 @@ test('main docs canonical uses /main/ and emits noindex', async () => { { rel: 'canonical', href: 'https://docs.subminer.moe/main/' }, ]); expect(head).toContainEqual(['meta', { name: 'robots', content: 'noindex,follow' }]); + expect(mainDocsConfig.sitemap).toBeUndefined(); process.env.SUBMINER_DOCS_CHANNEL = previousChannel; process.env.SUBMINER_DOCS_BASE = previousBase; }); -test('latest stable archive canonical points to root equivalent', async () => { - const previousChannel = process.env.SUBMINER_DOCS_CHANNEL; - const previousBase = process.env.SUBMINER_DOCS_BASE; - const previousVersion = process.env.SUBMINER_DOCS_VERSION; - const previousLatest = process.env.SUBMINER_DOCS_LATEST_STABLE; - process.env.SUBMINER_DOCS_CHANNEL = 'stable-archive'; - process.env.SUBMINER_DOCS_BASE = '/v/0.14.0/'; - process.env.SUBMINER_DOCS_VERSION = 'v0.14.0'; - process.env.SUBMINER_DOCS_LATEST_STABLE = 'v0.14.0'; - const { default: latestArchiveConfig } = await import('./.vitepress/config?latest-archive'); +test.each([ + ['latest stable', 'v0.14.0', '/v/0.14.0/', 'https://docs.subminer.moe/v/0.14.0/usage'], + ['superseded', 'v0.12.0', '/v/0.12.0/', 'https://docs.subminer.moe/v/0.12.0/usage'], +])( + '%s archive keeps a self-referential canonical and stays out of the index', + async (_label, version, base, expectedCanonical) => { + const previousChannel = process.env.SUBMINER_DOCS_CHANNEL; + const previousBase = process.env.SUBMINER_DOCS_BASE; + const previousVersion = process.env.SUBMINER_DOCS_VERSION; + const previousLatest = process.env.SUBMINER_DOCS_LATEST_STABLE; + process.env.SUBMINER_DOCS_CHANNEL = 'stable-archive'; + process.env.SUBMINER_DOCS_BASE = base; + process.env.SUBMINER_DOCS_VERSION = version; + process.env.SUBMINER_DOCS_LATEST_STABLE = 'v0.14.0'; + try { + const { default: archiveConfig } = await import(`./.vitepress/config?archive-${version}`); - const head = await latestArchiveConfig.transformHead?.(makeTransformContext('usage.md')); + const head = await archiveConfig.transformHead?.(makeTransformContext('usage.md')); - expect(head).toContainEqual([ - 'link', - { rel: 'canonical', href: 'https://docs.subminer.moe/usage' }, - ]); - - process.env.SUBMINER_DOCS_CHANNEL = previousChannel; - process.env.SUBMINER_DOCS_BASE = previousBase; - process.env.SUBMINER_DOCS_VERSION = previousVersion; - process.env.SUBMINER_DOCS_LATEST_STABLE = previousLatest; -}); + expect(head).toContainEqual(['link', { rel: 'canonical', href: expectedCanonical }]); + expect(head).toContainEqual(['meta', { name: 'robots', content: 'noindex,follow' }]); + // A sitemap here would advertise the archive tree we just excluded. + expect(archiveConfig.sitemap).toBeUndefined(); + } finally { + process.env.SUBMINER_DOCS_CHANNEL = previousChannel; + process.env.SUBMINER_DOCS_BASE = previousBase; + process.env.SUBMINER_DOCS_VERSION = previousVersion; + process.env.SUBMINER_DOCS_LATEST_STABLE = previousLatest; + } + }, +); test('stable archive theme links stay on the selected version', async () => { const previousCwd = process.cwd(); @@ -433,3 +442,22 @@ test('docs sitemap excludes duplicate README page from indexable URLs', async () expect(transformedItems?.map((item) => item.url)).toEqual(['', 'usage']); }); + +test('docs sitemap dates every URL from the tracked checkout', async () => { + const previousRepoDir = process.env.SUBMINER_DOCS_REPO_DIR; + // Production builds render from an untracked snapshot, so the date has to come from + // the real checkout rather than VitePress's own srcDir git lookup. + process.env.SUBMINER_DOCS_REPO_DIR = docsSiteDir; + try { + const { default: sitemapConfig } = await import('./.vitepress/config?sitemap-lastmod'); + + const items = await sitemapConfig.sitemap?.transformItems?.([{ url: '' }, { url: 'usage' }]); + + expect(items).toHaveLength(2); + for (const item of items ?? []) { + expect(item.lastmod).toMatch(/^\d{4}-\d{2}-\d{2}T/); + } + } finally { + process.env.SUBMINER_DOCS_REPO_DIR = previousRepoDir; + } +}); diff --git a/scripts/build-versioned-docs.ts b/scripts/build-versioned-docs.ts index b41dafae..012de33f 100644 --- a/scripts/build-versioned-docs.ts +++ b/scripts/build-versioned-docs.ts @@ -35,6 +35,17 @@ const archiveCacheRoot = join(repoRoot, '.tmp/docs-versioned-archive-cache'); const maxCloudflareFiles = 20_000; const maxCloudflareFileBytes = 25 * 1024 * 1024; +// Cloudflare Pages header rules for the whole deployment. Mirrors the `noindex,follow` +// meta tag the non-root channels emit, so the duplicate trees stay out of the index +// even for responses a crawler takes without parsing the HTML. +const deployHeaders = `# Generated by scripts/build-versioned-docs.ts. Do not edit by hand. +/main/* + X-Robots-Tag: noindex, follow + +/v/* + X-Robots-Tag: noindex, follow +`; + function run( command: string, args: string[], @@ -173,6 +184,7 @@ function buildDocs(options: { SUBMINER_DOCS_BASE: options.base, SUBMINER_DOCS_OUT_DIR: options.outDir, SUBMINER_DOCS_SOURCE_DIR: options.snapshotDocsSite, + SUBMINER_DOCS_REPO_DIR: currentDocsSite, SUBMINER_DOCS_CHANNEL: options.channel, SUBMINER_DOCS_VERSION: options.version ?? '', SUBMINER_DOCS_LATEST_STABLE: options.latestStable, @@ -378,6 +390,7 @@ function main() { }); writeFileSync(join(aggregateOutDir, 'versions.json'), `${JSON.stringify(manifest, null, 2)}\n`); + writeFileSync(join(aggregateOutDir, '_headers'), deployHeaders); assertCloudflarePagesLimits(aggregateOutDir); const prunedArchives = pruneArchiveCacheGenerations({ cacheRoot: archiveCacheRoot,