fix(docs): keep versioned pages out of search indexes

- Add self-canonical noindex signals and headers for archived docs
- Restore sitemap lastmod dates from the tracked checkout

changes/docs-site-index-hygiene.md

@@ -0,0 +1,5 @@
+type: internal
+area: docs
+
+- Excluded the `/main/` and `/v/<version>/` docs trees from search indexing with a self-referential canonical, `noindex,follow`, and a matching `X-Robots-Tag` header, so crawlers spend their budget on the current docs instead of ~30 archived copies of every page.
+- Restored `<lastmod>` dates in the docs sitemap, which were silently dropped because production builds render from an untracked release snapshot.

docs-site/.vitepress/config.ts

@@ -1,3 +1,4 @@
+import { spawnSync } from 'node:child_process';
 import { existsSync, readFileSync, statSync } from 'node:fs';
 import { extname, join, posix, resolve, sep } from 'node:path';
 import type { DefaultTheme, HeadConfig, TransformContext, UserConfig } from 'vitepress';
@@ -26,6 +27,9 @@ function optionalEnv(value: string | undefined): string | undefined {
 const base = normalizeBase(optionalEnv(process.env.SUBMINER_DOCS_BASE) ?? '/');
 const outDir = optionalEnv(process.env.SUBMINER_DOCS_OUT_DIR);
 const docsSourceDir = optionalEnv(process.env.SUBMINER_DOCS_SOURCE_DIR) ?? process.cwd();
+// The tracked `docs-site/` checkout, which stays a git working tree even when
+// `docsSourceDir` points at an untracked release snapshot. Used for git lookups only.
+const repoDocsDir = optionalEnv(process.env.SUBMINER_DOCS_REPO_DIR) ?? process.cwd();
 const channel = normalizeChannel(optionalEnv(process.env.SUBMINER_DOCS_CHANNEL));
 const docsVersion = optionalEnv(process.env.SUBMINER_DOCS_VERSION);
 const latestStable = optionalEnv(process.env.SUBMINER_DOCS_LATEST_STABLE) ?? 'v0.18.0';
@@ -82,15 +86,18 @@ function pageToRoute(page: string): string | null {
   return route ? `/${route}` : '/';
 }
 
+// Only the root channel is indexable. `main` and every /v/<version>/ archive are
+// near-verbatim copies of it, so they own their URL via a self-referential canonical
+// and are excluded from the index instead of being consolidated onto root. Uniform
+// self-canonical plus noindex avoids mixing noindex with a cross-page canonical,
+// which Google treats as a conflicting signal.
+const isIndexableChannel = channel === 'stable-root';
+
 function pageToCanonicalHref(page: string): string | null {
   const route = pageToRoute(page);
   if (!route) return null;
 
-  if (channel === 'main') {
+  if (!isIndexableChannel) {
-    return `${DOCS_HOSTNAME}${canonicalRouteWithBase(route)}`;
-  }
-
-  if (channel === 'stable-archive' && docsVersion !== latestStable) {
     return `${DOCS_HOSTNAME}${canonicalRouteWithBase(route)}`;
   }
 
@@ -106,7 +113,9 @@ function transformPageHead({ page }: TransformContext): HeadConfig[] {
   const href = pageToCanonicalHref(page);
   const head: HeadConfig[] = href ? [['link', { rel: 'canonical', href }]] : [];
 
-  if (channel === 'main') {
+  // Crawlable so links still pass through, but out of the index: ~30 archived copies
+  // of every page otherwise soak up the crawl budget the current docs need.
+  if (!isIndexableChannel) {
     head.push(['meta', { name: 'robots', content: 'noindex,follow' }]);
   }
 
@@ -287,6 +296,39 @@ const versionItems = [
   })),
 ];
 
+function sitemapUrlToPage(url: string): string {
+  const route = url.replace(/\.html$/, '').replace(/^\/+|\/+$/g, '');
+  return route ? `${route}.md` : 'index.md';
+}
+
+// VitePress derives <lastmod> by running `git log` inside its source dir. Production
+// builds point that at an untracked snapshot of the release tag, so the lookup comes
+// back empty and the sitemap ships with no dates at all. Resolve it from the tracked
+// checkout at the ref being built instead.
+function lastModifiedFor(url: string): string | undefined {
+  const ref = docsVersion && docsVersion !== 'main' ? docsVersion : 'HEAD';
+  const result = spawnSync('git', ['log', '-1', '--format=%cI', ref, '--', sitemapUrlToPage(url)], {
+    cwd: repoDocsDir,
+    encoding: 'utf8',
+  });
+
+  return (result.status === 0 && result.stdout.trim()) || undefined;
+}
+
+// Only the root channel publishes a sitemap. Archived and `main` builds would emit
+// their own copies listing the same canonical URLs, which just advertises the
+// duplicate trees we are trying to keep out of the index.
+const sitemap: UserConfig['sitemap'] = isIndexableChannel
+  ? {
+      hostname: DOCS_HOSTNAME,
+      transformItems(items) {
+        return items
+          .filter((item) => item.url !== 'README' && item.url !== `${DOCS_HOSTNAME}/README`)
+          .map((item) => ({ ...item, lastmod: item.lastmod ?? lastModifiedFor(item.url) }));
+      },
+    }
+  : undefined;
+
 const nav: DefaultTheme.NavItem[] = [
   { text: 'Home', link: '/' },
   { text: 'Get Started', link: '/installation' },
@@ -419,14 +461,7 @@ const config: UserConfig = {
   appearance: 'dark',
   cleanUrls: true,
   metaChunk: true,
-  sitemap: {
+  sitemap,
-    hostname: DOCS_HOSTNAME,
-    transformItems(items) {
-      return items.filter(
-        (item) => item.url !== 'README' && item.url !== `${DOCS_HOSTNAME}/README`,
-      );
-    },
-  },
   transformHead: transformPageHead,
   lastUpdated: true,
   srcExclude: ['subagents/**', 'README.md'],

docs-site/README.md

@@ -38,8 +38,10 @@ bun run docs:dev
 The public docs root is stable-only:
 
 - `/` serves the latest stable release docs.
-- `/main/` serves development docs from `main` and is marked `noindex,follow`.
+- `/main/` serves development docs from `main`.
 - `/v/<version>/` serves stable release archives.
 - Prerelease tags do not update the docs site.
 
+Only `/` is indexable. `/main/` and every `/v/<version>/` page carries a self-referential canonical plus `noindex,follow`, and the generated `_headers` file repeats that as an `X-Robots-Tag`. They stay crawlable so their links still resolve, but ~30 archived copies of every page would otherwise consume the crawl budget the current docs need. Only the root build emits `sitemap.xml`, and its `<lastmod>` dates come from `git log` against the tracked checkout at the released tag, because the build renders from an untracked snapshot that VitePress cannot date itself.
+
 Keep Cloudflare Git auto-deploy disabled. The production deploy is `.github/workflows/docs-pages.yml`, which uploads `.tmp/docs-versioned-site` with `--branch main` so tag-triggered runs update Production instead of creating preview deployments.

docs-site/seo.test.ts

@@ -56,34 +56,43 @@ test('main docs canonical uses /main/ and emits noindex', async () => {
     { rel: 'canonical', href: 'https://docs.subminer.moe/main/' },
   ]);
   expect(head).toContainEqual(['meta', { name: 'robots', content: 'noindex,follow' }]);
+  expect(mainDocsConfig.sitemap).toBeUndefined();
 
   process.env.SUBMINER_DOCS_CHANNEL = previousChannel;
   process.env.SUBMINER_DOCS_BASE = previousBase;
 });
 
-test('latest stable archive canonical points to root equivalent', async () => {
+test.each([
+  ['latest stable', 'v0.14.0', '/v/0.14.0/', 'https://docs.subminer.moe/v/0.14.0/usage'],
+  ['superseded', 'v0.12.0', '/v/0.12.0/', 'https://docs.subminer.moe/v/0.12.0/usage'],
+])(
+  '%s archive keeps a self-referential canonical and stays out of the index',
+  async (_label, version, base, expectedCanonical) => {
     const previousChannel = process.env.SUBMINER_DOCS_CHANNEL;
     const previousBase = process.env.SUBMINER_DOCS_BASE;
     const previousVersion = process.env.SUBMINER_DOCS_VERSION;
     const previousLatest = process.env.SUBMINER_DOCS_LATEST_STABLE;
     process.env.SUBMINER_DOCS_CHANNEL = 'stable-archive';
-  process.env.SUBMINER_DOCS_BASE = '/v/0.14.0/';
+    process.env.SUBMINER_DOCS_BASE = base;
-  process.env.SUBMINER_DOCS_VERSION = 'v0.14.0';
+    process.env.SUBMINER_DOCS_VERSION = version;
     process.env.SUBMINER_DOCS_LATEST_STABLE = 'v0.14.0';
-  const { default: latestArchiveConfig } = await import('./.vitepress/config?latest-archive');
+    try {
+      const { default: archiveConfig } = await import(`./.vitepress/config?archive-${version}`);
 
-  const head = await latestArchiveConfig.transformHead?.(makeTransformContext('usage.md'));
+      const head = await archiveConfig.transformHead?.(makeTransformContext('usage.md'));
-
-  expect(head).toContainEqual([
-    'link',
-    { rel: 'canonical', href: 'https://docs.subminer.moe/usage' },
-  ]);
 
+      expect(head).toContainEqual(['link', { rel: 'canonical', href: expectedCanonical }]);
+      expect(head).toContainEqual(['meta', { name: 'robots', content: 'noindex,follow' }]);
+      // A sitemap here would advertise the archive tree we just excluded.
+      expect(archiveConfig.sitemap).toBeUndefined();
+    } finally {
       process.env.SUBMINER_DOCS_CHANNEL = previousChannel;
       process.env.SUBMINER_DOCS_BASE = previousBase;
       process.env.SUBMINER_DOCS_VERSION = previousVersion;
       process.env.SUBMINER_DOCS_LATEST_STABLE = previousLatest;
-});
+    }
+  },
+);
 
 test('stable archive theme links stay on the selected version', async () => {
   const previousCwd = process.cwd();
@@ -433,3 +442,22 @@ test('docs sitemap excludes duplicate README page from indexable URLs', async ()
 
   expect(transformedItems?.map((item) => item.url)).toEqual(['', 'usage']);
 });
+
+test('docs sitemap dates every URL from the tracked checkout', async () => {
+  const previousRepoDir = process.env.SUBMINER_DOCS_REPO_DIR;
+  // Production builds render from an untracked snapshot, so the date has to come from
+  // the real checkout rather than VitePress's own srcDir git lookup.
+  process.env.SUBMINER_DOCS_REPO_DIR = docsSiteDir;
+  try {
+    const { default: sitemapConfig } = await import('./.vitepress/config?sitemap-lastmod');
+
+    const items = await sitemapConfig.sitemap?.transformItems?.([{ url: '' }, { url: 'usage' }]);
+
+    expect(items).toHaveLength(2);
+    for (const item of items ?? []) {
+      expect(item.lastmod).toMatch(/^\d{4}-\d{2}-\d{2}T/);
+    }
+  } finally {
+    process.env.SUBMINER_DOCS_REPO_DIR = previousRepoDir;
+  }
+});

scripts/build-versioned-docs.ts

@@ -35,6 +35,17 @@ const archiveCacheRoot = join(repoRoot, '.tmp/docs-versioned-archive-cache');
 const maxCloudflareFiles = 20_000;
 const maxCloudflareFileBytes = 25 * 1024 * 1024;
 
+// Cloudflare Pages header rules for the whole deployment. Mirrors the `noindex,follow`
+// meta tag the non-root channels emit, so the duplicate trees stay out of the index
+// even for responses a crawler takes without parsing the HTML.
+const deployHeaders = `# Generated by scripts/build-versioned-docs.ts. Do not edit by hand.
+/main/*
+  X-Robots-Tag: noindex, follow
+
+/v/*
+  X-Robots-Tag: noindex, follow
+`;
+
 function run(
   command: string,
   args: string[],
@@ -173,6 +184,7 @@ function buildDocs(options: {
       SUBMINER_DOCS_BASE: options.base,
       SUBMINER_DOCS_OUT_DIR: options.outDir,
       SUBMINER_DOCS_SOURCE_DIR: options.snapshotDocsSite,
+      SUBMINER_DOCS_REPO_DIR: currentDocsSite,
       SUBMINER_DOCS_CHANNEL: options.channel,
       SUBMINER_DOCS_VERSION: options.version ?? '',
       SUBMINER_DOCS_LATEST_STABLE: options.latestStable,
@@ -378,6 +390,7 @@ function main() {
   });
 
   writeFileSync(join(aggregateOutDir, 'versions.json'), `${JSON.stringify(manifest, null, 2)}\n`);
+  writeFileSync(join(aggregateOutDir, '_headers'), deployHeaders);
   assertCloudflarePagesLimits(aggregateOutDir);
   const prunedArchives = pruneArchiveCacheGenerations({
     cacheRoot: archiveCacheRoot,