fix(docs): keep versioned pages out of search indexes

- Add self-canonical noindex signals and headers for archived docs
- Restore sitemap lastmod dates from the tracked checkout

scripts/build-versioned-docs.ts

@@ -35,6 +35,17 @@ const archiveCacheRoot = join(repoRoot, '.tmp/docs-versioned-archive-cache');
 const maxCloudflareFiles = 20_000;
 const maxCloudflareFileBytes = 25 * 1024 * 1024;
 
+// Cloudflare Pages header rules for the whole deployment. Mirrors the `noindex,follow`
+// meta tag the non-root channels emit, so the duplicate trees stay out of the index
+// even for responses a crawler takes without parsing the HTML.
+const deployHeaders = `# Generated by scripts/build-versioned-docs.ts. Do not edit by hand.
+/main/*
+  X-Robots-Tag: noindex, follow
+
+/v/*
+  X-Robots-Tag: noindex, follow
+`;
+
 function run(
   command: string,
   args: string[],
@@ -173,6 +184,7 @@ function buildDocs(options: {
       SUBMINER_DOCS_BASE: options.base,
       SUBMINER_DOCS_OUT_DIR: options.outDir,
       SUBMINER_DOCS_SOURCE_DIR: options.snapshotDocsSite,
+      SUBMINER_DOCS_REPO_DIR: currentDocsSite,
       SUBMINER_DOCS_CHANNEL: options.channel,
       SUBMINER_DOCS_VERSION: options.version ?? '',
       SUBMINER_DOCS_LATEST_STABLE: options.latestStable,
@@ -378,6 +390,7 @@ function main() {
   });
 
   writeFileSync(join(aggregateOutDir, 'versions.json'), `${JSON.stringify(manifest, null, 2)}\n`);
+  writeFileSync(join(aggregateOutDir, '_headers'), deployHeaders);
   assertCloudflarePagesLimits(aggregateOutDir);
   const prunedArchives = pruneArchiveCacheGenerations({
     cacheRoot: archiveCacheRoot,