SubMiner/scripts/build-versioned-docs.ts

import { spawnSync } from 'node:child_process';
import {
  cpSync,
  existsSync,
  lstatSync,
  mkdirSync,
  readdirSync,
  rmSync,
  symlinkSync,
  writeFileSync,
} from 'node:fs';
import { join, resolve } from 'node:path';
import {
  archiveStoreEnvFromProcess,
  createArchiveStore,
  type DocsArchiveStore,
} from './docs-archive-store';
import { collectSharedAssetPaths, dedupeVersionedPublicAssets } from './docs-versioned-assets';
import {
  buildVersionManifest,
  renderVersionsPage,
  stableTagsWithDocs,
  versionOutputPath,
  versionPath,
} from './docs-versioning';

// Assembles the Cloudflare Pages deployment: latest stable at `/` and development docs
// at `/main/`. Stable archives under `/v/<version>/` are built once, uploaded to R2, and
// never rebuilt unless requested; this script only fills in archives R2 is missing.
//
// Flags:
//   --require-archives           fail instead of skipping archive sync without R2 credentials
//   --rebuild-archives=<list>    comma-separated tags (or `all`) to rebuild even if present

const repoRoot = resolve(__dirname, '..');
const currentDocsSite = join(repoRoot, 'docs-site');
const buildRoot = join(repoRoot, '.tmp/docs-versioned-build');
const aggregateOutDir = join(repoRoot, '.tmp/docs-versioned-site');
const archiveOutRoot = join(repoRoot, '.tmp/docs-versioned-archives');
const maxCloudflareFiles = 20_000;
const maxCloudflareFileBytes = 25 * 1024 * 1024;

// Cloudflare Pages header rules for the static deployment. Mirrors the `noindex,follow`
// meta tag the `main` channel emits, so the duplicate tree stays out of the index even
// for responses a crawler takes without parsing the HTML. `/v/*` is served by the
// archive Pages Function, which sets the same header itself.
const deployHeaders = `# Generated by scripts/build-versioned-docs.ts. Do not edit by hand.
/main/*
  X-Robots-Tag: noindex, follow
`;

function run(
  command: string,
  args: string[],
  options: { cwd?: string; env?: NodeJS.ProcessEnv } = {},
) {
  const result = spawnSync(command, args, {
    cwd: options.cwd ?? repoRoot,
    env: options.env ?? process.env,
    stdio: 'inherit',
  });

  if (result.status !== 0) {
    throw new Error(`Command failed: ${command} ${args.join(' ')}`);
  }
}

function capture(command: string, args: string[]): string {
  const result = spawnSync(command, args, {
    cwd: repoRoot,
    encoding: 'utf8',
  });

  if (result.status !== 0) {
    throw new Error(result.stderr || `Command failed: ${command} ${args.join(' ')}`);
  }

  return result.stdout;
}

function archiveDocsSite(ref: string, targetDir: string) {
  mkdirSync(targetDir, { recursive: true });
  const archive = spawnSync('git', ['archive', '--format=tar', ref, 'docs-site'], {
    cwd: repoRoot,
    encoding: 'buffer',
    maxBuffer: 1024 * 1024 * 1024,
  });

  if (archive.status !== 0 || !archive.stdout) {
    throw new Error(`Unable to archive docs-site from ${ref}`);
  }

  const extract = spawnSync('tar', ['-x', '-C', targetDir], {
    input: archive.stdout,
    stdio: ['pipe', 'inherit', 'inherit'],
  });

  if (extract.status !== 0) {
    throw new Error(`Unable to extract docs-site archive from ${ref}`);
  }
}

function copyCurrentDocsSite(targetDir: string) {
  mkdirSync(targetDir, { recursive: true });
  cpSync(currentDocsSite, join(targetDir, 'docs-site'), {
    recursive: true,
    dereference: false,
    filter: (source) =>
      !/[\\/]node_modules([\\/]|$)/.test(source) && !isGeneratedVitePressPath(source),
  });
}

function overlayCurrentVitePress(snapshotDocsSite: string) {
  const targetVitePress = join(snapshotDocsSite, '.vitepress');
  rmSync(targetVitePress, { recursive: true, force: true });
  cpSync(join(currentDocsSite, '.vitepress'), targetVitePress, {
    recursive: true,
    filter: (source) => !isGeneratedVitePressPath(source),
  });

  const currentThemeFonts = join(currentDocsSite, 'public/assets/fonts');
  if (existsSync(currentThemeFonts)) {
    cpSync(currentThemeFonts, join(snapshotDocsSite, 'public/assets/fonts'), {
      recursive: true,
      force: true,
    });
  }
}

function linkDocsDependencies(snapshotDocsSite: string) {
  const currentNodeModules = join(currentDocsSite, 'node_modules');
  const targetNodeModules = join(snapshotDocsSite, 'node_modules');

  if (!existsSync(currentNodeModules) || existsSync(targetNodeModules)) {
    return;
  }

  symlinkSync(currentNodeModules, targetNodeModules, 'dir');
}

function prepareSnapshot(name: string, ref?: string): string {
  const snapshotRoot = join(buildRoot, name);
  rmSync(snapshotRoot, { recursive: true, force: true });

  if (ref) {
    archiveDocsSite(ref, snapshotRoot);
  } else {
    copyCurrentDocsSite(snapshotRoot);
  }

  const snapshotDocsSite = join(snapshotRoot, 'docs-site');
  overlayCurrentVitePress(snapshotDocsSite);
  linkDocsDependencies(snapshotDocsSite);
  return snapshotDocsSite;
}

function tagHasDocsSite(tag: string): boolean {
  const result = spawnSync('git', ['cat-file', '-e', `${tag}:docs-site/package.json`], {
    cwd: repoRoot,
  });
  return result.status === 0;
}

function getStableVersions(): string[] {
  const tags = capture('git', ['tag', '--list', 'v*'])
    .split('\n')
    .map((tag) => tag.trim())
    .filter(Boolean);
  return stableTagsWithDocs(tags, tagHasDocsSite);
}

function buildDocs(options: {
  snapshotDocsSite: string;
  base: string;
  outDir: string;
  channel: string;
  version?: string;
}) {
  console.info(`[docs] building ${options.version ?? options.channel} -> ${options.base}`);
  run('bun', ['run', '--cwd', currentDocsSite, 'vitepress', 'build', options.snapshotDocsSite], {
    cwd: repoRoot,
    env: {
      ...process.env,
      SUBMINER_DOCS_BASE: options.base,
      SUBMINER_DOCS_OUT_DIR: options.outDir,
      SUBMINER_DOCS_SOURCE_DIR: options.snapshotDocsSite,
      SUBMINER_DOCS_REPO_DIR: currentDocsSite,
      SUBMINER_DOCS_CHANNEL: options.channel,
      SUBMINER_DOCS_VERSION: options.version ?? '',
      VITE_EXTRA_EXTENSIONS: 'jsonc',
    },
  });
}

function isGeneratedVitePressPath(path: string): boolean {
  return /[\\/]\.vitepress[\\/](cache|dist)([\\/]|$)/.test(path);
}

function assertCloudflarePagesLimits(root: string) {
  let fileCount = 0;
  const oversizedFiles: string[] = [];

  function walk(dir: string) {
    for (const entry of readdirSync(dir)) {
      const path = join(dir, entry);
      const stat = lstatSync(path);
      if (stat.isSymbolicLink()) {
        continue;
      }
      if (stat.isDirectory()) {
        walk(path);
        continue;
      }

      fileCount += 1;
      if (stat.size > maxCloudflareFileBytes) {
        oversizedFiles.push(path);
      }
    }
  }

  walk(root);

  if (fileCount > maxCloudflareFiles) {
    throw new Error(
      `Versioned docs output has ${fileCount} files; Cloudflare Pages free plan limit is ${maxCloudflareFiles}.`,
    );
  }

  if (oversizedFiles.length > 0) {
    throw new Error(`Versioned docs output has files over 25 MiB:\n${oversizedFiles.join('\n')}`);
  }
}

function currentCommit(): string {
  return capture('git', ['rev-parse', 'HEAD']).trim();
}

function parseArgs(argv: string[]): { requireArchives: boolean; rebuild: Set<string> | 'all' } {
  let requireArchives = false;
  let rebuild: Set<string> | 'all' = new Set();
  for (const arg of argv) {
    if (arg === '--require-archives') {
      requireArchives = true;
    } else if (arg.startsWith('--rebuild-archives=')) {
      const value = arg.slice('--rebuild-archives='.length).trim();
      rebuild =
        value === 'all'
          ? 'all'
          : new Set(
              value
                .split(',')
                .map((tag) => tag.trim())
                .filter(Boolean),
            );
    } else {
      throw new Error(`Unknown argument: ${arg}`);
    }
  }
  return { requireArchives, rebuild };
}

// Builds and uploads every stable archive that R2 does not already hold (or that was
// explicitly requested). Old tags are rendered with the current `.vitepress` overlay.
function syncArchives(options: {
  store: DocsArchiveStore;
  stableVersions: string[];
  rebuild: Set<string> | 'all';
}) {
  const builtFrom = currentCommit();
  for (const version of options.stableVersions) {
    const forced = options.rebuild === 'all' || options.rebuild.has(version);
    if (!forced && options.store.has(version)) {
      continue;
    }

    console.info(`[docs] building archive ${version}`);
    const outDir = join(archiveOutRoot, versionOutputPath(version));
    rmSync(outDir, { recursive: true, force: true });
    buildDocs({
      snapshotDocsSite: prepareSnapshot(version, version),
      base: versionPath(version),
      outDir,
      channel: 'stable-archive',
      version,
    });
    console.info(`[docs] uploading archive ${version}`);
    options.store.upload(version, outDir, builtFrom);
    rmSync(outDir, { recursive: true, force: true });
  }
}

function main() {
  const { requireArchives, rebuild } = parseArgs(process.argv.slice(2));
  const stableVersions = getStableVersions();
  const latestStable = stableVersions[0];

  if (!latestStable) {
    throw new Error('No stable release tags with docs-site/package.json found.');
  }

  if (rebuild !== 'all') {
    const unknown = [...rebuild].filter((tag) => !stableVersions.includes(tag));
    if (unknown.length > 0) {
      throw new Error(`Cannot rebuild unknown stable docs versions: ${unknown.join(', ')}`);
    }
  }

  const manifest = buildVersionManifest({ latestStable, stableVersions });
  const sharedAssetPaths = collectSharedAssetPaths(join(currentDocsSite, 'public/assets'));

  rmSync(buildRoot, { recursive: true, force: true });
  rmSync(aggregateOutDir, { recursive: true, force: true });
  mkdirSync(buildRoot, { recursive: true });
  mkdirSync(aggregateOutDir, { recursive: true });

  const store = createArchiveStore(archiveStoreEnvFromProcess(process.env));
  if (store) {
    syncArchives({ store, stableVersions, rebuild });
  } else if (requireArchives) {
    throw new Error(
      'Docs archive R2 credentials are missing (CLOUDFLARE_ACCOUNT_ID, DOCS_ARCHIVE_R2_ACCESS_KEY_ID, DOCS_ARCHIVE_R2_SECRET_ACCESS_KEY, DOCS_ARCHIVE_R2_BUCKET).',
    );
  } else {
    console.warn('[docs] R2 credentials not set; skipping /v/<version>/ archive sync');
  }

  const latestStableSnapshot = prepareSnapshot(latestStable, latestStable);
  writeFileSync(join(latestStableSnapshot, 'versions.md'), renderVersionsPage(manifest));
  buildDocs({
    snapshotDocsSite: latestStableSnapshot,
    base: '/',
    outDir: aggregateOutDir,
    channel: 'stable-root',
    version: latestStable,
  });

  const mainSnapshot = prepareSnapshot('main');
  buildDocs({
    snapshotDocsSite: mainSnapshot,
    base: '/main/',
    outDir: join(aggregateOutDir, 'main'),
    channel: 'main',
    version: 'main',
  });
  dedupeVersionedPublicAssets({
    outDir: join(aggregateOutDir, 'main'),
    base: '/main/',
    sharedAssetPaths,
  });

  writeFileSync(join(aggregateOutDir, 'versions.json'), `${JSON.stringify(manifest, null, 2)}\n`);
  writeFileSync(join(aggregateOutDir, '_headers'), deployHeaders);
  assertCloudflarePagesLimits(aggregateOutDir);
  rmSync(buildRoot, { recursive: true, force: true });
}

main();