ci(docs): serve frozen version archives from R2 (#269)

This commit is contained in:
2026-09-24 15:28:58 -07:00
committed by GitHub
parent 5096ca217b
commit bc73a02623
19 changed files with 651 additions and 835 deletions
+99 -146
View File
@@ -1,49 +1,52 @@
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import {
cpSync,
existsSync,
lstatSync,
mkdirSync,
readFileSync,
readdirSync,
readlinkSync,
rmSync,
symlinkSync,
writeFileSync,
} from 'node:fs';
import { join, resolve } from 'node:path';
import {
collectSharedAssetPaths,
dedupeVersionedPublicAssets,
pruneArchiveCacheGenerations,
} from './docs-versioned-assets';
archiveStoreEnvFromProcess,
createArchiveStore,
type DocsArchiveStore,
} from './docs-archive-store';
import { collectSharedAssetPaths, dedupeVersionedPublicAssets } from './docs-versioned-assets';
import {
buildVersionManifest,
renderVersionsPage,
stableTagsWithDocs,
versionArchiveCacheKey,
versionArchiveCacheName,
versionOutputPath,
versionPath,
} from './docs-versioning';
// Assembles the Cloudflare Pages deployment: latest stable at `/` and development docs
// at `/main/`. Stable archives under `/v/<version>/` are built once, uploaded to R2, and
// never rebuilt unless requested; this script only fills in archives R2 is missing.
//
// Flags:
// --require-archives fail instead of skipping archive sync without R2 credentials
// --rebuild-archives=<list> comma-separated tags (or `all`) to rebuild even if present
const repoRoot = resolve(__dirname, '..');
const currentDocsSite = join(repoRoot, 'docs-site');
const buildRoot = join(repoRoot, '.tmp/docs-versioned-build');
const aggregateOutDir = join(repoRoot, '.tmp/docs-versioned-site');
const archiveCacheRoot = join(repoRoot, '.tmp/docs-versioned-archive-cache');
const archiveOutRoot = join(repoRoot, '.tmp/docs-versioned-archives');
const maxCloudflareFiles = 20_000;
const maxCloudflareFileBytes = 25 * 1024 * 1024;
// Cloudflare Pages header rules for the whole deployment. Mirrors the `noindex,follow`
// meta tag the non-root channels emit, so the duplicate trees stay out of the index
// even for responses a crawler takes without parsing the HTML.
// Cloudflare Pages header rules for the static deployment. Mirrors the `noindex,follow`
// meta tag the `main` channel emits, so the duplicate tree stays out of the index even
// for responses a crawler takes without parsing the HTML. `/v/*` is served by the
// archive Pages Function, which sets the same header itself.
const deployHeaders = `# Generated by scripts/build-versioned-docs.ts. Do not edit by hand.
/main/*
X-Robots-Tag: noindex, follow
/v/*
X-Robots-Tag: noindex, follow
`;
function run(
@@ -103,8 +106,7 @@ function copyCurrentDocsSite(targetDir: string) {
recursive: true,
dereference: false,
filter: (source) =>
!/[\\/]node_modules([\\/]|$)/.test(source) &&
!/[\\/]\\.vitepress[\\/]dist([\\/]|$)/.test(source),
!/[\\/]node_modules([\\/]|$)/.test(source) && !isGeneratedVitePressPath(source),
});
}
@@ -173,8 +175,6 @@ function buildDocs(options: {
outDir: string;
channel: string;
version?: string;
latestStable: string;
manifestJson: string;
}) {
console.info(`[docs] building ${options.version ?? options.channel} -> ${options.base}`);
run('bun', ['run', '--cwd', currentDocsSite, 'vitepress', 'build', options.snapshotDocsSite], {
@@ -187,98 +187,13 @@ function buildDocs(options: {
SUBMINER_DOCS_REPO_DIR: currentDocsSite,
SUBMINER_DOCS_CHANNEL: options.channel,
SUBMINER_DOCS_VERSION: options.version ?? '',
SUBMINER_DOCS_LATEST_STABLE: options.latestStable,
SUBMINER_DOCS_VERSION_MANIFEST: options.manifestJson,
VITE_EXTRA_EXTENSIONS: 'jsonc',
},
});
}
function updateHashWithPath(hash: ReturnType<typeof createHash>, path: string) {
if (isSharedInternalsHashIgnoredPath(path)) {
return;
}
const stat = lstatSync(path);
const relativePath = path.replace(repoRoot, '');
if (stat.isSymbolicLink()) {
hash.update(`symlink:${relativePath}`);
hash.update(readlinkSync(path));
return;
}
if (stat.isDirectory()) {
hash.update(`dir:${relativePath}`);
for (const entry of readdirSync(path).sort()) {
updateHashWithPath(hash, join(path, entry));
}
return;
}
hash.update(`file:${relativePath}`);
hash.update(readFileSync(path));
}
function isGeneratedVitePressPath(path: string): boolean {
return /[\\/]\\.vitepress[\\/](cache|dist)([\\/]|$)/.test(path);
}
function isSharedInternalsHashIgnoredPath(path: string): boolean {
return isGeneratedVitePressPath(path) || /\.test\.[cm]?[jt]s$/.test(path);
}
function computeSharedInternalsHash(): string {
const hash = createHash('sha256');
hash.update(
`version-link-origin:${process.env.SUBMINER_DOCS_VERSION_LINK_ORIGIN === 'local' ? 'local' : 'production'}`,
);
const paths = [
join(currentDocsSite, '.vitepress'),
join(currentDocsSite, 'public/assets/fonts'),
join(currentDocsSite, 'package.json'),
join(currentDocsSite, 'bun.lock'),
join(repoRoot, 'scripts/build-versioned-docs.ts'),
join(repoRoot, 'scripts/docs-versioning.ts'),
];
for (const path of paths) {
if (existsSync(path)) {
updateHashWithPath(hash, path);
}
}
return hash.digest('hex');
}
function archiveCachePath(version: string, sharedInternalsHash: string): string {
return join(archiveCacheRoot, versionArchiveCacheName(version, sharedInternalsHash));
}
function restoreCachedArchive(version: string, sharedInternalsHash: string): boolean {
const cachedArchive = archiveCachePath(version, sharedInternalsHash);
if (!existsSync(cachedArchive)) {
return false;
}
console.info(`[docs] cache hit ${version}`);
cpSync(cachedArchive, join(aggregateOutDir, versionOutputPath(version)), {
recursive: true,
force: true,
});
return true;
}
function saveArchiveCache(version: string, sharedInternalsHash: string) {
const outputPath = join(aggregateOutDir, versionOutputPath(version));
if (!existsSync(outputPath)) {
return;
}
const cachedArchive = archiveCachePath(version, sharedInternalsHash);
rmSync(cachedArchive, { recursive: true, force: true });
mkdirSync(archiveCacheRoot, { recursive: true });
cpSync(outputPath, cachedArchive, { recursive: true, force: true });
return /[\\/]\.vitepress[\\/](cache|dist)([\\/]|$)/.test(path);
}
function assertCloudflarePagesLimits(root: string) {
@@ -317,7 +232,66 @@ function assertCloudflarePagesLimits(root: string) {
}
}
function currentCommit(): string {
return capture('git', ['rev-parse', 'HEAD']).trim();
}
function parseArgs(argv: string[]): { requireArchives: boolean; rebuild: Set<string> | 'all' } {
let requireArchives = false;
let rebuild: Set<string> | 'all' = new Set();
for (const arg of argv) {
if (arg === '--require-archives') {
requireArchives = true;
} else if (arg.startsWith('--rebuild-archives=')) {
const value = arg.slice('--rebuild-archives='.length).trim();
rebuild =
value === 'all'
? 'all'
: new Set(
value
.split(',')
.map((tag) => tag.trim())
.filter(Boolean),
);
} else {
throw new Error(`Unknown argument: ${arg}`);
}
}
return { requireArchives, rebuild };
}
// Builds and uploads every stable archive that R2 does not already hold (or that was
// explicitly requested). Old tags are rendered with the current `.vitepress` overlay.
function syncArchives(options: {
store: DocsArchiveStore;
stableVersions: string[];
rebuild: Set<string> | 'all';
}) {
const builtFrom = currentCommit();
for (const version of options.stableVersions) {
const forced = options.rebuild === 'all' || options.rebuild.has(version);
if (!forced && options.store.has(version)) {
continue;
}
console.info(`[docs] building archive ${version}`);
const outDir = join(archiveOutRoot, versionOutputPath(version));
rmSync(outDir, { recursive: true, force: true });
buildDocs({
snapshotDocsSite: prepareSnapshot(version, version),
base: versionPath(version),
outDir,
channel: 'stable-archive',
version,
});
console.info(`[docs] uploading archive ${version}`);
options.store.upload(version, outDir, builtFrom);
rmSync(outDir, { recursive: true, force: true });
}
}
function main() {
const { requireArchives, rebuild } = parseArgs(process.argv.slice(2));
const stableVersions = getStableVersions();
const latestStable = stableVersions[0];
@@ -325,54 +299,42 @@ function main() {
throw new Error('No stable release tags with docs-site/package.json found.');
}
if (rebuild !== 'all') {
const unknown = [...rebuild].filter((tag) => !stableVersions.includes(tag));
if (unknown.length > 0) {
throw new Error(`Cannot rebuild unknown stable docs versions: ${unknown.join(', ')}`);
}
}
const manifest = buildVersionManifest({ latestStable, stableVersions });
const manifestJson = JSON.stringify(manifest);
const sharedInternalsHash = computeSharedInternalsHash();
const archiveCacheKey = versionArchiveCacheKey({ sharedInternalsHash, manifestJson });
const sharedAssetPaths = collectSharedAssetPaths(join(currentDocsSite, 'public/assets'));
console.info(`[docs] archive cache key ${archiveCacheKey.slice(0, 12)}`);
rmSync(buildRoot, { recursive: true, force: true });
rmSync(aggregateOutDir, { recursive: true, force: true });
mkdirSync(buildRoot, { recursive: true });
mkdirSync(aggregateOutDir, { recursive: true });
const store = createArchiveStore(archiveStoreEnvFromProcess(process.env));
if (store) {
syncArchives({ store, stableVersions, rebuild });
} else if (requireArchives) {
throw new Error(
'Docs archive R2 credentials are missing (CLOUDFLARE_ACCOUNT_ID, DOCS_ARCHIVE_R2_ACCESS_KEY_ID, DOCS_ARCHIVE_R2_SECRET_ACCESS_KEY, DOCS_ARCHIVE_R2_BUCKET).',
);
} else {
console.warn('[docs] R2 credentials not set; skipping /v/<version>/ archive sync');
}
const latestStableSnapshot = prepareSnapshot(latestStable, latestStable);
writeFileSync(join(latestStableSnapshot, 'versions.md'), renderVersionsPage(manifest));
buildDocs({
snapshotDocsSite: latestStableSnapshot,
base: '/',
outDir: aggregateOutDir,
channel: 'stable-root',
version: latestStable,
latestStable,
manifestJson,
});
for (const version of stableVersions) {
if (restoreCachedArchive(version, archiveCacheKey)) {
continue;
}
console.info(`[docs] rebuilding archive ${version}`);
const snapshot =
version === latestStable ? latestStableSnapshot : prepareSnapshot(version, version);
buildDocs({
snapshotDocsSite: snapshot,
base: versionPath(version),
outDir: join(aggregateOutDir, versionOutputPath(version)),
channel: 'stable-archive',
version,
latestStable,
manifestJson,
});
dedupeVersionedPublicAssets({
outDir: join(aggregateOutDir, versionOutputPath(version)),
base: versionPath(version),
sharedAssetPaths,
});
saveArchiveCache(version, archiveCacheKey);
}
const mainSnapshot = prepareSnapshot('main');
buildDocs({
snapshotDocsSite: mainSnapshot,
@@ -380,8 +342,6 @@ function main() {
outDir: join(aggregateOutDir, 'main'),
channel: 'main',
version: 'main',
latestStable,
manifestJson,
});
dedupeVersionedPublicAssets({
outDir: join(aggregateOutDir, 'main'),
@@ -392,13 +352,6 @@ function main() {
writeFileSync(join(aggregateOutDir, 'versions.json'), `${JSON.stringify(manifest, null, 2)}\n`);
writeFileSync(join(aggregateOutDir, '_headers'), deployHeaders);
assertCloudflarePagesLimits(aggregateOutDir);
const prunedArchives = pruneArchiveCacheGenerations({
cacheRoot: archiveCacheRoot,
activeCacheKey: archiveCacheKey,
});
if (prunedArchives.length > 0) {
console.info(`[docs] pruned ${prunedArchives.length} stale archive cache directories`);
}
rmSync(buildRoot, { recursive: true, force: true });
}
+107
View File
@@ -0,0 +1,107 @@
import { spawnSync } from 'node:child_process';
import { writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { versionOutputPath } from './docs-versioning';
// Frozen `/v/<version>/` doc builds live in an R2 bucket and are served by the Pages
// Function in `docs-site/functions/v/[[path]].ts`, so they never count toward the Pages
// deployment. Transfers go through the AWS CLI's S3 API (preinstalled on GitHub runners).
// Written last on upload; an archive without it is treated as missing and rebuilt.
export const ARCHIVE_MARKER = '_archive.json';
export type DocsArchiveStore = {
has(version: string): boolean;
upload(version: string, dir: string, builtFrom: string): void;
};
export type DocsArchiveStoreEnv = {
accountId?: string;
accessKeyId?: string;
secretAccessKey?: string;
bucket?: string;
};
export function archiveStoreEnvFromProcess(env: NodeJS.ProcessEnv): DocsArchiveStoreEnv {
return {
accountId: env.CLOUDFLARE_ACCOUNT_ID,
accessKeyId: env.DOCS_ARCHIVE_R2_ACCESS_KEY_ID,
secretAccessKey: env.DOCS_ARCHIVE_R2_SECRET_ACCESS_KEY,
bucket: env.DOCS_ARCHIVE_R2_BUCKET,
};
}
export function archiveKeyPrefix(version: string): string {
return `${versionOutputPath(version)}/`;
}
// Returns null when credentials are absent so local builds can skip archive sync.
export function createArchiveStore(config: DocsArchiveStoreEnv): DocsArchiveStore | null {
const { accountId, accessKeyId, secretAccessKey, bucket } = config;
if (!accountId || !accessKeyId || !secretAccessKey || !bucket) {
return null;
}
const endpoint = `https://${accountId}.r2.cloudflarestorage.com`;
const env: NodeJS.ProcessEnv = {
...process.env,
AWS_ACCESS_KEY_ID: accessKeyId,
AWS_SECRET_ACCESS_KEY: secretAccessKey,
AWS_DEFAULT_REGION: 'auto',
// AWS CLI >= 2.23 sends CRC checksums by default, which R2 does not fully support.
AWS_REQUEST_CHECKSUM_CALCULATION: 'when_required',
AWS_RESPONSE_CHECKSUM_VALIDATION: 'when_required',
};
function aws(args: string[]) {
return spawnSync('aws', [...args, '--endpoint-url', endpoint], {
env,
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
});
}
return {
has(version) {
const result = aws([
's3api',
'head-object',
'--bucket',
bucket,
'--key',
`${archiveKeyPrefix(version)}${ARCHIVE_MARKER}`,
]);
if (result.error) throw result.error;
if (result.status === 0) return true;
if (/\b404\b|Not Found/i.test(result.stderr)) return false;
throw new Error(`Unable to check docs archive ${version}: ${result.stderr.trim()}`);
},
upload(version, dir, builtFrom) {
const target = `s3://${bucket}/${archiveKeyPrefix(version)}`;
// Unmark first so a rebuild that fails partway is retried by the next deploy
// instead of being skipped as complete. Deleting a missing key succeeds.
const unmark = aws(['s3', 'rm', `${target}${ARCHIVE_MARKER}`, '--only-show-errors']);
if (unmark.error) throw unmark.error;
if (unmark.status !== 0) {
throw new Error(`Unable to unmark docs archive ${version}: ${unmark.stderr.trim()}`);
}
const sync = aws(['s3', 'sync', dir, target, '--only-show-errors']);
if (sync.error) throw sync.error;
if (sync.status !== 0) {
throw new Error(`Unable to upload docs archive ${version}: ${sync.stderr.trim()}`);
}
const markerPath = join(dir, ARCHIVE_MARKER);
writeFileSync(
markerPath,
`${JSON.stringify({ version, builtFrom, builtAt: new Date().toISOString() }, null, 2)}\n`,
);
const marker = aws(['s3', 'cp', markerPath, `${target}${ARCHIVE_MARKER}`]);
if (marker.status !== 0) {
throw new Error(`Unable to mark docs archive ${version}: ${marker.stderr.trim()}`);
}
},
};
}
+1 -30
View File
@@ -3,11 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from
import { rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
dedupeVersionedPublicAssets,
pruneArchiveCacheGenerations,
rewriteSharedAssetReferences,
} from './docs-versioned-assets';
import { dedupeVersionedPublicAssets, rewriteSharedAssetReferences } from './docs-versioned-assets';
function tempDir() {
return mkdtempSync(join(tmpdir(), 'subminer-docs-versioned-assets-'));
@@ -97,28 +93,3 @@ describe('docs versioned asset dedupe', () => {
}
});
});
describe('docs archive cache pruning', () => {
test('removes stale cache generations while keeping the active generation', async () => {
const dir = tempDir();
try {
mkdirSync(join(dir, 'active123456-v0.14.0'), { recursive: true });
mkdirSync(join(dir, 'stale654321-v0.14.0'), { recursive: true });
mkdirSync(join(dir, 'stale654321-v0.13.0'), { recursive: true });
const removed = pruneArchiveCacheGenerations({
cacheRoot: dir,
activeCacheKey: 'active123456abcdef',
});
expect(removed.sort()).toEqual([
join(dir, 'stale654321-v0.13.0'),
join(dir, 'stale654321-v0.14.0'),
]);
expect(existsSync(join(dir, 'active123456-v0.14.0'))).toBe(true);
expect(existsSync(join(dir, 'stale654321-v0.14.0'))).toBe(false);
} finally {
await rm(dir, { recursive: true, force: true });
}
});
});
-27
View File
@@ -119,30 +119,3 @@ function removeEmptyDirectories(root: string) {
rmSync(root, { recursive: true, force: true });
}
}
export function pruneArchiveCacheGenerations(options: {
cacheRoot: string;
activeCacheKey: string;
}): string[] {
if (!existsSync(options.cacheRoot)) {
return [];
}
const activePrefix = options.activeCacheKey.slice(0, 12);
const removed: string[] = [];
for (const entry of readdirSync(options.cacheRoot)) {
const path = join(options.cacheRoot, entry);
if (!lstatSync(path).isDirectory()) {
continue;
}
if (entry.startsWith(`${activePrefix}-`)) {
continue;
}
rmSync(path, { recursive: true, force: true });
removed.push(path);
}
return removed;
}
+10 -16
View File
@@ -2,10 +2,9 @@ import { describe, expect, test } from 'bun:test';
import {
buildVersionManifest,
compareStableVersionsDesc,
versionArchiveCacheKey,
isStableReleaseTag,
renderVersionsPage,
stableTagsWithDocs,
versionArchiveCacheName,
versionOutputPath,
versionPath,
} from './docs-versioning';
@@ -47,21 +46,16 @@ describe('docs versioning helpers', () => {
});
});
test('archive cache names are normalized by version and shared internals hash', () => {
expect(versionArchiveCacheName('v0.14.0', 'abcdef1234567890')).toBe('abcdef123456-v0.14.0');
});
test('versions page links every build with full page loads', () => {
const page = renderVersionsPage(
buildVersionManifest({ latestStable: 'v0.14.0', stableVersions: ['v0.14.0', 'v0.13.0'] }),
);
test('archive cache keys change when manifest contents change', () => {
const firstKey = versionArchiveCacheKey({
sharedInternalsHash: 'abcdef1234567890',
manifestJson: '{"latestStable":"v0.14.0"}',
});
const secondKey = versionArchiveCacheKey({
sharedInternalsHash: 'abcdef1234567890',
manifestJson: '{"latestStable":"v0.15.0"}',
});
expect(firstKey).not.toBe(secondKey);
expect(page).toContain('<a href="/" target="_self">Latest stable (v0.14.0)</a>');
expect(page).toContain('<a href="/main/" target="_self">main</a>');
expect(page).toContain('<a href="/v/0.14.0/" target="_self">v0.14.0</a>');
expect(page.indexOf('/v/0.14.0/')).toBeLessThan(page.indexOf('/v/0.13.0/'));
expect(page).toContain('<a href="/v/0.13.0/" target="_self">v0.13.0</a>');
});
test('archive output paths stay relative for filesystem joins', () => {
+24 -18
View File
@@ -1,5 +1,3 @@
import { createHash } from 'node:crypto';
export type DocsVersionEntry = {
version: string;
path: string;
@@ -55,22 +53,6 @@ export function versionOutputPath(version: string): string {
return `v/${version.replace(/^v/, '')}`;
}
export function versionArchiveCacheName(version: string, sharedInternalsHash: string): string {
return `${sharedInternalsHash.slice(0, 12)}-${version}`;
}
export function versionArchiveCacheKey(options: {
sharedInternalsHash: string;
manifestJson: string;
}): string {
const hash = createHash('sha256');
hash.update('shared-internals:');
hash.update(options.sharedInternalsHash);
hash.update('\nmanifest:');
hash.update(options.manifestJson);
return hash.digest('hex');
}
export function stableTagsWithDocs(
tags: string[],
hasDocsSite: (tag: string) => boolean,
@@ -94,3 +76,27 @@ export function buildVersionManifest(options: {
})),
};
}
// Markdown for the root-only `/versions` page. Archives link here instead of baking the
// release list into their nav, so an archive never needs a rebuild when a new tag ships.
// Raw anchors with `target="_self"` keep VitePress from treating the other builds as
// dead links or routing to them client-side.
export function renderVersionsPage(manifest: DocsVersionManifest): string {
const link = (path: string, text: string) => `<a href="${path}" target="_self">${text}</a>`;
return [
'---',
'title: Documentation versions',
'description: Every published version of the SubMiner documentation.',
'---',
'',
'# Documentation versions',
'',
`- ${link('/', `Latest stable (${manifest.latestStable})`)}`,
`- ${link('/main/', 'main')}: development docs, may describe unreleased behavior`,
'',
'## Stable releases',
'',
...manifest.versions.map((entry) => `- ${link(entry.path, entry.version)}`),
'',
].join('\n');
}
-41
View File
@@ -1,41 +0,0 @@
import { spawnSync } from 'node:child_process';
import { resolve } from 'node:path';
import { buildVersionManifest, stableTagsWithDocs } from './docs-versioning';
const repoRoot = resolve(__dirname, '..');
function capture(command: string, args: string[]): string {
const result = spawnSync(command, args, {
cwd: repoRoot,
encoding: 'utf8',
});
if (result.status !== 0) {
throw new Error(result.stderr || `Command failed: ${command} ${args.join(' ')}`);
}
return result.stdout;
}
function tagHasDocsSite(tag: string): boolean {
const result = spawnSync('git', ['cat-file', '-e', `${tag}:docs-site/package.json`], {
cwd: repoRoot,
});
return result.status === 0;
}
const stableVersions = stableTagsWithDocs(
capture('git', ['tag', '--list', 'v*'])
.split('\n')
.map((tag) => tag.trim())
.filter(Boolean),
tagHasDocsSite,
);
const latestStable = stableVersions[0];
if (!latestStable) {
throw new Error('No stable release tags with docs-site/package.json found.');
}
process.stdout.write(JSON.stringify(buildVersionManifest({ latestStable, stableVersions })));