sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-09-25 05:16:19 -07:00
Code Issues Packages Projects Releases Wiki Activity

ci(docs): serve frozen version archives from R2 (#269)

Browse Source
This commit is contained in:
sudacode
2026-09-24 15:28:58 -07:00
committed by GitHub
parent 5096ca217b
commit bc73a02623
19 changed files with 651 additions and 835 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build-versioned-docs.ts
+99 -146
View File
@@ -1,49 +1,52 @@
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import {
cpSync,
existsSync,
lstatSync,
mkdirSync,
readFileSync,
readdirSync,
readlinkSync,
rmSync,
symlinkSync,
writeFileSync,
} from 'node:fs';
import { join, resolve } from 'node:path';
import {
collectSharedAssetPaths,
dedupeVersionedPublicAssets,
pruneArchiveCacheGenerations,
} from './docs-versioned-assets';
archiveStoreEnvFromProcess,
createArchiveStore,
type DocsArchiveStore,
} from './docs-archive-store';
import { collectSharedAssetPaths, dedupeVersionedPublicAssets } from './docs-versioned-assets';
import {
buildVersionManifest,
renderVersionsPage,
stableTagsWithDocs,
versionArchiveCacheKey,
versionArchiveCacheName,
versionOutputPath,
versionPath,
} from './docs-versioning';
// Assembles the Cloudflare Pages deployment: latest stable at `/` and development docs
// at `/main/`. Stable archives under `/v/<version>/` are built once, uploaded to R2, and
// never rebuilt unless requested; this script only fills in archives R2 is missing.
//
// Flags:
// --require-archives fail instead of skipping archive sync without R2 credentials
// --rebuild-archives=<list> comma-separated tags (or `all`) to rebuild even if present
const repoRoot = resolve(__dirname, '..');
const currentDocsSite = join(repoRoot, 'docs-site');
const buildRoot = join(repoRoot, '.tmp/docs-versioned-build');
const aggregateOutDir = join(repoRoot, '.tmp/docs-versioned-site');
const archiveCacheRoot = join(repoRoot, '.tmp/docs-versioned-archive-cache');
const archiveOutRoot = join(repoRoot, '.tmp/docs-versioned-archives');
const maxCloudflareFiles = 20_000;
const maxCloudflareFileBytes = 25 * 1024 * 1024;
// Cloudflare Pages header rules for the whole deployment. Mirrors the `noindex,follow`
// meta tag the non-root channels emit, so the duplicate trees stay out of the index
// even for responses a crawler takes without parsing the HTML.
// Cloudflare Pages header rules for the static deployment. Mirrors the `noindex,follow`
// meta tag the `main` channel emits, so the duplicate tree stays out of the index even
// for responses a crawler takes without parsing the HTML. `/v/*` is served by the
// archive Pages Function, which sets the same header itself.
const deployHeaders = `# Generated by scripts/build-versioned-docs.ts. Do not edit by hand.
/main/*
X-Robots-Tag: noindex, follow
/v/*
X-Robots-Tag: noindex, follow
`;
function run(
@@ -103,8 +106,7 @@ function copyCurrentDocsSite(targetDir: string) {
recursive: true,
dereference: false,
filter: (source) =>
!/[\\/]node_modules([\\/]|$)/.test(source) &&
!/[\\/]\\.vitepress[\\/]dist([\\/]|$)/.test(source),
!/[\\/]node_modules([\\/]|$)/.test(source) && !isGeneratedVitePressPath(source),
});
}
@@ -173,8 +175,6 @@ function buildDocs(options: {
outDir: string;
channel: string;
version?: string;
latestStable: string;
manifestJson: string;
}) {
console.info(`[docs] building ${options.version ?? options.channel} -> ${options.base}`);
run('bun', ['run', '--cwd', currentDocsSite, 'vitepress', 'build', options.snapshotDocsSite], {
@@ -187,98 +187,13 @@ function buildDocs(options: {
SUBMINER_DOCS_REPO_DIR: currentDocsSite,
SUBMINER_DOCS_CHANNEL: options.channel,
SUBMINER_DOCS_VERSION: options.version ?? '',
SUBMINER_DOCS_LATEST_STABLE: options.latestStable,
SUBMINER_DOCS_VERSION_MANIFEST: options.manifestJson,
VITE_EXTRA_EXTENSIONS: 'jsonc',
},
});
}
function updateHashWithPath(hash: ReturnType<typeof createHash>, path: string) {
if (isSharedInternalsHashIgnoredPath(path)) {
return;
}
const stat = lstatSync(path);
const relativePath = path.replace(repoRoot, '');
if (stat.isSymbolicLink()) {
hash.update(`symlink:${relativePath}`);
hash.update(readlinkSync(path));
return;
}
if (stat.isDirectory()) {
hash.update(`dir:${relativePath}`);
for (const entry of readdirSync(path).sort()) {
updateHashWithPath(hash, join(path, entry));
}
return;
}
hash.update(`file:${relativePath}`);
hash.update(readFileSync(path));
}
function isGeneratedVitePressPath(path: string): boolean {
return /[\\/]\\.vitepress[\\/](cache|dist)([\\/]|$)/.test(path);
}
function isSharedInternalsHashIgnoredPath(path: string): boolean {
return isGeneratedVitePressPath(path) || /\.test\.[cm]?[jt]s$/.test(path);
}
function computeSharedInternalsHash(): string {
const hash = createHash('sha256');
hash.update(
`version-link-origin:${process.env.SUBMINER_DOCS_VERSION_LINK_ORIGIN === 'local' ? 'local' : 'production'}`,
);
const paths = [
join(currentDocsSite, '.vitepress'),
join(currentDocsSite, 'public/assets/fonts'),
join(currentDocsSite, 'package.json'),
join(currentDocsSite, 'bun.lock'),
join(repoRoot, 'scripts/build-versioned-docs.ts'),
join(repoRoot, 'scripts/docs-versioning.ts'),
];
for (const path of paths) {
if (existsSync(path)) {
updateHashWithPath(hash, path);
}
}
return hash.digest('hex');
}
function archiveCachePath(version: string, sharedInternalsHash: string): string {
return join(archiveCacheRoot, versionArchiveCacheName(version, sharedInternalsHash));
}
function restoreCachedArchive(version: string, sharedInternalsHash: string): boolean {
const cachedArchive = archiveCachePath(version, sharedInternalsHash);
if (!existsSync(cachedArchive)) {
return false;
}
console.info(`[docs] cache hit ${version}`);
cpSync(cachedArchive, join(aggregateOutDir, versionOutputPath(version)), {
recursive: true,
force: true,
});
return true;
}
function saveArchiveCache(version: string, sharedInternalsHash: string) {
const outputPath = join(aggregateOutDir, versionOutputPath(version));
if (!existsSync(outputPath)) {
return;
}
const cachedArchive = archiveCachePath(version, sharedInternalsHash);
rmSync(cachedArchive, { recursive: true, force: true });
mkdirSync(archiveCacheRoot, { recursive: true });
cpSync(outputPath, cachedArchive, { recursive: true, force: true });
return /[\\/]\.vitepress[\\/](cache|dist)([\\/]|$)/.test(path);
}
function assertCloudflarePagesLimits(root: string) {
@@ -317,7 +232,66 @@ function assertCloudflarePagesLimits(root: string) {
}
}
function currentCommit(): string {
return capture('git', ['rev-parse', 'HEAD']).trim();
}
function parseArgs(argv: string[]): { requireArchives: boolean; rebuild: Set<string> | 'all' } {
let requireArchives = false;
let rebuild: Set<string> | 'all' = new Set();
for (const arg of argv) {
if (arg === '--require-archives') {
requireArchives = true;
} else if (arg.startsWith('--rebuild-archives=')) {
const value = arg.slice('--rebuild-archives='.length).trim();
rebuild =
value === 'all'
? 'all'
: new Set(
value
.split(',')
.map((tag) => tag.trim())
.filter(Boolean),
);
} else {
throw new Error(`Unknown argument: ${arg}`);
}
}
return { requireArchives, rebuild };
}
// Builds and uploads every stable archive that R2 does not already hold (or that was
// explicitly requested). Old tags are rendered with the current `.vitepress` overlay.
function syncArchives(options: {
store: DocsArchiveStore;
stableVersions: string[];
rebuild: Set<string> | 'all';
}) {
const builtFrom = currentCommit();
for (const version of options.stableVersions) {
const forced = options.rebuild === 'all' || options.rebuild.has(version);
if (!forced && options.store.has(version)) {
continue;
}
console.info(`[docs] building archive ${version}`);
const outDir = join(archiveOutRoot, versionOutputPath(version));
rmSync(outDir, { recursive: true, force: true });
buildDocs({
snapshotDocsSite: prepareSnapshot(version, version),
base: versionPath(version),
outDir,
channel: 'stable-archive',
version,
});
console.info(`[docs] uploading archive ${version}`);
options.store.upload(version, outDir, builtFrom);
rmSync(outDir, { recursive: true, force: true });
}
}
function main() {
const { requireArchives, rebuild } = parseArgs(process.argv.slice(2));
const stableVersions = getStableVersions();
const latestStable = stableVersions[0];
@@ -325,54 +299,42 @@ function main() {
throw new Error('No stable release tags with docs-site/package.json found.');
}
if (rebuild !== 'all') {
const unknown = [...rebuild].filter((tag) => !stableVersions.includes(tag));
if (unknown.length > 0) {
throw new Error(`Cannot rebuild unknown stable docs versions: ${unknown.join(', ')}`);
}
}
const manifest = buildVersionManifest({ latestStable, stableVersions });
const manifestJson = JSON.stringify(manifest);
const sharedInternalsHash = computeSharedInternalsHash();
const archiveCacheKey = versionArchiveCacheKey({ sharedInternalsHash, manifestJson });
const sharedAssetPaths = collectSharedAssetPaths(join(currentDocsSite, 'public/assets'));
console.info(`[docs] archive cache key ${archiveCacheKey.slice(0, 12)}`);
rmSync(buildRoot, { recursive: true, force: true });
rmSync(aggregateOutDir, { recursive: true, force: true });
mkdirSync(buildRoot, { recursive: true });
mkdirSync(aggregateOutDir, { recursive: true });
const store = createArchiveStore(archiveStoreEnvFromProcess(process.env));
if (store) {
syncArchives({ store, stableVersions, rebuild });
} else if (requireArchives) {
throw new Error(
'Docs archive R2 credentials are missing (CLOUDFLARE_ACCOUNT_ID, DOCS_ARCHIVE_R2_ACCESS_KEY_ID, DOCS_ARCHIVE_R2_SECRET_ACCESS_KEY, DOCS_ARCHIVE_R2_BUCKET).',
);
} else {
console.warn('[docs] R2 credentials not set; skipping /v/<version>/ archive sync');
}
const latestStableSnapshot = prepareSnapshot(latestStable, latestStable);
writeFileSync(join(latestStableSnapshot, 'versions.md'), renderVersionsPage(manifest));
buildDocs({
snapshotDocsSite: latestStableSnapshot,
base: '/',
outDir: aggregateOutDir,
channel: 'stable-root',
version: latestStable,
latestStable,
manifestJson,
});
for (const version of stableVersions) {
if (restoreCachedArchive(version, archiveCacheKey)) {
continue;
}
console.info(`[docs] rebuilding archive ${version}`);
const snapshot =
version === latestStable ? latestStableSnapshot : prepareSnapshot(version, version);
buildDocs({
snapshotDocsSite: snapshot,
base: versionPath(version),
outDir: join(aggregateOutDir, versionOutputPath(version)),
channel: 'stable-archive',
version,
latestStable,
manifestJson,
});
dedupeVersionedPublicAssets({
outDir: join(aggregateOutDir, versionOutputPath(version)),
base: versionPath(version),
sharedAssetPaths,
});
saveArchiveCache(version, archiveCacheKey);
}
const mainSnapshot = prepareSnapshot('main');
buildDocs({
snapshotDocsSite: mainSnapshot,
@@ -380,8 +342,6 @@ function main() {
outDir: join(aggregateOutDir, 'main'),
channel: 'main',
version: 'main',
latestStable,
manifestJson,
});
dedupeVersionedPublicAssets({
outDir: join(aggregateOutDir, 'main'),
@@ -392,13 +352,6 @@ function main() {
writeFileSync(join(aggregateOutDir, 'versions.json'), `${JSON.stringify(manifest, null, 2)}\n`);
writeFileSync(join(aggregateOutDir, '_headers'), deployHeaders);
assertCloudflarePagesLimits(aggregateOutDir);
const prunedArchives = pruneArchiveCacheGenerations({
cacheRoot: archiveCacheRoot,
activeCacheKey: archiveCacheKey,
});
if (prunedArchives.length > 0) {
console.info(`[docs] pruned ${prunedArchives.length} stale archive cache directories`);
}
rmSync(buildRoot, { recursive: true, force: true });
}