mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-18 12:18:29 -07:00
fix(dictionary): prevent freezes and restore AppImage notifications (#205)
This commit is contained in:
@@ -0,0 +1,5 @@
|
|||||||
|
type: fixed
|
||||||
|
area: dictionary
|
||||||
|
|
||||||
|
- Character dictionary generation, merged rebuilds, and imports no longer freeze the app (and trigger the compositor's "application not responding" dialog) on large dictionaries; snapshot reads/writes, archive building, and the character image/name lookup caches now do their heavy work off the UI's critical path.
|
||||||
|
- Desktop progress notifications now update in place on Linux AppImage installs too: the AppImage's bundled libraries broke the system notify-send helper, which silently forced the flickering close-and-reopen notification fallback.
|
||||||
@@ -1,6 +1,22 @@
|
|||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
import { createNotifySendReplacer, resolveDefaultNotificationIconPath } from './notification';
|
import {
|
||||||
|
buildNotifySendEnv,
|
||||||
|
createNotifySendReplacer,
|
||||||
|
resolveDefaultNotificationIconPath,
|
||||||
|
} from './notification';
|
||||||
|
|
||||||
|
test('notify-send child environment drops the AppImage library-path override', () => {
|
||||||
|
const env = buildNotifySendEnv({
|
||||||
|
LD_LIBRARY_PATH: '/tmp/.mount_SubMinXXXXXX/usr/lib',
|
||||||
|
DBUS_SESSION_BUS_ADDRESS: 'unix:path=/run/user/1000/bus',
|
||||||
|
HOME: '/home/user',
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(env.LD_LIBRARY_PATH, undefined);
|
||||||
|
assert.equal(env.DBUS_SESSION_BUS_ADDRESS, 'unix:path=/run/user/1000/bus');
|
||||||
|
assert.equal(env.HOME, '/home/user');
|
||||||
|
});
|
||||||
|
|
||||||
test('default notification icon resolves packaged SubMiner asset when no per-notification icon is provided', () => {
|
test('default notification icon resolves packaged SubMiner asset when no per-notification icon is provided', () => {
|
||||||
const path = resolveDefaultNotificationIconPath({
|
const path = resolveDefaultNotificationIconPath({
|
||||||
|
|||||||
@@ -203,8 +203,19 @@ export function createNotifySendReplacer(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Electron AppImages export `LD_LIBRARY_PATH=<mount>/usr/lib`, whose bundled libnotify predates the
|
||||||
|
* symbols the system notify-send links against, so an inherited environment kills the child with a
|
||||||
|
* symbol lookup error before it can send anything. A system binary resolves its own libraries fine,
|
||||||
|
* so the override is dropped entirely rather than filtered.
|
||||||
|
*/
|
||||||
|
export function buildNotifySendEnv(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
|
||||||
|
const { LD_LIBRARY_PATH: _dropped, ...rest } = env;
|
||||||
|
return rest;
|
||||||
|
}
|
||||||
|
|
||||||
const showLinuxReplaceableNotification = createNotifySendReplacer((args, callback) =>
|
const showLinuxReplaceableNotification = createNotifySendReplacer((args, callback) =>
|
||||||
execFile('notify-send', args, { timeout: 5_000 }, (error, stdout) =>
|
execFile('notify-send', args, { timeout: 5_000, env: buildNotifySendEnv() }, (error, stdout) =>
|
||||||
callback(error, stdout ?? ''),
|
callback(error, stdout ?? ''),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -244,13 +244,13 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
const findCachedSnapshotForSeriesKey = (
|
const findCachedSnapshotForSeriesKey = async (
|
||||||
seriesKey: string,
|
seriesKey: string,
|
||||||
fallbackSeriesKey?: string,
|
fallbackSeriesKey?: string,
|
||||||
): CharacterDictionarySnapshot | null => {
|
): Promise<CharacterDictionarySnapshot | null> => {
|
||||||
const acceptedKeys = new Set([seriesKey, fallbackSeriesKey].filter(Boolean));
|
const acceptedKeys = new Set([seriesKey, fallbackSeriesKey].filter(Boolean));
|
||||||
return (
|
return (
|
||||||
readCachedSnapshots(outputDir).find((snapshot) => {
|
(await readCachedSnapshots(outputDir)).find((snapshot) => {
|
||||||
const snapshotSeriesKey = buildCharacterDictionarySeriesKey({
|
const snapshotSeriesKey = buildCharacterDictionarySeriesKey({
|
||||||
mediaPath: null,
|
mediaPath: null,
|
||||||
mediaTitle: snapshot.mediaTitle,
|
mediaTitle: snapshot.mediaTitle,
|
||||||
@@ -293,7 +293,9 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
|
|
||||||
const cachedResolution = readCachedMediaResolution(outputDir, seriesKey);
|
const cachedResolution = readCachedMediaResolution(outputDir, seriesKey);
|
||||||
if (cachedResolution) {
|
if (cachedResolution) {
|
||||||
const cachedSnapshot = readSnapshot(getSnapshotPath(outputDir, cachedResolution.mediaId));
|
const cachedSnapshot = await readSnapshot(
|
||||||
|
getSnapshotPath(outputDir, cachedResolution.mediaId),
|
||||||
|
);
|
||||||
if (cachedSnapshot) {
|
if (cachedSnapshot) {
|
||||||
deps.logInfo?.(
|
deps.logInfo?.(
|
||||||
`[dictionary] cached AniList match: ${cachedSnapshot.mediaTitle} -> AniList ${cachedSnapshot.mediaId}`,
|
`[dictionary] cached AniList match: ${cachedSnapshot.mediaTitle} -> AniList ${cachedSnapshot.mediaId}`,
|
||||||
@@ -305,7 +307,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const cachedSnapshot = findCachedSnapshotForSeriesKey(seriesKey, unscopedSeriesKey);
|
const cachedSnapshot = await findCachedSnapshotForSeriesKey(seriesKey, unscopedSeriesKey);
|
||||||
if (cachedSnapshot) {
|
if (cachedSnapshot) {
|
||||||
writeCachedMediaResolution(outputDir, {
|
writeCachedMediaResolution(outputDir, {
|
||||||
seriesKey,
|
seriesKey,
|
||||||
@@ -348,7 +350,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
progress?: CharacterDictionarySnapshotProgressCallbacks,
|
progress?: CharacterDictionarySnapshotProgressCallbacks,
|
||||||
): Promise<CharacterDictionarySnapshotResult> => {
|
): Promise<CharacterDictionarySnapshotResult> => {
|
||||||
const snapshotPath = getSnapshotPath(outputDir, mediaId);
|
const snapshotPath = getSnapshotPath(outputDir, mediaId);
|
||||||
const cachedSnapshot = readSnapshot(snapshotPath);
|
const cachedSnapshot = await readSnapshot(snapshotPath);
|
||||||
const refreshReason = cachedSnapshot ? getCachedSnapshotRefreshReason(cachedSnapshot) : null;
|
const refreshReason = cachedSnapshot ? getCachedSnapshotRefreshReason(cachedSnapshot) : null;
|
||||||
if (cachedSnapshot && refreshReason === null) {
|
if (cachedSnapshot && refreshReason === null) {
|
||||||
deps.logInfo?.(`[dictionary] snapshot hit for AniList ${mediaId}`);
|
deps.logInfo?.(`[dictionary] snapshot hit for AniList ${mediaId}`);
|
||||||
@@ -485,7 +487,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
resolvedNameSplits,
|
resolvedNameSplits,
|
||||||
nameSplitSource,
|
nameSplitSource,
|
||||||
);
|
);
|
||||||
writeSnapshot(snapshotPath, snapshot);
|
await writeSnapshot(snapshotPath, snapshot);
|
||||||
deps.logInfo?.(
|
deps.logInfo?.(
|
||||||
`[dictionary] stored snapshot for AniList ${mediaId}: ${snapshot.entryCount} terms`,
|
`[dictionary] stored snapshot for AniList ${mediaId}: ${snapshot.entryCount} terms`,
|
||||||
);
|
);
|
||||||
@@ -526,19 +528,22 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
const snapshotResults = await Promise.all(
|
const snapshotResults = await Promise.all(
|
||||||
normalizedMediaIds.map((mediaId) => getOrCreateSnapshot(mediaId)),
|
normalizedMediaIds.map((mediaId) => getOrCreateSnapshot(mediaId)),
|
||||||
);
|
);
|
||||||
const snapshots = snapshotResults.map(({ mediaId }) => {
|
// Sequential on purpose: each snapshot parse is a chunk of main-thread work, so reading them
|
||||||
const snapshot = readSnapshot(getSnapshotPath(outputDir, mediaId));
|
// one at a time keeps the event loop breathing between files.
|
||||||
|
const snapshots: CharacterDictionarySnapshot[] = [];
|
||||||
|
for (const { mediaId } of snapshotResults) {
|
||||||
|
const snapshot = await readSnapshot(getSnapshotPath(outputDir, mediaId));
|
||||||
if (!snapshot) {
|
if (!snapshot) {
|
||||||
throw new Error(`Missing character dictionary snapshot for AniList ${mediaId}.`);
|
throw new Error(`Missing character dictionary snapshot for AniList ${mediaId}.`);
|
||||||
}
|
}
|
||||||
return snapshot;
|
snapshots.push(snapshot);
|
||||||
});
|
}
|
||||||
const revision = buildMergedRevision(normalizedMediaIds, snapshots);
|
const revision = buildMergedRevision(normalizedMediaIds, snapshots);
|
||||||
const description =
|
const description =
|
||||||
snapshots.length === 1
|
snapshots.length === 1
|
||||||
? `Character names from ${snapshots[0]!.mediaTitle}`
|
? `Character names from ${snapshots[0]!.mediaTitle}`
|
||||||
: `Character names from ${snapshots.length} recent anime`;
|
: `Character names from ${snapshots.length} recent anime`;
|
||||||
const { zipPath, entryCount } = buildDictionaryZip(
|
const { zipPath, entryCount } = await buildDictionaryZip(
|
||||||
getMergedZipPath(outputDir),
|
getMergedZipPath(outputDir),
|
||||||
CHARACTER_DICTIONARY_MERGED_TITLE,
|
CHARACTER_DICTIONARY_MERGED_TITLE,
|
||||||
description,
|
description,
|
||||||
@@ -633,7 +638,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
resolvedMedia.title,
|
resolvedMedia.title,
|
||||||
waitForAniListRequestSlot,
|
waitForAniListRequestSlot,
|
||||||
);
|
);
|
||||||
const storedSnapshot = readSnapshot(getSnapshotPath(outputDir, resolvedMedia.id));
|
const storedSnapshot = await readSnapshot(getSnapshotPath(outputDir, resolvedMedia.id));
|
||||||
if (!storedSnapshot) {
|
if (!storedSnapshot) {
|
||||||
throw new Error(`Snapshot missing after generation for AniList ${resolvedMedia.id}.`);
|
throw new Error(`Snapshot missing after generation for AniList ${resolvedMedia.id}.`);
|
||||||
}
|
}
|
||||||
@@ -642,7 +647,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
const description = `Character names from ${storedSnapshot.mediaTitle} [AniList media ID ${resolvedMedia.id}]`;
|
const description = `Character names from ${storedSnapshot.mediaTitle} [AniList media ID ${resolvedMedia.id}]`;
|
||||||
const zipPath = path.join(outputDir, `anilist-${resolvedMedia.id}.zip`);
|
const zipPath = path.join(outputDir, `anilist-${resolvedMedia.id}.zip`);
|
||||||
deps.logInfo?.(`[dictionary] building ZIP for AniList ${resolvedMedia.id}`);
|
deps.logInfo?.(`[dictionary] building ZIP for AniList ${resolvedMedia.id}`);
|
||||||
buildDictionaryZip(
|
await buildDictionaryZip(
|
||||||
zipPath,
|
zipPath,
|
||||||
dictionaryTitle,
|
dictionaryTitle,
|
||||||
description,
|
description,
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import * as fs from 'fs';
|
|||||||
import * as os from 'os';
|
import * as os from 'os';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
|
import { isDeepStrictEqual } from 'node:util';
|
||||||
import { getSnapshotPath, readSnapshot, writeSnapshot } from './cache';
|
import { getSnapshotPath, readSnapshot, writeSnapshot } from './cache';
|
||||||
import { CHARACTER_DICTIONARY_FORMAT_VERSION } from './constants';
|
import { CHARACTER_DICTIONARY_FORMAT_VERSION } from './constants';
|
||||||
import type { CharacterDictionarySnapshot } from './types';
|
import type { CharacterDictionarySnapshot } from './types';
|
||||||
@@ -29,17 +30,72 @@ function createSnapshot(): CharacterDictionarySnapshot {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
test('writeSnapshot persists and readSnapshot restores current-format snapshots', () => {
|
test('writeSnapshot persists and readSnapshot restores current-format snapshots', async () => {
|
||||||
const outputDir = makeTempDir();
|
const outputDir = makeTempDir();
|
||||||
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
||||||
const snapshot = createSnapshot();
|
const snapshot = createSnapshot();
|
||||||
|
|
||||||
writeSnapshot(snapshotPath, snapshot);
|
await writeSnapshot(snapshotPath, snapshot);
|
||||||
|
|
||||||
assert.deepEqual(readSnapshot(snapshotPath), { ...snapshot, nameSplitSource: 'heuristic' });
|
assert.deepEqual(await readSnapshot(snapshotPath), { ...snapshot, nameSplitSource: 'heuristic' });
|
||||||
});
|
});
|
||||||
|
|
||||||
test('readSnapshot preserves the mecab name-split source and defaults missing values to heuristic', () => {
|
// A manual generate and an auto-sync can both land on the same media, so two writes for one
|
||||||
|
// snapshot can overlap. They must not stream into a shared temp file and interleave into a
|
||||||
|
// half-and-half snapshot.
|
||||||
|
test('concurrent writeSnapshot calls for the same media leave one complete snapshot', async () => {
|
||||||
|
const outputDir = makeTempDir();
|
||||||
|
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
||||||
|
const base = createSnapshot();
|
||||||
|
// Distinct titles, lengths, and term text so the surviving file can be pinned to exactly one
|
||||||
|
// writer rather than merely "a snapshot that parses". A shared temp file is caught by the
|
||||||
|
// losing writers failing to rename; interleaved content is only caught when the timing happens
|
||||||
|
// to leave a mix, which is why the assertion checks identity rather than shape.
|
||||||
|
const variants: CharacterDictionarySnapshot[] = ['alpha', 'beta', 'gamma'].map((label, index) => {
|
||||||
|
const entryCount = 400 + index * 100;
|
||||||
|
return {
|
||||||
|
...base,
|
||||||
|
mediaTitle: `${base.mediaTitle} ${label}`,
|
||||||
|
entryCount,
|
||||||
|
termEntries: Array.from({ length: entryCount }, (_entry, entryIndex) => [
|
||||||
|
`${label}${entryIndex}`,
|
||||||
|
'なまえ',
|
||||||
|
'name primary',
|
||||||
|
'',
|
||||||
|
75,
|
||||||
|
[`${label} character ${entryIndex} `.repeat(600)],
|
||||||
|
0,
|
||||||
|
'',
|
||||||
|
]) as CharacterDictionarySnapshot['termEntries'],
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
await Promise.all(variants.map((variant) => writeSnapshot(snapshotPath, variant)));
|
||||||
|
|
||||||
|
const restored = await readSnapshot(snapshotPath);
|
||||||
|
const expected = variants.map((variant) => ({
|
||||||
|
...variant,
|
||||||
|
nameSplitSource: 'heuristic' as const,
|
||||||
|
}));
|
||||||
|
const matches = expected.filter((candidate) => isDeepStrictEqual(restored, candidate));
|
||||||
|
assert.equal(
|
||||||
|
matches.length,
|
||||||
|
1,
|
||||||
|
`expected exactly one writer's complete snapshot to survive, got ${
|
||||||
|
restored === null
|
||||||
|
? 'an unreadable file'
|
||||||
|
: `entryCount=${restored.entryCount}, terms=${restored.termEntries.length}, title=${restored.mediaTitle}`
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Every writer cleaned up after itself, so no temp files are left behind.
|
||||||
|
const leftovers = fs
|
||||||
|
.readdirSync(path.dirname(snapshotPath))
|
||||||
|
.filter((name) => name.includes('.tmp-'));
|
||||||
|
assert.deepEqual(leftovers, []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('readSnapshot preserves the mecab name-split source and defaults missing values to heuristic', async () => {
|
||||||
const outputDir = makeTempDir();
|
const outputDir = makeTempDir();
|
||||||
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
||||||
const snapshot: CharacterDictionarySnapshot = {
|
const snapshot: CharacterDictionarySnapshot = {
|
||||||
@@ -47,12 +103,12 @@ test('readSnapshot preserves the mecab name-split source and defaults missing va
|
|||||||
nameSplitSource: 'mecab',
|
nameSplitSource: 'mecab',
|
||||||
};
|
};
|
||||||
|
|
||||||
writeSnapshot(snapshotPath, snapshot);
|
await writeSnapshot(snapshotPath, snapshot);
|
||||||
|
|
||||||
assert.equal(readSnapshot(snapshotPath)?.nameSplitSource, 'mecab');
|
assert.equal((await readSnapshot(snapshotPath))?.nameSplitSource, 'mecab');
|
||||||
});
|
});
|
||||||
|
|
||||||
test('readSnapshot ignores snapshots written with an older format version', () => {
|
test('readSnapshot ignores snapshots written with an older format version', async () => {
|
||||||
const outputDir = makeTempDir();
|
const outputDir = makeTempDir();
|
||||||
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
||||||
const staleSnapshot = {
|
const staleSnapshot = {
|
||||||
@@ -63,10 +119,10 @@ test('readSnapshot ignores snapshots written with an older format version', () =
|
|||||||
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
|
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
|
||||||
fs.writeFileSync(snapshotPath, JSON.stringify(staleSnapshot), 'utf8');
|
fs.writeFileSync(snapshotPath, JSON.stringify(staleSnapshot), 'utf8');
|
||||||
|
|
||||||
assert.equal(readSnapshot(snapshotPath), null);
|
assert.equal(await readSnapshot(snapshotPath), null);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('readSnapshot ignores v15 snapshots with stale romanized character-name entries', () => {
|
test('readSnapshot ignores v15 snapshots with stale romanized character-name entries', async () => {
|
||||||
const outputDir = makeTempDir();
|
const outputDir = makeTempDir();
|
||||||
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
||||||
const staleSnapshot = {
|
const staleSnapshot = {
|
||||||
@@ -78,5 +134,5 @@ test('readSnapshot ignores v15 snapshots with stale romanized character-name ent
|
|||||||
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
|
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
|
||||||
fs.writeFileSync(snapshotPath, JSON.stringify(staleSnapshot), 'utf8');
|
fs.writeFileSync(snapshotPath, JSON.stringify(staleSnapshot), 'utf8');
|
||||||
|
|
||||||
assert.equal(readSnapshot(snapshotPath), null);
|
assert.equal(await readSnapshot(snapshotPath), null);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -102,24 +102,42 @@ export function writeCachedMediaResolution(
|
|||||||
writeMediaResolutionEntries(outputDir, [...remaining, normalized]);
|
writeMediaResolutionEntries(outputDir, [...remaining, normalized]);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function readCachedSnapshots(outputDir: string): CharacterDictionarySnapshot[] {
|
/**
|
||||||
|
* Snapshots for long series run to hundreds of MB each, so everything here reads them off the main
|
||||||
|
* thread's critical path: file IO is async and only the unavoidable JSON.parse runs on the loop,
|
||||||
|
* one file at a time. Reading the whole directory synchronously used to block the process for
|
||||||
|
* multiple seconds, long enough for the compositor to declare the app unresponsive mid-playback.
|
||||||
|
*/
|
||||||
|
export async function readCachedSnapshots(
|
||||||
|
outputDir: string,
|
||||||
|
): Promise<CharacterDictionarySnapshot[]> {
|
||||||
let entries: fs.Dirent[] = [];
|
let entries: fs.Dirent[] = [];
|
||||||
try {
|
try {
|
||||||
entries = fs.readdirSync(getSnapshotsDir(outputDir), { withFileTypes: true });
|
entries = await fs.promises.readdir(getSnapshotsDir(outputDir), { withFileTypes: true });
|
||||||
} catch {
|
} catch {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
return entries
|
const names = entries
|
||||||
.filter((entry) => entry.isFile() && /^anilist-\d+\.json$/.test(entry.name))
|
.filter((entry) => entry.isFile() && /^anilist-\d+\.json$/.test(entry.name))
|
||||||
.sort((left, right) => left.name.localeCompare(right.name))
|
.map((entry) => entry.name)
|
||||||
.map((entry) => readSnapshot(path.join(getSnapshotsDir(outputDir), entry.name)))
|
.sort((left, right) => left.localeCompare(right));
|
||||||
.filter((snapshot): snapshot is CharacterDictionarySnapshot => snapshot !== null);
|
|
||||||
|
const snapshots: CharacterDictionarySnapshot[] = [];
|
||||||
|
for (const name of names) {
|
||||||
|
const snapshot = await readSnapshot(path.join(getSnapshotsDir(outputDir), name));
|
||||||
|
if (snapshot) {
|
||||||
|
snapshots.push(snapshot);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return snapshots;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot | null {
|
export async function readSnapshot(
|
||||||
|
snapshotPath: string,
|
||||||
|
): Promise<CharacterDictionarySnapshot | null> {
|
||||||
try {
|
try {
|
||||||
const raw = fs.readFileSync(snapshotPath, 'utf8');
|
const raw = await fs.promises.readFile(snapshotPath, 'utf8');
|
||||||
const parsed = JSON.parse(raw) as Partial<CharacterDictionarySnapshot>;
|
const parsed = JSON.parse(raw) as Partial<CharacterDictionarySnapshot>;
|
||||||
if (!parsed || typeof parsed !== 'object') {
|
if (!parsed || typeof parsed !== 'object') {
|
||||||
return null;
|
return null;
|
||||||
@@ -150,9 +168,64 @@ export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function writeSnapshot(snapshotPath: string, snapshot: CharacterDictionarySnapshot): void {
|
// Flushing in a few-MB batches keeps each stringify-and-write slice short; a single
|
||||||
|
// JSON.stringify of a large snapshot blocks the event loop for seconds.
|
||||||
|
const SNAPSHOT_WRITE_FLUSH_BYTES = 4 * 1024 * 1024;
|
||||||
|
|
||||||
|
// Distinguishes concurrent writes of the same snapshot within one process; the pid alone only
|
||||||
|
// separates processes, so two overlapping writers would otherwise stream into the same temp file.
|
||||||
|
let snapshotWriteSequence = 0;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Streams the snapshot to disk piece by piece instead of stringifying it in one shot, then renames
|
||||||
|
* the finished file into place so a crash mid-write (or two concurrent writers for the same media)
|
||||||
|
* can never leave a torn file where a snapshot used to be.
|
||||||
|
*/
|
||||||
|
export async function writeSnapshot(
|
||||||
|
snapshotPath: string,
|
||||||
|
snapshot: CharacterDictionarySnapshot,
|
||||||
|
): Promise<void> {
|
||||||
ensureDir(path.dirname(snapshotPath));
|
ensureDir(path.dirname(snapshotPath));
|
||||||
fs.writeFileSync(snapshotPath, JSON.stringify(snapshot, null, 2), 'utf8');
|
snapshotWriteSequence += 1;
|
||||||
|
const tempPath = `${snapshotPath}.tmp-${process.pid}-${snapshotWriteSequence}`;
|
||||||
|
const handle = await fs.promises.open(tempPath, 'w');
|
||||||
|
try {
|
||||||
|
let buffered: string[] = [];
|
||||||
|
let bufferedBytes = 0;
|
||||||
|
const push = async (chunk: string): Promise<void> => {
|
||||||
|
buffered.push(chunk);
|
||||||
|
bufferedBytes += chunk.length;
|
||||||
|
if (bufferedBytes >= SNAPSHOT_WRITE_FLUSH_BYTES) {
|
||||||
|
const joined = buffered.join('');
|
||||||
|
buffered = [];
|
||||||
|
bufferedBytes = 0;
|
||||||
|
await handle.write(joined, null, 'utf8');
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const writeArray = async (key: string, items: readonly unknown[]): Promise<void> => {
|
||||||
|
await push(`,${JSON.stringify(key)}:[`);
|
||||||
|
for (let i = 0; i < items.length; i += 1) {
|
||||||
|
await push(`${i > 0 ? ',' : ''}${JSON.stringify(items[i])}`);
|
||||||
|
}
|
||||||
|
await push(']');
|
||||||
|
};
|
||||||
|
|
||||||
|
const { termEntries, images, ...scalars } = snapshot;
|
||||||
|
const head = JSON.stringify(scalars);
|
||||||
|
await push(head.slice(0, -1));
|
||||||
|
await writeArray('termEntries', termEntries);
|
||||||
|
await writeArray('images', images);
|
||||||
|
await push('}');
|
||||||
|
if (buffered.length > 0) {
|
||||||
|
await handle.write(buffered.join(''), null, 'utf8');
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
await handle.close();
|
||||||
|
await fs.promises.rm(tempPath, { force: true });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
await handle.close();
|
||||||
|
await fs.promises.rename(tempPath, snapshotPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildMergedRevision(
|
export function buildMergedRevision(
|
||||||
|
|||||||
@@ -11,6 +11,22 @@ import {
|
|||||||
} from './image-lookup';
|
} from './image-lookup';
|
||||||
import type { CharacterDictionarySnapshot } from './types';
|
import type { CharacterDictionarySnapshot } from './types';
|
||||||
|
|
||||||
|
// Lookup indexes rebuild in the background while gets serve stale data, so tests poll until the
|
||||||
|
// refresh they triggered has landed.
|
||||||
|
async function waitForRefresh<T>(probe: () => T | null | undefined): Promise<T> {
|
||||||
|
const deadline = Date.now() + 5000;
|
||||||
|
for (;;) {
|
||||||
|
const value = probe();
|
||||||
|
if (value !== null && value !== undefined) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
if (Date.now() > deadline) {
|
||||||
|
throw new Error('timed out waiting for background snapshot refresh');
|
||||||
|
}
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const PNG_1X1_BASE64 =
|
const PNG_1X1_BASE64 =
|
||||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+nmX8AAAAASUVORK5CYII=';
|
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+nmX8AAAAASUVORK5CYII=';
|
||||||
|
|
||||||
@@ -18,7 +34,7 @@ function makeTempDir(): string {
|
|||||||
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-character-image-lookup-'));
|
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-character-image-lookup-'));
|
||||||
}
|
}
|
||||||
|
|
||||||
test('buildCharacterNameImageIndexFromSnapshots maps name terms to character portrait data URLs', () => {
|
test('buildCharacterNameImageIndexFromSnapshots maps name terms to character portrait data URLs', async () => {
|
||||||
const outputDir = makeTempDir();
|
const outputDir = makeTempDir();
|
||||||
const snapshot: CharacterDictionarySnapshot = {
|
const snapshot: CharacterDictionarySnapshot = {
|
||||||
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
|
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
|
||||||
@@ -75,9 +91,9 @@ test('buildCharacterNameImageIndexFromSnapshots maps name terms to character por
|
|||||||
{ path: 'img/m130298-va456.png', dataBase64: 'BBBB' },
|
{ path: 'img/m130298-va456.png', dataBase64: 'BBBB' },
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
|
await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
|
||||||
|
|
||||||
const index = buildCharacterNameImageIndexFromSnapshots(outputDir);
|
const index = await buildCharacterNameImageIndexFromSnapshots(outputDir);
|
||||||
|
|
||||||
assert.deepEqual(index.get('アレクシア'), {
|
assert.deepEqual(index.get('アレクシア'), {
|
||||||
src: 'data:image/png;base64,AAAA',
|
src: 'data:image/png;base64,AAAA',
|
||||||
@@ -85,7 +101,7 @@ test('buildCharacterNameImageIndexFromSnapshots maps name terms to character por
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test('buildCharacterNameImageIndexFromSnapshots sniffs image MIME from bytes before path extension', () => {
|
test('buildCharacterNameImageIndexFromSnapshots sniffs image MIME from bytes before path extension', async () => {
|
||||||
const outputDir = makeTempDir();
|
const outputDir = makeTempDir();
|
||||||
const snapshot: CharacterDictionarySnapshot = {
|
const snapshot: CharacterDictionarySnapshot = {
|
||||||
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
|
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
|
||||||
@@ -116,14 +132,14 @@ test('buildCharacterNameImageIndexFromSnapshots sniffs image MIME from bytes bef
|
|||||||
],
|
],
|
||||||
images: [{ path: 'img/m130298-c123.jpg', dataBase64: PNG_1X1_BASE64 }],
|
images: [{ path: 'img/m130298-c123.jpg', dataBase64: PNG_1X1_BASE64 }],
|
||||||
};
|
};
|
||||||
writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
|
await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
|
||||||
|
|
||||||
const index = buildCharacterNameImageIndexFromSnapshots(outputDir);
|
const index = await buildCharacterNameImageIndexFromSnapshots(outputDir);
|
||||||
|
|
||||||
assert.equal(index.get('アレクシア')?.src, `data:image/png;base64,${PNG_1X1_BASE64}`);
|
assert.equal(index.get('アレクシア')?.src, `data:image/png;base64,${PNG_1X1_BASE64}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('createCharacterDictionaryImageLookup can scope duplicate names to the current media', () => {
|
test('createCharacterDictionaryImageLookup can scope duplicate names to the current media', async () => {
|
||||||
const outputDir = makeTempDir();
|
const outputDir = makeTempDir();
|
||||||
const towerSnapshot: CharacterDictionarySnapshot = {
|
const towerSnapshot: CharacterDictionarySnapshot = {
|
||||||
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
|
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
|
||||||
@@ -173,15 +189,16 @@ test('createCharacterDictionaryImageLookup can scope duplicate names to the curr
|
|||||||
],
|
],
|
||||||
images: [{ path: 'img/m21202-c2.png', dataBase64: 'KONOSUBA' }],
|
images: [{ path: 'img/m21202-c2.png', dataBase64: 'KONOSUBA' }],
|
||||||
};
|
};
|
||||||
writeSnapshot(getSnapshotPath(outputDir, towerSnapshot.mediaId), towerSnapshot);
|
await writeSnapshot(getSnapshotPath(outputDir, towerSnapshot.mediaId), towerSnapshot);
|
||||||
writeSnapshot(getSnapshotPath(outputDir, konosubaSnapshot.mediaId), konosubaSnapshot);
|
await writeSnapshot(getSnapshotPath(outputDir, konosubaSnapshot.mediaId), konosubaSnapshot);
|
||||||
|
|
||||||
const lookup = createCharacterDictionaryImageLookup({ outputDir });
|
const lookup = createCharacterDictionaryImageLookup({ outputDir });
|
||||||
|
|
||||||
assert.equal(lookup.get('カズ', 21202)?.alt, 'Kazuma');
|
const scoped = await waitForRefresh(() => lookup.get('カズ', 21202));
|
||||||
|
assert.equal(scoped.alt, 'Kazuma');
|
||||||
});
|
});
|
||||||
|
|
||||||
test('createCharacterDictionaryImageLookup does not fall back globally on scoped miss', () => {
|
test('createCharacterDictionaryImageLookup does not fall back globally on scoped miss', async () => {
|
||||||
const outputDir = makeTempDir();
|
const outputDir = makeTempDir();
|
||||||
const snapshot: CharacterDictionarySnapshot = {
|
const snapshot: CharacterDictionarySnapshot = {
|
||||||
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
|
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
|
||||||
@@ -208,10 +225,11 @@ test('createCharacterDictionaryImageLookup does not fall back globally on scoped
|
|||||||
],
|
],
|
||||||
images: [{ path: 'img/m115230-c1.png', dataBase64: 'TOWER' }],
|
images: [{ path: 'img/m115230-c1.png', dataBase64: 'TOWER' }],
|
||||||
};
|
};
|
||||||
writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
|
await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
|
||||||
|
|
||||||
const lookup = createCharacterDictionaryImageLookup({ outputDir });
|
const lookup = createCharacterDictionaryImageLookup({ outputDir });
|
||||||
|
|
||||||
|
const unscoped = await waitForRefresh(() => lookup.get('カズ'));
|
||||||
|
assert.equal(unscoped.alt, 'Kaz');
|
||||||
assert.equal(lookup.get('カズ', 21202), null);
|
assert.equal(lookup.get('カズ', 21202), null);
|
||||||
assert.equal(lookup.get('カズ')?.alt, 'Kaz');
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -204,11 +204,11 @@ function getSnapshotDirectorySignature(outputDir: string): string {
|
|||||||
return parts.sort().join('|');
|
return parts.sort().join('|');
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildCharacterNameImageIndexFromSnapshots(
|
export async function buildCharacterNameImageIndexFromSnapshots(
|
||||||
outputDir: string,
|
outputDir: string,
|
||||||
): Map<string, CharacterNameImage> {
|
): Promise<Map<string, CharacterNameImage>> {
|
||||||
const index = new Map<string, CharacterNameImage>();
|
const index = new Map<string, CharacterNameImage>();
|
||||||
for (const snapshot of readCachedSnapshots(outputDir)) {
|
for (const snapshot of await readCachedSnapshots(outputDir)) {
|
||||||
appendSnapshotImages(index, snapshot);
|
appendSnapshotImages(index, snapshot);
|
||||||
}
|
}
|
||||||
return index;
|
return index;
|
||||||
@@ -228,7 +228,12 @@ export function createCharacterDictionaryImageLookup(deps: {
|
|||||||
let signature: string | null = null;
|
let signature: string | null = null;
|
||||||
let index = new Map<string, CharacterNameImage>();
|
let index = new Map<string, CharacterNameImage>();
|
||||||
let indexByMediaId = new Map<number, Map<string, CharacterNameImage>>();
|
let indexByMediaId = new Map<number, Map<string, CharacterNameImage>>();
|
||||||
|
let refreshInFlight = false;
|
||||||
|
|
||||||
|
// Rebuilding means re-reading every cached snapshot (potentially GBs of JSON), which used to run
|
||||||
|
// synchronously inside a lookup and froze the whole app right after a snapshot changed. Lookups
|
||||||
|
// now serve the previous index while a single background rebuild catches up; the swap is atomic
|
||||||
|
// and the signature only advances once the rebuild it belongs to has landed.
|
||||||
function refreshIfNeeded(): void {
|
function refreshIfNeeded(): void {
|
||||||
if (!outputDir) {
|
if (!outputDir) {
|
||||||
index = new Map<string, CharacterNameImage>();
|
index = new Map<string, CharacterNameImage>();
|
||||||
@@ -237,20 +242,30 @@ export function createCharacterDictionaryImageLookup(deps: {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const nextSignature = getSnapshotDirectorySignature(outputDir);
|
const nextSignature = getSnapshotDirectorySignature(outputDir);
|
||||||
if (nextSignature === signature) {
|
if (nextSignature === signature || refreshInFlight) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
signature = nextSignature;
|
refreshInFlight = true;
|
||||||
index = new Map<string, CharacterNameImage>();
|
void (async () => {
|
||||||
indexByMediaId = new Map<number, Map<string, CharacterNameImage>>();
|
try {
|
||||||
for (const snapshot of readCachedSnapshots(outputDir)) {
|
const snapshots = await readCachedSnapshots(outputDir);
|
||||||
appendSnapshotImages(index, snapshot);
|
const nextIndex = new Map<string, CharacterNameImage>();
|
||||||
const mediaIndex = new Map<string, CharacterNameImage>();
|
const nextIndexByMediaId = new Map<number, Map<string, CharacterNameImage>>();
|
||||||
appendSnapshotImages(mediaIndex, snapshot);
|
for (const snapshot of snapshots) {
|
||||||
if (mediaIndex.size > 0) {
|
appendSnapshotImages(nextIndex, snapshot);
|
||||||
indexByMediaId.set(snapshot.mediaId, mediaIndex);
|
const mediaIndex = new Map<string, CharacterNameImage>();
|
||||||
|
appendSnapshotImages(mediaIndex, snapshot);
|
||||||
|
if (mediaIndex.size > 0) {
|
||||||
|
nextIndexByMediaId.set(snapshot.mediaId, mediaIndex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
index = nextIndex;
|
||||||
|
indexByMediaId = nextIndexByMediaId;
|
||||||
|
signature = nextSignature;
|
||||||
|
} finally {
|
||||||
|
refreshInFlight = false;
|
||||||
}
|
}
|
||||||
}
|
})();
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -32,17 +32,33 @@ function writeSnapshot(outputDir: string, mediaId: number, entries: Array<[strin
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function withTempDir<T>(run: (dir: string) => T): T {
|
async function withTempDir<T>(run: (dir: string) => Promise<T> | T): Promise<T> {
|
||||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-name-candidates-'));
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-name-candidates-'));
|
||||||
try {
|
try {
|
||||||
return run(dir);
|
return await run(dir);
|
||||||
} finally {
|
} finally {
|
||||||
fs.rmSync(dir, { recursive: true, force: true });
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
test('collects terms and readings for the current media', () => {
|
// The snapshot index rebuilds in the background while lookups serve stale data, so tests poll the
|
||||||
withTempDir((dir) => {
|
// probe until the refresh they triggered has landed.
|
||||||
|
async function waitForRefresh<T>(probe: () => T | null | undefined): Promise<T> {
|
||||||
|
const deadline = Date.now() + 5000;
|
||||||
|
for (;;) {
|
||||||
|
const value = probe();
|
||||||
|
if (value !== null && value !== undefined) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
if (Date.now() > deadline) {
|
||||||
|
throw new Error('timed out waiting for background snapshot refresh');
|
||||||
|
}
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
test('collects terms and readings for the current media', async () => {
|
||||||
|
await withTempDir(async (dir) => {
|
||||||
writeSnapshot(dir, 1, [
|
writeSnapshot(dir, 1, [
|
||||||
['ミナト', 'みなと'],
|
['ミナト', 'みなと'],
|
||||||
['湊', 'みなと'],
|
['湊', 'みなと'],
|
||||||
@@ -53,17 +69,16 @@ test('collects terms and readings for the current media', () => {
|
|||||||
outputDir: dir,
|
outputDir: dir,
|
||||||
getCurrentMediaId: () => 1,
|
getCurrentMediaId: () => 1,
|
||||||
});
|
});
|
||||||
const candidates = lookup.get();
|
const candidates = await waitForRefresh(() => lookup.get());
|
||||||
|
|
||||||
assert.ok(candidates);
|
|
||||||
assert.deepEqual([...candidates.forms].sort(), ['みなと', 'ミナト', '湊'].sort());
|
assert.deepEqual([...candidates.forms].sort(), ['みなと', 'ミナト', '湊'].sort());
|
||||||
// Deduplicated: both entries share the みなと reading.
|
// Deduplicated: both entries share the みなと reading.
|
||||||
assert.equal(candidates.forms.length, 3);
|
assert.equal(candidates.forms.length, 3);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test('returns null without a media scope so the scanner stays exhaustive', () => {
|
test('returns null without a media scope so the scanner stays exhaustive', async () => {
|
||||||
withTempDir((dir) => {
|
await withTempDir(async (dir) => {
|
||||||
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
||||||
|
|
||||||
const lookup = createCharacterNameCandidateLookup({
|
const lookup = createCharacterNameCandidateLookup({
|
||||||
@@ -71,12 +86,14 @@ test('returns null without a media scope so the scanner stays exhaustive', () =>
|
|||||||
getCurrentMediaId: () => null,
|
getCurrentMediaId: () => null,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// The explicitly-scoped probe proves the index has loaded before the unscoped case is judged.
|
||||||
|
await waitForRefresh(() => lookup.get(1));
|
||||||
assert.equal(lookup.get(), null);
|
assert.equal(lookup.get(), null);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test('returns null for a media with no cached snapshot', () => {
|
test('returns null for a media with no cached snapshot', async () => {
|
||||||
withTempDir((dir) => {
|
await withTempDir(async (dir) => {
|
||||||
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
||||||
|
|
||||||
const lookup = createCharacterNameCandidateLookup({
|
const lookup = createCharacterNameCandidateLookup({
|
||||||
@@ -84,29 +101,31 @@ test('returns null for a media with no cached snapshot', () => {
|
|||||||
getCurrentMediaId: () => 999,
|
getCurrentMediaId: () => 999,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await waitForRefresh(() => lookup.get(1));
|
||||||
assert.equal(lookup.get(), null);
|
assert.equal(lookup.get(), null);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test('key changes when the snapshot content changes', () => {
|
test('key changes when the snapshot content changes', async () => {
|
||||||
withTempDir((dir) => {
|
await withTempDir(async (dir) => {
|
||||||
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
||||||
const lookup = createCharacterNameCandidateLookup({
|
const lookup = createCharacterNameCandidateLookup({
|
||||||
outputDir: dir,
|
outputDir: dir,
|
||||||
getCurrentMediaId: () => 1,
|
getCurrentMediaId: () => 1,
|
||||||
});
|
});
|
||||||
const first = lookup.get();
|
const first = await waitForRefresh(() => lookup.get());
|
||||||
|
|
||||||
writeSnapshot(dir, 1, [
|
writeSnapshot(dir, 1, [
|
||||||
['ミナト', 'みなと'],
|
['ミナト', 'みなと'],
|
||||||
['アクア', 'あくあ'],
|
['アクア', 'あくあ'],
|
||||||
]);
|
]);
|
||||||
lookup.invalidate();
|
lookup.invalidate();
|
||||||
const second = lookup.get();
|
const second = await waitForRefresh(() => {
|
||||||
|
const candidates = lookup.get();
|
||||||
|
return candidates && candidates.forms.length === 4 ? candidates : null;
|
||||||
|
});
|
||||||
|
|
||||||
assert.ok(first && second);
|
|
||||||
assert.notEqual(first.key, second.key);
|
assert.notEqual(first.key, second.key);
|
||||||
assert.equal(second.forms.length, 4);
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -114,8 +133,8 @@ test('key changes when the snapshot content changes', () => {
|
|||||||
// directory every call. Asserted behaviorally: an unannounced on-disk change is
|
// directory every call. Asserted behaviorally: an unannounced on-disk change is
|
||||||
// invisible until the recheck interval elapses, which can only be true if the
|
// invisible until the recheck interval elapses, which can only be true if the
|
||||||
// filesystem is not consulted per lookup.
|
// filesystem is not consulted per lookup.
|
||||||
test('does not re-read the snapshot directory on every lookup', () => {
|
test('does not re-read the snapshot directory on every lookup', async () => {
|
||||||
withTempDir((dir) => {
|
await withTempDir(async (dir) => {
|
||||||
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
||||||
let nowMs = 1_000_000;
|
let nowMs = 1_000_000;
|
||||||
const lookup = createCharacterNameCandidateLookup({
|
const lookup = createCharacterNameCandidateLookup({
|
||||||
@@ -124,6 +143,7 @@ test('does not re-read the snapshot directory on every lookup', () => {
|
|||||||
now: () => nowMs,
|
now: () => nowMs,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await waitForRefresh(() => lookup.get());
|
||||||
assert.equal(lookup.get()?.forms.length, 2);
|
assert.equal(lookup.get()?.forms.length, 2);
|
||||||
|
|
||||||
writeSnapshot(dir, 1, [
|
writeSnapshot(dir, 1, [
|
||||||
@@ -135,12 +155,16 @@ test('does not re-read the snapshot directory on every lookup', () => {
|
|||||||
assert.equal(lookup.get()?.forms.length, 2, 'expected the cached list within the interval');
|
assert.equal(lookup.get()?.forms.length, 2, 'expected the cached list within the interval');
|
||||||
|
|
||||||
nowMs += 10_000;
|
nowMs += 10_000;
|
||||||
assert.equal(lookup.get()?.forms.length, 4, 'expected a refresh past the interval');
|
const refreshed = await waitForRefresh(() => {
|
||||||
|
const candidates = lookup.get();
|
||||||
|
return candidates && candidates.forms.length === 4 ? candidates : null;
|
||||||
|
});
|
||||||
|
assert.equal(refreshed.forms.length, 4, 'expected a refresh past the interval');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test('invalidate picks up a snapshot change immediately', () => {
|
test('invalidate picks up a snapshot change on the next refresh', async () => {
|
||||||
withTempDir((dir) => {
|
await withTempDir(async (dir) => {
|
||||||
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
||||||
let nowMs = 1_000_000;
|
let nowMs = 1_000_000;
|
||||||
const lookup = createCharacterNameCandidateLookup({
|
const lookup = createCharacterNameCandidateLookup({
|
||||||
@@ -149,6 +173,7 @@ test('invalidate picks up a snapshot change immediately', () => {
|
|||||||
now: () => nowMs,
|
now: () => nowMs,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await waitForRefresh(() => lookup.get());
|
||||||
assert.equal(lookup.get()?.forms.length, 2);
|
assert.equal(lookup.get()?.forms.length, 2);
|
||||||
|
|
||||||
writeSnapshot(dir, 1, [
|
writeSnapshot(dir, 1, [
|
||||||
@@ -158,6 +183,10 @@ test('invalidate picks up a snapshot change immediately', () => {
|
|||||||
nowMs += 1;
|
nowMs += 1;
|
||||||
lookup.invalidate();
|
lookup.invalidate();
|
||||||
|
|
||||||
assert.equal(lookup.get()?.forms.length, 4);
|
const refreshed = await waitForRefresh(() => {
|
||||||
|
const candidates = lookup.get();
|
||||||
|
return candidates && candidates.forms.length === 4 ? candidates : null;
|
||||||
|
});
|
||||||
|
assert.equal(refreshed.forms.length, 4);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -98,7 +98,12 @@ export function createCharacterNameCandidateLookup(deps: {
|
|||||||
let signature: string | null = null;
|
let signature: string | null = null;
|
||||||
let lastSignatureCheckAtMs = 0;
|
let lastSignatureCheckAtMs = 0;
|
||||||
let formsByMediaId = new Map<number, string[]>();
|
let formsByMediaId = new Map<number, string[]>();
|
||||||
|
let refreshInFlight = false;
|
||||||
|
|
||||||
|
// Same stale-while-revalidate shape as the image lookup: the rebuild re-reads every cached
|
||||||
|
// snapshot, so it runs in the background while lookups keep serving the previous forms. The
|
||||||
|
// signature only advances once its rebuild has landed, so a failed or superseded rebuild is
|
||||||
|
// retried on the next signature check.
|
||||||
function refreshIfNeeded(): void {
|
function refreshIfNeeded(): void {
|
||||||
if (!outputDir) {
|
if (!outputDir) {
|
||||||
formsByMediaId = new Map<number, string[]>();
|
formsByMediaId = new Map<number, string[]>();
|
||||||
@@ -114,17 +119,26 @@ export function createCharacterNameCandidateLookup(deps: {
|
|||||||
}
|
}
|
||||||
lastSignatureCheckAtMs = nowMs;
|
lastSignatureCheckAtMs = nowMs;
|
||||||
const nextSignature = getSnapshotDirectorySignature(outputDir);
|
const nextSignature = getSnapshotDirectorySignature(outputDir);
|
||||||
if (nextSignature === signature) {
|
if (nextSignature === signature || refreshInFlight) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
signature = nextSignature;
|
refreshInFlight = true;
|
||||||
formsByMediaId = new Map<number, string[]>();
|
void (async () => {
|
||||||
for (const snapshot of readCachedSnapshots(outputDir)) {
|
try {
|
||||||
const forms = collectSnapshotNameForms(snapshot);
|
const snapshots = await readCachedSnapshots(outputDir);
|
||||||
if (forms.length > 0) {
|
const nextFormsByMediaId = new Map<number, string[]>();
|
||||||
formsByMediaId.set(snapshot.mediaId, forms);
|
for (const snapshot of snapshots) {
|
||||||
|
const forms = collectSnapshotNameForms(snapshot);
|
||||||
|
if (forms.length > 0) {
|
||||||
|
nextFormsByMediaId.set(snapshot.mediaId, forms);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
formsByMediaId = nextFormsByMediaId;
|
||||||
|
signature = nextSignature;
|
||||||
|
} finally {
|
||||||
|
refreshInFlight = false;
|
||||||
}
|
}
|
||||||
}
|
})();
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ function createSnapshotWithoutImages(): CharacterDictionarySnapshot {
|
|||||||
test('generateForCurrentMedia refreshes same-version snapshots missing images when inline images are enabled', async () => {
|
test('generateForCurrentMedia refreshes same-version snapshots missing images when inline images are enabled', async () => {
|
||||||
const userDataPath = makeTempDir();
|
const userDataPath = makeTempDir();
|
||||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||||
writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
|
await writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
|
||||||
const originalFetch = globalThis.fetch;
|
const originalFetch = globalThis.fetch;
|
||||||
const fetchUrls: string[] = [];
|
const fetchUrls: string[] = [];
|
||||||
|
|
||||||
@@ -124,7 +124,7 @@ test('generateForCurrentMedia refreshes same-version snapshots missing images wh
|
|||||||
test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable', async () => {
|
test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable', async () => {
|
||||||
const userDataPath = makeTempDir();
|
const userDataPath = makeTempDir();
|
||||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||||
writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
await writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
||||||
...createSnapshotWithoutImages(),
|
...createSnapshotWithoutImages(),
|
||||||
nameSplitSource: 'heuristic',
|
nameSplitSource: 'heuristic',
|
||||||
});
|
});
|
||||||
@@ -213,7 +213,7 @@ test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable'
|
|||||||
test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is available', async () => {
|
test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is available', async () => {
|
||||||
const userDataPath = makeTempDir();
|
const userDataPath = makeTempDir();
|
||||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||||
writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
await writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
||||||
...createSnapshotWithoutImages(),
|
...createSnapshotWithoutImages(),
|
||||||
nameSplitSource: 'mecab',
|
nameSplitSource: 'mecab',
|
||||||
});
|
});
|
||||||
@@ -253,7 +253,7 @@ test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is availabl
|
|||||||
test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is unavailable', async () => {
|
test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is unavailable', async () => {
|
||||||
const userDataPath = makeTempDir();
|
const userDataPath = makeTempDir();
|
||||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||||
writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
await writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
||||||
...createSnapshotWithoutImages(),
|
...createSnapshotWithoutImages(),
|
||||||
nameSplitSource: 'heuristic',
|
nameSplitSource: 'heuristic',
|
||||||
});
|
});
|
||||||
@@ -293,7 +293,7 @@ test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is una
|
|||||||
test('generateForCurrentMedia keeps same-version snapshots without images when inline images are disabled', async () => {
|
test('generateForCurrentMedia keeps same-version snapshots without images when inline images are disabled', async () => {
|
||||||
const userDataPath = makeTempDir();
|
const userDataPath = makeTempDir();
|
||||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||||
writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
|
await writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
|
||||||
const originalFetch = globalThis.fetch;
|
const originalFetch = globalThis.fetch;
|
||||||
|
|
||||||
globalThis.fetch = (async (input: string | URL | Request) => {
|
globalThis.fetch = (async (input: string | URL | Request) => {
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ function readStoredZipEntries(zipPath: string): Map<string, Buffer> {
|
|||||||
return entries;
|
return entries;
|
||||||
}
|
}
|
||||||
|
|
||||||
test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', () => {
|
test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', async () => {
|
||||||
const tempDir = makeTempDir();
|
const tempDir = makeTempDir();
|
||||||
const outputPath = path.join(tempDir, 'dictionary.zip');
|
const outputPath = path.join(tempDir, 'dictionary.zip');
|
||||||
const termEntries: CharacterDictionaryTermEntry[] = [
|
const termEntries: CharacterDictionaryTermEntry[] = [
|
||||||
@@ -62,7 +62,7 @@ test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', ()
|
|||||||
);
|
);
|
||||||
}) as typeof Buffer.concat;
|
}) as typeof Buffer.concat;
|
||||||
|
|
||||||
const result = buildDictionaryZip(
|
const result = await buildDictionaryZip(
|
||||||
outputPath,
|
outputPath,
|
||||||
'Dictionary Title',
|
'Dictionary Title',
|
||||||
'Dictionary Description',
|
'Dictionary Description',
|
||||||
@@ -106,11 +106,11 @@ test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', ()
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('readDictionaryZipRevision reads the built revision and rejects foreign archives', () => {
|
test('readDictionaryZipRevision reads the built revision and rejects foreign archives', async () => {
|
||||||
const dir = makeTempDir();
|
const dir = makeTempDir();
|
||||||
try {
|
try {
|
||||||
const zipPath = path.join(dir, 'merged.zip');
|
const zipPath = path.join(dir, 'merged.zip');
|
||||||
buildDictionaryZip(
|
await buildDictionaryZip(
|
||||||
zipPath,
|
zipPath,
|
||||||
'SubMiner Character Dictionary',
|
'SubMiner Character Dictionary',
|
||||||
'Character names',
|
'Character names',
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import { readStoredZipFirstFile, writeStoredZip } from '../../shared/stored-zip';
|
import { readStoredZipFirstFile, writeStoredZipAsync } from '../../shared/stored-zip';
|
||||||
import { ensureDir } from './fs-utils';
|
import { ensureDir } from './fs-utils';
|
||||||
import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } from './types';
|
import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } from './types';
|
||||||
|
|
||||||
@@ -48,14 +48,14 @@ export function readDictionaryZipRevision(zipPath: string): string | null {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildDictionaryZip(
|
export async function buildDictionaryZip(
|
||||||
outputPath: string,
|
outputPath: string,
|
||||||
dictionaryTitle: string,
|
dictionaryTitle: string,
|
||||||
description: string,
|
description: string,
|
||||||
revision: string,
|
revision: string,
|
||||||
termEntries: CharacterDictionaryTermEntry[],
|
termEntries: CharacterDictionaryTermEntry[],
|
||||||
images: CharacterDictionarySnapshotImage[],
|
images: CharacterDictionarySnapshotImage[],
|
||||||
): { zipPath: string; entryCount: number } {
|
): Promise<{ zipPath: string; entryCount: number }> {
|
||||||
ensureDir(path.dirname(outputPath));
|
ensureDir(path.dirname(outputPath));
|
||||||
|
|
||||||
function* zipFiles(): Iterable<{ name: string; data: Buffer }> {
|
function* zipFiles(): Iterable<{ name: string; data: Buffer }> {
|
||||||
@@ -78,7 +78,11 @@ export function buildDictionaryZip(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const entriesPerBank = 10_000;
|
// Each bank is stringified in one shot, so the bank size sets the longest single block in the
|
||||||
|
// build. 10k entries measured ~38MB and ~135ms per bank on a real merged dictionary; 2k keeps
|
||||||
|
// every bank under the archive writer's yield budget at ~27ms. Yomitan reads any number of
|
||||||
|
// term_bank_N.json files, so this only changes how the terms are split across them.
|
||||||
|
const entriesPerBank = 2_000;
|
||||||
for (let i = 0; i < termEntries.length; i += entriesPerBank) {
|
for (let i = 0; i < termEntries.length; i += entriesPerBank) {
|
||||||
yield {
|
yield {
|
||||||
name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`,
|
name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`,
|
||||||
@@ -87,6 +91,6 @@ export function buildDictionaryZip(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
writeStoredZip(outputPath, zipFiles());
|
await writeStoredZipAsync(outputPath, zipFiles());
|
||||||
return { zipPath: outputPath, entryCount: termEntries.length };
|
return { zipPath: outputPath, entryCount: termEntries.length };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import test from 'node:test';
|
||||||
|
import { readStoredZipFirstFile, writeStoredZip, writeStoredZipAsync } from './stored-zip';
|
||||||
|
|
||||||
|
function makeTempDir(): string {
|
||||||
|
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-stored-zip-'));
|
||||||
|
}
|
||||||
|
|
||||||
|
function readEntries(zipPath: string): Map<string, Buffer> {
|
||||||
|
const archive = fs.readFileSync(zipPath);
|
||||||
|
const entries = new Map<string, Buffer>();
|
||||||
|
let cursor = 0;
|
||||||
|
|
||||||
|
while (cursor + 4 <= archive.length) {
|
||||||
|
const signature = archive.readUInt32LE(cursor);
|
||||||
|
if (signature === 0x02014b50 || signature === 0x06054b50) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
assert.equal(signature, 0x04034b50, `unexpected local file header at offset ${cursor}`);
|
||||||
|
const size = archive.readUInt32LE(cursor + 18);
|
||||||
|
const nameLength = archive.readUInt16LE(cursor + 26);
|
||||||
|
const extraLength = archive.readUInt16LE(cursor + 28);
|
||||||
|
const nameStart = cursor + 30;
|
||||||
|
const dataStart = nameStart + nameLength + extraLength;
|
||||||
|
entries.set(
|
||||||
|
archive.subarray(nameStart, nameStart + nameLength).toString('utf8'),
|
||||||
|
Buffer.from(archive.subarray(dataStart, dataStart + size)),
|
||||||
|
);
|
||||||
|
cursor = dataStart + size;
|
||||||
|
}
|
||||||
|
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The async writer yields on a byte budget between entries, so an entry that is itself larger than
|
||||||
|
// that budget is the case where the accounting could drift: the offsets, CRCs, and central
|
||||||
|
// directory all have to come out identical to the synchronous writer.
|
||||||
|
test('writeStoredZipAsync writes a correct archive when one entry exceeds the yield budget', async () => {
|
||||||
|
const dir = makeTempDir();
|
||||||
|
try {
|
||||||
|
// Comfortably past the writer's 8MB yield budget.
|
||||||
|
const oversized = Buffer.alloc(10 * 1024 * 1024);
|
||||||
|
for (let i = 0; i < oversized.length; i += 1) {
|
||||||
|
oversized[i] = i % 251;
|
||||||
|
}
|
||||||
|
const files = [
|
||||||
|
{ name: 'index.json', data: Buffer.from('{"revision":"rev-1"}', 'utf8') },
|
||||||
|
{ name: 'big.bin', data: oversized },
|
||||||
|
{ name: 'after.txt', data: Buffer.from('written after the oversized entry', 'utf8') },
|
||||||
|
];
|
||||||
|
|
||||||
|
const asyncPath = path.join(dir, 'async.zip');
|
||||||
|
const syncPath = path.join(dir, 'sync.zip');
|
||||||
|
const asyncResult = await writeStoredZipAsync(asyncPath, files);
|
||||||
|
const syncResult = writeStoredZip(syncPath, files);
|
||||||
|
|
||||||
|
assert.equal(asyncResult.entryCount, 3);
|
||||||
|
assert.deepEqual(asyncResult, syncResult);
|
||||||
|
// Byte-identical to the synchronous writer: yielding mid-archive changed no offset or CRC.
|
||||||
|
assert.ok(fs.readFileSync(asyncPath).equals(fs.readFileSync(syncPath)));
|
||||||
|
|
||||||
|
const entries = readEntries(asyncPath);
|
||||||
|
assert.deepEqual([...entries.keys()], ['index.json', 'big.bin', 'after.txt']);
|
||||||
|
assert.ok(entries.get('big.bin')!.equals(oversized));
|
||||||
|
assert.equal(entries.get('after.txt')!.toString('utf8'), 'written after the oversized entry');
|
||||||
|
// The trailing records still parse, which is what proves the archive is complete.
|
||||||
|
assert.equal(readStoredZipFirstFile(asyncPath)?.name, 'index.json');
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
+103
-47
@@ -1,4 +1,5 @@
|
|||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
|
import * as zlib from 'zlib';
|
||||||
|
|
||||||
type ZipEntry = {
|
type ZipEntry = {
|
||||||
name: string;
|
name: string;
|
||||||
@@ -36,10 +37,17 @@ const CRC32_TABLE = (() => {
|
|||||||
return table;
|
return table;
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
// Native CRC32 (Node >= 20.15) runs at native throughput, which matters for the multi-hundred-MB
|
||||||
|
// dictionary archives; the table loop stays as a fallback for runtimes without it.
|
||||||
|
const nativeCrc32 = (zlib as { crc32?: (data: Uint8Array, value?: number) => number }).crc32;
|
||||||
|
|
||||||
function crc32(data: Buffer): number {
|
function crc32(data: Buffer): number {
|
||||||
|
if (typeof nativeCrc32 === 'function') {
|
||||||
|
return nativeCrc32(data) >>> 0;
|
||||||
|
}
|
||||||
let crc = 0xffffffff;
|
let crc = 0xffffffff;
|
||||||
for (const byte of data) {
|
for (let i = 0; i < data.length; i += 1) {
|
||||||
crc = CRC32_TABLE[(crc ^ byte) & 0xff]! ^ (crc >>> 8);
|
crc = CRC32_TABLE[(crc ^ data[i]!) & 0xff]! ^ (crc >>> 8);
|
||||||
}
|
}
|
||||||
return (crc ^ 0xffffffff) >>> 0;
|
return (crc ^ 0xffffffff) >>> 0;
|
||||||
}
|
}
|
||||||
@@ -294,59 +302,70 @@ function writeBuffer(fd: number, buffer: Buffer): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ZipWriteState = {
|
||||||
|
entries: ZipEntry[];
|
||||||
|
offset: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Appends one stored entry (local header + data) and returns the bytes written. */
|
||||||
|
function appendStoredZipFile(fd: number, state: ZipWriteState, file: StoredZipFile): number {
|
||||||
|
const fileName = Buffer.from(file.name, 'utf8');
|
||||||
|
const fileSize = file.data.length;
|
||||||
|
if (fileName.length > ZIP32_MAX_UINT16) {
|
||||||
|
throw new RangeError(`ZIP entry name too long: ${file.name}`);
|
||||||
|
}
|
||||||
|
if (fileSize > ZIP32_MAX_UINT32) {
|
||||||
|
throw new RangeError(`ZIP entry too large for ZIP32: ${file.name}`);
|
||||||
|
}
|
||||||
|
if (state.offset > ZIP32_MAX_UINT32) {
|
||||||
|
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
|
||||||
|
}
|
||||||
|
const fileCrc32 = crc32(file.data);
|
||||||
|
const localHeader = createLocalFileHeader(fileName, fileCrc32, fileSize);
|
||||||
|
const nextOffset = state.offset + localHeader.length + fileSize;
|
||||||
|
if (nextOffset > ZIP32_MAX_UINT32) {
|
||||||
|
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
|
||||||
|
}
|
||||||
|
writeBuffer(fd, localHeader);
|
||||||
|
writeBuffer(fd, file.data);
|
||||||
|
state.entries.push({
|
||||||
|
name: file.name,
|
||||||
|
crc32: fileCrc32,
|
||||||
|
size: fileSize,
|
||||||
|
localHeaderOffset: state.offset,
|
||||||
|
});
|
||||||
|
const written = nextOffset - state.offset;
|
||||||
|
state.offset = nextOffset;
|
||||||
|
return written;
|
||||||
|
}
|
||||||
|
|
||||||
|
function finishStoredZip(fd: number, state: ZipWriteState): void {
|
||||||
|
const centralStart = state.offset;
|
||||||
|
if (centralStart > ZIP32_MAX_UINT32) {
|
||||||
|
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
|
||||||
|
}
|
||||||
|
for (const entry of state.entries) {
|
||||||
|
const centralHeader = createCentralDirectoryHeader(entry);
|
||||||
|
writeBuffer(fd, centralHeader);
|
||||||
|
state.offset += centralHeader.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
const centralSize = state.offset - centralStart;
|
||||||
|
writeBuffer(fd, createEndOfCentralDirectory(state.entries.length, centralSize, centralStart));
|
||||||
|
}
|
||||||
|
|
||||||
export function writeStoredZip(
|
export function writeStoredZip(
|
||||||
outputPath: string,
|
outputPath: string,
|
||||||
files: Iterable<StoredZipFile>,
|
files: Iterable<StoredZipFile>,
|
||||||
): { entryCount: number } {
|
): { entryCount: number } {
|
||||||
const entries: ZipEntry[] = [];
|
const state: ZipWriteState = { entries: [], offset: 0 };
|
||||||
let offset = 0;
|
|
||||||
const fd = fs.openSync(outputPath, 'w');
|
const fd = fs.openSync(outputPath, 'w');
|
||||||
|
|
||||||
try {
|
try {
|
||||||
for (const file of files) {
|
for (const file of files) {
|
||||||
const fileName = Buffer.from(file.name, 'utf8');
|
appendStoredZipFile(fd, state, file);
|
||||||
const fileSize = file.data.length;
|
|
||||||
if (fileName.length > ZIP32_MAX_UINT16) {
|
|
||||||
throw new RangeError(`ZIP entry name too long: ${file.name}`);
|
|
||||||
}
|
|
||||||
if (fileSize > ZIP32_MAX_UINT32) {
|
|
||||||
throw new RangeError(`ZIP entry too large for ZIP32: ${file.name}`);
|
|
||||||
}
|
|
||||||
if (offset > ZIP32_MAX_UINT32) {
|
|
||||||
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
|
|
||||||
}
|
|
||||||
const fileCrc32 = crc32(file.data);
|
|
||||||
const localHeader = createLocalFileHeader(fileName, fileCrc32, fileSize);
|
|
||||||
const nextOffset = offset + localHeader.length + fileSize;
|
|
||||||
if (nextOffset > ZIP32_MAX_UINT32) {
|
|
||||||
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
|
|
||||||
}
|
|
||||||
writeBuffer(fd, localHeader);
|
|
||||||
writeBuffer(fd, file.data);
|
|
||||||
entries.push({
|
|
||||||
name: file.name,
|
|
||||||
crc32: fileCrc32,
|
|
||||||
size: fileSize,
|
|
||||||
localHeaderOffset: offset,
|
|
||||||
});
|
|
||||||
if (nextOffset > ZIP32_MAX_UINT32) {
|
|
||||||
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
|
|
||||||
}
|
|
||||||
offset = nextOffset;
|
|
||||||
}
|
}
|
||||||
|
finishStoredZip(fd, state);
|
||||||
const centralStart = offset;
|
|
||||||
if (centralStart > ZIP32_MAX_UINT32) {
|
|
||||||
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
|
|
||||||
}
|
|
||||||
for (const entry of entries) {
|
|
||||||
const centralHeader = createCentralDirectoryHeader(entry);
|
|
||||||
writeBuffer(fd, centralHeader);
|
|
||||||
offset += centralHeader.length;
|
|
||||||
}
|
|
||||||
|
|
||||||
const centralSize = offset - centralStart;
|
|
||||||
writeBuffer(fd, createEndOfCentralDirectory(entries.length, centralSize, centralStart));
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
fs.closeSync(fd);
|
fs.closeSync(fd);
|
||||||
fs.rmSync(outputPath, { force: true });
|
fs.rmSync(outputPath, { force: true });
|
||||||
@@ -354,5 +373,42 @@ export function writeStoredZip(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fs.closeSync(fd);
|
fs.closeSync(fd);
|
||||||
return { entryCount: entries.length };
|
return { entryCount: state.entries.length };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Yielding roughly every 8MB keeps individual event-loop blocks in the low tens of milliseconds
|
||||||
|
// while adding a negligible number of macrotask hops even for the largest merged dictionary.
|
||||||
|
const ASYNC_ZIP_YIELD_BYTE_BUDGET = 8 * 1024 * 1024;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Same archive as {@link writeStoredZip}, written without starving the event loop: entry
|
||||||
|
* generation, CRC, and writes proceed in byte-budgeted slices with a macrotask yield in between.
|
||||||
|
* Multi-hundred-MB dictionary archives previously blocked the main process long enough for the
|
||||||
|
* compositor to declare the app unresponsive.
|
||||||
|
*/
|
||||||
|
export async function writeStoredZipAsync(
|
||||||
|
outputPath: string,
|
||||||
|
files: Iterable<StoredZipFile>,
|
||||||
|
): Promise<{ entryCount: number }> {
|
||||||
|
const state: ZipWriteState = { entries: [], offset: 0 };
|
||||||
|
const fd = fs.openSync(outputPath, 'w');
|
||||||
|
|
||||||
|
try {
|
||||||
|
let bytesSinceYield = 0;
|
||||||
|
for (const file of files) {
|
||||||
|
bytesSinceYield += appendStoredZipFile(fd, state, file);
|
||||||
|
if (bytesSinceYield >= ASYNC_ZIP_YIELD_BYTE_BUDGET) {
|
||||||
|
bytesSinceYield = 0;
|
||||||
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finishStoredZip(fd, state);
|
||||||
|
} catch (error) {
|
||||||
|
fs.closeSync(fd);
|
||||||
|
fs.rmSync(outputPath, { force: true });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.closeSync(fd);
|
||||||
|
return { entryCount: state.entries.length };
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user