Compare commits

...
16 changed files with 536 additions and 160 deletions
@@ -0,0 +1,5 @@
type: fixed
area: dictionary
- Character dictionary generation, merged rebuilds, and imports no longer freeze the app (and trigger the compositor's "application not responding" dialog) on large dictionaries; snapshot reads/writes, archive building, and the character image/name lookup caches now do their heavy work off the UI's critical path.
- Desktop progress notifications now update in place on Linux AppImage installs too: the AppImage's bundled libraries broke the system notify-send helper, which silently forced the flickering close-and-reopen notification fallback.
+17 -1
View File
@@ -1,6 +1,22 @@
import assert from 'node:assert/strict'; import assert from 'node:assert/strict';
import test from 'node:test'; import test from 'node:test';
import { createNotifySendReplacer, resolveDefaultNotificationIconPath } from './notification'; import {
buildNotifySendEnv,
createNotifySendReplacer,
resolveDefaultNotificationIconPath,
} from './notification';
test('notify-send child environment drops the AppImage library-path override', () => {
const env = buildNotifySendEnv({
LD_LIBRARY_PATH: '/tmp/.mount_SubMinXXXXXX/usr/lib',
DBUS_SESSION_BUS_ADDRESS: 'unix:path=/run/user/1000/bus',
HOME: '/home/user',
});
assert.equal(env.LD_LIBRARY_PATH, undefined);
assert.equal(env.DBUS_SESSION_BUS_ADDRESS, 'unix:path=/run/user/1000/bus');
assert.equal(env.HOME, '/home/user');
});
test('default notification icon resolves packaged SubMiner asset when no per-notification icon is provided', () => { test('default notification icon resolves packaged SubMiner asset when no per-notification icon is provided', () => {
const path = resolveDefaultNotificationIconPath({ const path = resolveDefaultNotificationIconPath({
+12 -1
View File
@@ -203,8 +203,19 @@ export function createNotifySendReplacer(
}; };
} }
/**
* Electron AppImages export `LD_LIBRARY_PATH=<mount>/usr/lib`, whose bundled libnotify predates the
* symbols the system notify-send links against, so an inherited environment kills the child with a
* symbol lookup error before it can send anything. A system binary resolves its own libraries fine,
* so the override is dropped entirely rather than filtered.
*/
export function buildNotifySendEnv(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
const { LD_LIBRARY_PATH: _dropped, ...rest } = env;
return rest;
}
const showLinuxReplaceableNotification = createNotifySendReplacer((args, callback) => const showLinuxReplaceableNotification = createNotifySendReplacer((args, callback) =>
execFile('notify-send', args, { timeout: 5_000 }, (error, stdout) => execFile('notify-send', args, { timeout: 5_000, env: buildNotifySendEnv() }, (error, stdout) =>
callback(error, stdout ?? ''), callback(error, stdout ?? ''),
), ),
); );
+19 -14
View File
@@ -244,13 +244,13 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
}; };
}; };
const findCachedSnapshotForSeriesKey = ( const findCachedSnapshotForSeriesKey = async (
seriesKey: string, seriesKey: string,
fallbackSeriesKey?: string, fallbackSeriesKey?: string,
): CharacterDictionarySnapshot | null => { ): Promise<CharacterDictionarySnapshot | null> => {
const acceptedKeys = new Set([seriesKey, fallbackSeriesKey].filter(Boolean)); const acceptedKeys = new Set([seriesKey, fallbackSeriesKey].filter(Boolean));
return ( return (
readCachedSnapshots(outputDir).find((snapshot) => { (await readCachedSnapshots(outputDir)).find((snapshot) => {
const snapshotSeriesKey = buildCharacterDictionarySeriesKey({ const snapshotSeriesKey = buildCharacterDictionarySeriesKey({
mediaPath: null, mediaPath: null,
mediaTitle: snapshot.mediaTitle, mediaTitle: snapshot.mediaTitle,
@@ -293,7 +293,9 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
const cachedResolution = readCachedMediaResolution(outputDir, seriesKey); const cachedResolution = readCachedMediaResolution(outputDir, seriesKey);
if (cachedResolution) { if (cachedResolution) {
const cachedSnapshot = readSnapshot(getSnapshotPath(outputDir, cachedResolution.mediaId)); const cachedSnapshot = await readSnapshot(
getSnapshotPath(outputDir, cachedResolution.mediaId),
);
if (cachedSnapshot) { if (cachedSnapshot) {
deps.logInfo?.( deps.logInfo?.(
`[dictionary] cached AniList match: ${cachedSnapshot.mediaTitle} -> AniList ${cachedSnapshot.mediaId}`, `[dictionary] cached AniList match: ${cachedSnapshot.mediaTitle} -> AniList ${cachedSnapshot.mediaId}`,
@@ -305,7 +307,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
} }
} }
const cachedSnapshot = findCachedSnapshotForSeriesKey(seriesKey, unscopedSeriesKey); const cachedSnapshot = await findCachedSnapshotForSeriesKey(seriesKey, unscopedSeriesKey);
if (cachedSnapshot) { if (cachedSnapshot) {
writeCachedMediaResolution(outputDir, { writeCachedMediaResolution(outputDir, {
seriesKey, seriesKey,
@@ -348,7 +350,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
progress?: CharacterDictionarySnapshotProgressCallbacks, progress?: CharacterDictionarySnapshotProgressCallbacks,
): Promise<CharacterDictionarySnapshotResult> => { ): Promise<CharacterDictionarySnapshotResult> => {
const snapshotPath = getSnapshotPath(outputDir, mediaId); const snapshotPath = getSnapshotPath(outputDir, mediaId);
const cachedSnapshot = readSnapshot(snapshotPath); const cachedSnapshot = await readSnapshot(snapshotPath);
const refreshReason = cachedSnapshot ? getCachedSnapshotRefreshReason(cachedSnapshot) : null; const refreshReason = cachedSnapshot ? getCachedSnapshotRefreshReason(cachedSnapshot) : null;
if (cachedSnapshot && refreshReason === null) { if (cachedSnapshot && refreshReason === null) {
deps.logInfo?.(`[dictionary] snapshot hit for AniList ${mediaId}`); deps.logInfo?.(`[dictionary] snapshot hit for AniList ${mediaId}`);
@@ -485,7 +487,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
resolvedNameSplits, resolvedNameSplits,
nameSplitSource, nameSplitSource,
); );
writeSnapshot(snapshotPath, snapshot); await writeSnapshot(snapshotPath, snapshot);
deps.logInfo?.( deps.logInfo?.(
`[dictionary] stored snapshot for AniList ${mediaId}: ${snapshot.entryCount} terms`, `[dictionary] stored snapshot for AniList ${mediaId}: ${snapshot.entryCount} terms`,
); );
@@ -526,19 +528,22 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
const snapshotResults = await Promise.all( const snapshotResults = await Promise.all(
normalizedMediaIds.map((mediaId) => getOrCreateSnapshot(mediaId)), normalizedMediaIds.map((mediaId) => getOrCreateSnapshot(mediaId)),
); );
const snapshots = snapshotResults.map(({ mediaId }) => { // Sequential on purpose: each snapshot parse is a chunk of main-thread work, so reading them
const snapshot = readSnapshot(getSnapshotPath(outputDir, mediaId)); // one at a time keeps the event loop breathing between files.
const snapshots: CharacterDictionarySnapshot[] = [];
for (const { mediaId } of snapshotResults) {
const snapshot = await readSnapshot(getSnapshotPath(outputDir, mediaId));
if (!snapshot) { if (!snapshot) {
throw new Error(`Missing character dictionary snapshot for AniList ${mediaId}.`); throw new Error(`Missing character dictionary snapshot for AniList ${mediaId}.`);
} }
return snapshot; snapshots.push(snapshot);
}); }
const revision = buildMergedRevision(normalizedMediaIds, snapshots); const revision = buildMergedRevision(normalizedMediaIds, snapshots);
const description = const description =
snapshots.length === 1 snapshots.length === 1
? `Character names from ${snapshots[0]!.mediaTitle}` ? `Character names from ${snapshots[0]!.mediaTitle}`
: `Character names from ${snapshots.length} recent anime`; : `Character names from ${snapshots.length} recent anime`;
const { zipPath, entryCount } = buildDictionaryZip( const { zipPath, entryCount } = await buildDictionaryZip(
getMergedZipPath(outputDir), getMergedZipPath(outputDir),
CHARACTER_DICTIONARY_MERGED_TITLE, CHARACTER_DICTIONARY_MERGED_TITLE,
description, description,
@@ -633,7 +638,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
resolvedMedia.title, resolvedMedia.title,
waitForAniListRequestSlot, waitForAniListRequestSlot,
); );
const storedSnapshot = readSnapshot(getSnapshotPath(outputDir, resolvedMedia.id)); const storedSnapshot = await readSnapshot(getSnapshotPath(outputDir, resolvedMedia.id));
if (!storedSnapshot) { if (!storedSnapshot) {
throw new Error(`Snapshot missing after generation for AniList ${resolvedMedia.id}.`); throw new Error(`Snapshot missing after generation for AniList ${resolvedMedia.id}.`);
} }
@@ -642,7 +647,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
const description = `Character names from ${storedSnapshot.mediaTitle} [AniList media ID ${resolvedMedia.id}]`; const description = `Character names from ${storedSnapshot.mediaTitle} [AniList media ID ${resolvedMedia.id}]`;
const zipPath = path.join(outputDir, `anilist-${resolvedMedia.id}.zip`); const zipPath = path.join(outputDir, `anilist-${resolvedMedia.id}.zip`);
deps.logInfo?.(`[dictionary] building ZIP for AniList ${resolvedMedia.id}`); deps.logInfo?.(`[dictionary] building ZIP for AniList ${resolvedMedia.id}`);
buildDictionaryZip( await buildDictionaryZip(
zipPath, zipPath,
dictionaryTitle, dictionaryTitle,
description, description,
@@ -3,6 +3,7 @@ import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
import * as path from 'path'; import * as path from 'path';
import test from 'node:test'; import test from 'node:test';
import { isDeepStrictEqual } from 'node:util';
import { getSnapshotPath, readSnapshot, writeSnapshot } from './cache'; import { getSnapshotPath, readSnapshot, writeSnapshot } from './cache';
import { CHARACTER_DICTIONARY_FORMAT_VERSION } from './constants'; import { CHARACTER_DICTIONARY_FORMAT_VERSION } from './constants';
import type { CharacterDictionarySnapshot } from './types'; import type { CharacterDictionarySnapshot } from './types';
@@ -29,17 +30,72 @@ function createSnapshot(): CharacterDictionarySnapshot {
}; };
} }
test('writeSnapshot persists and readSnapshot restores current-format snapshots', () => { test('writeSnapshot persists and readSnapshot restores current-format snapshots', async () => {
const outputDir = makeTempDir(); const outputDir = makeTempDir();
const snapshotPath = getSnapshotPath(outputDir, 130298); const snapshotPath = getSnapshotPath(outputDir, 130298);
const snapshot = createSnapshot(); const snapshot = createSnapshot();
writeSnapshot(snapshotPath, snapshot); await writeSnapshot(snapshotPath, snapshot);
assert.deepEqual(readSnapshot(snapshotPath), { ...snapshot, nameSplitSource: 'heuristic' }); assert.deepEqual(await readSnapshot(snapshotPath), { ...snapshot, nameSplitSource: 'heuristic' });
}); });
test('readSnapshot preserves the mecab name-split source and defaults missing values to heuristic', () => { // A manual generate and an auto-sync can both land on the same media, so two writes for one
// snapshot can overlap. They must not stream into a shared temp file and interleave into a
// half-and-half snapshot.
test('concurrent writeSnapshot calls for the same media leave one complete snapshot', async () => {
const outputDir = makeTempDir();
const snapshotPath = getSnapshotPath(outputDir, 130298);
const base = createSnapshot();
// Distinct titles, lengths, and term text so the surviving file can be pinned to exactly one
// writer rather than merely "a snapshot that parses". A shared temp file is caught by the
// losing writers failing to rename; interleaved content is only caught when the timing happens
// to leave a mix, which is why the assertion checks identity rather than shape.
const variants: CharacterDictionarySnapshot[] = ['alpha', 'beta', 'gamma'].map((label, index) => {
const entryCount = 400 + index * 100;
return {
...base,
mediaTitle: `${base.mediaTitle} ${label}`,
entryCount,
termEntries: Array.from({ length: entryCount }, (_entry, entryIndex) => [
`${label}${entryIndex}`,
'なまえ',
'name primary',
'',
75,
[`${label} character ${entryIndex} `.repeat(600)],
0,
'',
]) as CharacterDictionarySnapshot['termEntries'],
};
});
await Promise.all(variants.map((variant) => writeSnapshot(snapshotPath, variant)));
const restored = await readSnapshot(snapshotPath);
const expected = variants.map((variant) => ({
...variant,
nameSplitSource: 'heuristic' as const,
}));
const matches = expected.filter((candidate) => isDeepStrictEqual(restored, candidate));
assert.equal(
matches.length,
1,
`expected exactly one writer's complete snapshot to survive, got ${
restored === null
? 'an unreadable file'
: `entryCount=${restored.entryCount}, terms=${restored.termEntries.length}, title=${restored.mediaTitle}`
}`,
);
// Every writer cleaned up after itself, so no temp files are left behind.
const leftovers = fs
.readdirSync(path.dirname(snapshotPath))
.filter((name) => name.includes('.tmp-'));
assert.deepEqual(leftovers, []);
});
test('readSnapshot preserves the mecab name-split source and defaults missing values to heuristic', async () => {
const outputDir = makeTempDir(); const outputDir = makeTempDir();
const snapshotPath = getSnapshotPath(outputDir, 130298); const snapshotPath = getSnapshotPath(outputDir, 130298);
const snapshot: CharacterDictionarySnapshot = { const snapshot: CharacterDictionarySnapshot = {
@@ -47,12 +103,12 @@ test('readSnapshot preserves the mecab name-split source and defaults missing va
nameSplitSource: 'mecab', nameSplitSource: 'mecab',
}; };
writeSnapshot(snapshotPath, snapshot); await writeSnapshot(snapshotPath, snapshot);
assert.equal(readSnapshot(snapshotPath)?.nameSplitSource, 'mecab'); assert.equal((await readSnapshot(snapshotPath))?.nameSplitSource, 'mecab');
}); });
test('readSnapshot ignores snapshots written with an older format version', () => { test('readSnapshot ignores snapshots written with an older format version', async () => {
const outputDir = makeTempDir(); const outputDir = makeTempDir();
const snapshotPath = getSnapshotPath(outputDir, 130298); const snapshotPath = getSnapshotPath(outputDir, 130298);
const staleSnapshot = { const staleSnapshot = {
@@ -63,10 +119,10 @@ test('readSnapshot ignores snapshots written with an older format version', () =
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true }); fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
fs.writeFileSync(snapshotPath, JSON.stringify(staleSnapshot), 'utf8'); fs.writeFileSync(snapshotPath, JSON.stringify(staleSnapshot), 'utf8');
assert.equal(readSnapshot(snapshotPath), null); assert.equal(await readSnapshot(snapshotPath), null);
}); });
test('readSnapshot ignores v15 snapshots with stale romanized character-name entries', () => { test('readSnapshot ignores v15 snapshots with stale romanized character-name entries', async () => {
const outputDir = makeTempDir(); const outputDir = makeTempDir();
const snapshotPath = getSnapshotPath(outputDir, 130298); const snapshotPath = getSnapshotPath(outputDir, 130298);
const staleSnapshot = { const staleSnapshot = {
@@ -78,5 +134,5 @@ test('readSnapshot ignores v15 snapshots with stale romanized character-name ent
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true }); fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
fs.writeFileSync(snapshotPath, JSON.stringify(staleSnapshot), 'utf8'); fs.writeFileSync(snapshotPath, JSON.stringify(staleSnapshot), 'utf8');
assert.equal(readSnapshot(snapshotPath), null); assert.equal(await readSnapshot(snapshotPath), null);
}); });
+83 -10
View File
@@ -102,24 +102,42 @@ export function writeCachedMediaResolution(
writeMediaResolutionEntries(outputDir, [...remaining, normalized]); writeMediaResolutionEntries(outputDir, [...remaining, normalized]);
} }
export function readCachedSnapshots(outputDir: string): CharacterDictionarySnapshot[] { /**
* Snapshots for long series run to hundreds of MB each, so everything here reads them off the main
* thread's critical path: file IO is async and only the unavoidable JSON.parse runs on the loop,
* one file at a time. Reading the whole directory synchronously used to block the process for
* multiple seconds, long enough for the compositor to declare the app unresponsive mid-playback.
*/
export async function readCachedSnapshots(
outputDir: string,
): Promise<CharacterDictionarySnapshot[]> {
let entries: fs.Dirent[] = []; let entries: fs.Dirent[] = [];
try { try {
entries = fs.readdirSync(getSnapshotsDir(outputDir), { withFileTypes: true }); entries = await fs.promises.readdir(getSnapshotsDir(outputDir), { withFileTypes: true });
} catch { } catch {
return []; return [];
} }
return entries const names = entries
.filter((entry) => entry.isFile() && /^anilist-\d+\.json$/.test(entry.name)) .filter((entry) => entry.isFile() && /^anilist-\d+\.json$/.test(entry.name))
.sort((left, right) => left.name.localeCompare(right.name)) .map((entry) => entry.name)
.map((entry) => readSnapshot(path.join(getSnapshotsDir(outputDir), entry.name))) .sort((left, right) => left.localeCompare(right));
.filter((snapshot): snapshot is CharacterDictionarySnapshot => snapshot !== null);
const snapshots: CharacterDictionarySnapshot[] = [];
for (const name of names) {
const snapshot = await readSnapshot(path.join(getSnapshotsDir(outputDir), name));
if (snapshot) {
snapshots.push(snapshot);
}
}
return snapshots;
} }
export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot | null { export async function readSnapshot(
snapshotPath: string,
): Promise<CharacterDictionarySnapshot | null> {
try { try {
const raw = fs.readFileSync(snapshotPath, 'utf8'); const raw = await fs.promises.readFile(snapshotPath, 'utf8');
const parsed = JSON.parse(raw) as Partial<CharacterDictionarySnapshot>; const parsed = JSON.parse(raw) as Partial<CharacterDictionarySnapshot>;
if (!parsed || typeof parsed !== 'object') { if (!parsed || typeof parsed !== 'object') {
return null; return null;
@@ -150,9 +168,64 @@ export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot
} }
} }
export function writeSnapshot(snapshotPath: string, snapshot: CharacterDictionarySnapshot): void { // Flushing in a few-MB batches keeps each stringify-and-write slice short; a single
// JSON.stringify of a large snapshot blocks the event loop for seconds.
const SNAPSHOT_WRITE_FLUSH_BYTES = 4 * 1024 * 1024;
// Distinguishes concurrent writes of the same snapshot within one process; the pid alone only
// separates processes, so two overlapping writers would otherwise stream into the same temp file.
let snapshotWriteSequence = 0;
/**
* Streams the snapshot to disk piece by piece instead of stringifying it in one shot, then renames
* the finished file into place so a crash mid-write (or two concurrent writers for the same media)
* can never leave a torn file where a snapshot used to be.
*/
export async function writeSnapshot(
snapshotPath: string,
snapshot: CharacterDictionarySnapshot,
): Promise<void> {
ensureDir(path.dirname(snapshotPath)); ensureDir(path.dirname(snapshotPath));
fs.writeFileSync(snapshotPath, JSON.stringify(snapshot, null, 2), 'utf8'); snapshotWriteSequence += 1;
const tempPath = `${snapshotPath}.tmp-${process.pid}-${snapshotWriteSequence}`;
const handle = await fs.promises.open(tempPath, 'w');
try {
let buffered: string[] = [];
let bufferedBytes = 0;
const push = async (chunk: string): Promise<void> => {
buffered.push(chunk);
bufferedBytes += chunk.length;
if (bufferedBytes >= SNAPSHOT_WRITE_FLUSH_BYTES) {
const joined = buffered.join('');
buffered = [];
bufferedBytes = 0;
await handle.write(joined, null, 'utf8');
}
};
const writeArray = async (key: string, items: readonly unknown[]): Promise<void> => {
await push(`,${JSON.stringify(key)}:[`);
for (let i = 0; i < items.length; i += 1) {
await push(`${i > 0 ? ',' : ''}${JSON.stringify(items[i])}`);
}
await push(']');
};
const { termEntries, images, ...scalars } = snapshot;
const head = JSON.stringify(scalars);
await push(head.slice(0, -1));
await writeArray('termEntries', termEntries);
await writeArray('images', images);
await push('}');
if (buffered.length > 0) {
await handle.write(buffered.join(''), null, 'utf8');
}
} catch (error) {
await handle.close();
await fs.promises.rm(tempPath, { force: true });
throw error;
}
await handle.close();
await fs.promises.rename(tempPath, snapshotPath);
} }
export function buildMergedRevision( export function buildMergedRevision(
@@ -11,6 +11,22 @@ import {
} from './image-lookup'; } from './image-lookup';
import type { CharacterDictionarySnapshot } from './types'; import type { CharacterDictionarySnapshot } from './types';
// Lookup indexes rebuild in the background while gets serve stale data, so tests poll until the
// refresh they triggered has landed.
async function waitForRefresh<T>(probe: () => T | null | undefined): Promise<T> {
const deadline = Date.now() + 5000;
for (;;) {
const value = probe();
if (value !== null && value !== undefined) {
return value;
}
if (Date.now() > deadline) {
throw new Error('timed out waiting for background snapshot refresh');
}
await new Promise((resolve) => setTimeout(resolve, 5));
}
}
const PNG_1X1_BASE64 = const PNG_1X1_BASE64 =
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+nmX8AAAAASUVORK5CYII='; 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+nmX8AAAAASUVORK5CYII=';
@@ -18,7 +34,7 @@ function makeTempDir(): string {
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-character-image-lookup-')); return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-character-image-lookup-'));
} }
test('buildCharacterNameImageIndexFromSnapshots maps name terms to character portrait data URLs', () => { test('buildCharacterNameImageIndexFromSnapshots maps name terms to character portrait data URLs', async () => {
const outputDir = makeTempDir(); const outputDir = makeTempDir();
const snapshot: CharacterDictionarySnapshot = { const snapshot: CharacterDictionarySnapshot = {
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION, formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
@@ -75,9 +91,9 @@ test('buildCharacterNameImageIndexFromSnapshots maps name terms to character por
{ path: 'img/m130298-va456.png', dataBase64: 'BBBB' }, { path: 'img/m130298-va456.png', dataBase64: 'BBBB' },
], ],
}; };
writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot); await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
const index = buildCharacterNameImageIndexFromSnapshots(outputDir); const index = await buildCharacterNameImageIndexFromSnapshots(outputDir);
assert.deepEqual(index.get('アレクシア'), { assert.deepEqual(index.get('アレクシア'), {
src: 'data:image/png;base64,AAAA', src: 'data:image/png;base64,AAAA',
@@ -85,7 +101,7 @@ test('buildCharacterNameImageIndexFromSnapshots maps name terms to character por
}); });
}); });
test('buildCharacterNameImageIndexFromSnapshots sniffs image MIME from bytes before path extension', () => { test('buildCharacterNameImageIndexFromSnapshots sniffs image MIME from bytes before path extension', async () => {
const outputDir = makeTempDir(); const outputDir = makeTempDir();
const snapshot: CharacterDictionarySnapshot = { const snapshot: CharacterDictionarySnapshot = {
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION, formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
@@ -116,14 +132,14 @@ test('buildCharacterNameImageIndexFromSnapshots sniffs image MIME from bytes bef
], ],
images: [{ path: 'img/m130298-c123.jpg', dataBase64: PNG_1X1_BASE64 }], images: [{ path: 'img/m130298-c123.jpg', dataBase64: PNG_1X1_BASE64 }],
}; };
writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot); await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
const index = buildCharacterNameImageIndexFromSnapshots(outputDir); const index = await buildCharacterNameImageIndexFromSnapshots(outputDir);
assert.equal(index.get('アレクシア')?.src, `data:image/png;base64,${PNG_1X1_BASE64}`); assert.equal(index.get('アレクシア')?.src, `data:image/png;base64,${PNG_1X1_BASE64}`);
}); });
test('createCharacterDictionaryImageLookup can scope duplicate names to the current media', () => { test('createCharacterDictionaryImageLookup can scope duplicate names to the current media', async () => {
const outputDir = makeTempDir(); const outputDir = makeTempDir();
const towerSnapshot: CharacterDictionarySnapshot = { const towerSnapshot: CharacterDictionarySnapshot = {
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION, formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
@@ -173,15 +189,16 @@ test('createCharacterDictionaryImageLookup can scope duplicate names to the curr
], ],
images: [{ path: 'img/m21202-c2.png', dataBase64: 'KONOSUBA' }], images: [{ path: 'img/m21202-c2.png', dataBase64: 'KONOSUBA' }],
}; };
writeSnapshot(getSnapshotPath(outputDir, towerSnapshot.mediaId), towerSnapshot); await writeSnapshot(getSnapshotPath(outputDir, towerSnapshot.mediaId), towerSnapshot);
writeSnapshot(getSnapshotPath(outputDir, konosubaSnapshot.mediaId), konosubaSnapshot); await writeSnapshot(getSnapshotPath(outputDir, konosubaSnapshot.mediaId), konosubaSnapshot);
const lookup = createCharacterDictionaryImageLookup({ outputDir }); const lookup = createCharacterDictionaryImageLookup({ outputDir });
assert.equal(lookup.get('カズ', 21202)?.alt, 'Kazuma'); const scoped = await waitForRefresh(() => lookup.get('カズ', 21202));
assert.equal(scoped.alt, 'Kazuma');
}); });
test('createCharacterDictionaryImageLookup does not fall back globally on scoped miss', () => { test('createCharacterDictionaryImageLookup does not fall back globally on scoped miss', async () => {
const outputDir = makeTempDir(); const outputDir = makeTempDir();
const snapshot: CharacterDictionarySnapshot = { const snapshot: CharacterDictionarySnapshot = {
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION, formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
@@ -208,10 +225,11 @@ test('createCharacterDictionaryImageLookup does not fall back globally on scoped
], ],
images: [{ path: 'img/m115230-c1.png', dataBase64: 'TOWER' }], images: [{ path: 'img/m115230-c1.png', dataBase64: 'TOWER' }],
}; };
writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot); await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
const lookup = createCharacterDictionaryImageLookup({ outputDir }); const lookup = createCharacterDictionaryImageLookup({ outputDir });
const unscoped = await waitForRefresh(() => lookup.get('カズ'));
assert.equal(unscoped.alt, 'Kaz');
assert.equal(lookup.get('カズ', 21202), null); assert.equal(lookup.get('カズ', 21202), null);
assert.equal(lookup.get('カズ')?.alt, 'Kaz');
}); });
@@ -204,11 +204,11 @@ function getSnapshotDirectorySignature(outputDir: string): string {
return parts.sort().join('|'); return parts.sort().join('|');
} }
export function buildCharacterNameImageIndexFromSnapshots( export async function buildCharacterNameImageIndexFromSnapshots(
outputDir: string, outputDir: string,
): Map<string, CharacterNameImage> { ): Promise<Map<string, CharacterNameImage>> {
const index = new Map<string, CharacterNameImage>(); const index = new Map<string, CharacterNameImage>();
for (const snapshot of readCachedSnapshots(outputDir)) { for (const snapshot of await readCachedSnapshots(outputDir)) {
appendSnapshotImages(index, snapshot); appendSnapshotImages(index, snapshot);
} }
return index; return index;
@@ -228,7 +228,12 @@ export function createCharacterDictionaryImageLookup(deps: {
let signature: string | null = null; let signature: string | null = null;
let index = new Map<string, CharacterNameImage>(); let index = new Map<string, CharacterNameImage>();
let indexByMediaId = new Map<number, Map<string, CharacterNameImage>>(); let indexByMediaId = new Map<number, Map<string, CharacterNameImage>>();
let refreshInFlight = false;
// Rebuilding means re-reading every cached snapshot (potentially GBs of JSON), which used to run
// synchronously inside a lookup and froze the whole app right after a snapshot changed. Lookups
// now serve the previous index while a single background rebuild catches up; the swap is atomic
// and the signature only advances once the rebuild it belongs to has landed.
function refreshIfNeeded(): void { function refreshIfNeeded(): void {
if (!outputDir) { if (!outputDir) {
index = new Map<string, CharacterNameImage>(); index = new Map<string, CharacterNameImage>();
@@ -237,20 +242,30 @@ export function createCharacterDictionaryImageLookup(deps: {
return; return;
} }
const nextSignature = getSnapshotDirectorySignature(outputDir); const nextSignature = getSnapshotDirectorySignature(outputDir);
if (nextSignature === signature) { if (nextSignature === signature || refreshInFlight) {
return; return;
} }
signature = nextSignature; refreshInFlight = true;
index = new Map<string, CharacterNameImage>(); void (async () => {
indexByMediaId = new Map<number, Map<string, CharacterNameImage>>(); try {
for (const snapshot of readCachedSnapshots(outputDir)) { const snapshots = await readCachedSnapshots(outputDir);
appendSnapshotImages(index, snapshot); const nextIndex = new Map<string, CharacterNameImage>();
const mediaIndex = new Map<string, CharacterNameImage>(); const nextIndexByMediaId = new Map<number, Map<string, CharacterNameImage>>();
appendSnapshotImages(mediaIndex, snapshot); for (const snapshot of snapshots) {
if (mediaIndex.size > 0) { appendSnapshotImages(nextIndex, snapshot);
indexByMediaId.set(snapshot.mediaId, mediaIndex); const mediaIndex = new Map<string, CharacterNameImage>();
appendSnapshotImages(mediaIndex, snapshot);
if (mediaIndex.size > 0) {
nextIndexByMediaId.set(snapshot.mediaId, mediaIndex);
}
}
index = nextIndex;
indexByMediaId = nextIndexByMediaId;
signature = nextSignature;
} finally {
refreshInFlight = false;
} }
} })();
} }
return { return {
@@ -32,17 +32,33 @@ function writeSnapshot(outputDir: string, mediaId: number, entries: Array<[strin
); );
} }
function withTempDir<T>(run: (dir: string) => T): T { async function withTempDir<T>(run: (dir: string) => Promise<T> | T): Promise<T> {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-name-candidates-')); const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-name-candidates-'));
try { try {
return run(dir); return await run(dir);
} finally { } finally {
fs.rmSync(dir, { recursive: true, force: true }); fs.rmSync(dir, { recursive: true, force: true });
} }
} }
test('collects terms and readings for the current media', () => { // The snapshot index rebuilds in the background while lookups serve stale data, so tests poll the
withTempDir((dir) => { // probe until the refresh they triggered has landed.
async function waitForRefresh<T>(probe: () => T | null | undefined): Promise<T> {
const deadline = Date.now() + 5000;
for (;;) {
const value = probe();
if (value !== null && value !== undefined) {
return value;
}
if (Date.now() > deadline) {
throw new Error('timed out waiting for background snapshot refresh');
}
await new Promise((resolve) => setTimeout(resolve, 5));
}
}
test('collects terms and readings for the current media', async () => {
await withTempDir(async (dir) => {
writeSnapshot(dir, 1, [ writeSnapshot(dir, 1, [
['ミナト', 'みなと'], ['ミナト', 'みなと'],
['湊', 'みなと'], ['湊', 'みなと'],
@@ -53,17 +69,16 @@ test('collects terms and readings for the current media', () => {
outputDir: dir, outputDir: dir,
getCurrentMediaId: () => 1, getCurrentMediaId: () => 1,
}); });
const candidates = lookup.get(); const candidates = await waitForRefresh(() => lookup.get());
assert.ok(candidates);
assert.deepEqual([...candidates.forms].sort(), ['みなと', 'ミナト', '湊'].sort()); assert.deepEqual([...candidates.forms].sort(), ['みなと', 'ミナト', '湊'].sort());
// Deduplicated: both entries share the みなと reading. // Deduplicated: both entries share the みなと reading.
assert.equal(candidates.forms.length, 3); assert.equal(candidates.forms.length, 3);
}); });
}); });
test('returns null without a media scope so the scanner stays exhaustive', () => { test('returns null without a media scope so the scanner stays exhaustive', async () => {
withTempDir((dir) => { await withTempDir(async (dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]); writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
const lookup = createCharacterNameCandidateLookup({ const lookup = createCharacterNameCandidateLookup({
@@ -71,12 +86,14 @@ test('returns null without a media scope so the scanner stays exhaustive', () =>
getCurrentMediaId: () => null, getCurrentMediaId: () => null,
}); });
// The explicitly-scoped probe proves the index has loaded before the unscoped case is judged.
await waitForRefresh(() => lookup.get(1));
assert.equal(lookup.get(), null); assert.equal(lookup.get(), null);
}); });
}); });
test('returns null for a media with no cached snapshot', () => { test('returns null for a media with no cached snapshot', async () => {
withTempDir((dir) => { await withTempDir(async (dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]); writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
const lookup = createCharacterNameCandidateLookup({ const lookup = createCharacterNameCandidateLookup({
@@ -84,29 +101,31 @@ test('returns null for a media with no cached snapshot', () => {
getCurrentMediaId: () => 999, getCurrentMediaId: () => 999,
}); });
await waitForRefresh(() => lookup.get(1));
assert.equal(lookup.get(), null); assert.equal(lookup.get(), null);
}); });
}); });
test('key changes when the snapshot content changes', () => { test('key changes when the snapshot content changes', async () => {
withTempDir((dir) => { await withTempDir(async (dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]); writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
const lookup = createCharacterNameCandidateLookup({ const lookup = createCharacterNameCandidateLookup({
outputDir: dir, outputDir: dir,
getCurrentMediaId: () => 1, getCurrentMediaId: () => 1,
}); });
const first = lookup.get(); const first = await waitForRefresh(() => lookup.get());
writeSnapshot(dir, 1, [ writeSnapshot(dir, 1, [
['ミナト', 'みなと'], ['ミナト', 'みなと'],
['アクア', 'あくあ'], ['アクア', 'あくあ'],
]); ]);
lookup.invalidate(); lookup.invalidate();
const second = lookup.get(); const second = await waitForRefresh(() => {
const candidates = lookup.get();
return candidates && candidates.forms.length === 4 ? candidates : null;
});
assert.ok(first && second);
assert.notEqual(first.key, second.key); assert.notEqual(first.key, second.key);
assert.equal(second.forms.length, 4);
}); });
}); });
@@ -114,8 +133,8 @@ test('key changes when the snapshot content changes', () => {
// directory every call. Asserted behaviorally: an unannounced on-disk change is // directory every call. Asserted behaviorally: an unannounced on-disk change is
// invisible until the recheck interval elapses, which can only be true if the // invisible until the recheck interval elapses, which can only be true if the
// filesystem is not consulted per lookup. // filesystem is not consulted per lookup.
test('does not re-read the snapshot directory on every lookup', () => { test('does not re-read the snapshot directory on every lookup', async () => {
withTempDir((dir) => { await withTempDir(async (dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]); writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
let nowMs = 1_000_000; let nowMs = 1_000_000;
const lookup = createCharacterNameCandidateLookup({ const lookup = createCharacterNameCandidateLookup({
@@ -124,6 +143,7 @@ test('does not re-read the snapshot directory on every lookup', () => {
now: () => nowMs, now: () => nowMs,
}); });
await waitForRefresh(() => lookup.get());
assert.equal(lookup.get()?.forms.length, 2); assert.equal(lookup.get()?.forms.length, 2);
writeSnapshot(dir, 1, [ writeSnapshot(dir, 1, [
@@ -135,12 +155,16 @@ test('does not re-read the snapshot directory on every lookup', () => {
assert.equal(lookup.get()?.forms.length, 2, 'expected the cached list within the interval'); assert.equal(lookup.get()?.forms.length, 2, 'expected the cached list within the interval');
nowMs += 10_000; nowMs += 10_000;
assert.equal(lookup.get()?.forms.length, 4, 'expected a refresh past the interval'); const refreshed = await waitForRefresh(() => {
const candidates = lookup.get();
return candidates && candidates.forms.length === 4 ? candidates : null;
});
assert.equal(refreshed.forms.length, 4, 'expected a refresh past the interval');
}); });
}); });
test('invalidate picks up a snapshot change immediately', () => { test('invalidate picks up a snapshot change on the next refresh', async () => {
withTempDir((dir) => { await withTempDir(async (dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]); writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
let nowMs = 1_000_000; let nowMs = 1_000_000;
const lookup = createCharacterNameCandidateLookup({ const lookup = createCharacterNameCandidateLookup({
@@ -149,6 +173,7 @@ test('invalidate picks up a snapshot change immediately', () => {
now: () => nowMs, now: () => nowMs,
}); });
await waitForRefresh(() => lookup.get());
assert.equal(lookup.get()?.forms.length, 2); assert.equal(lookup.get()?.forms.length, 2);
writeSnapshot(dir, 1, [ writeSnapshot(dir, 1, [
@@ -158,6 +183,10 @@ test('invalidate picks up a snapshot change immediately', () => {
nowMs += 1; nowMs += 1;
lookup.invalidate(); lookup.invalidate();
assert.equal(lookup.get()?.forms.length, 4); const refreshed = await waitForRefresh(() => {
const candidates = lookup.get();
return candidates && candidates.forms.length === 4 ? candidates : null;
});
assert.equal(refreshed.forms.length, 4);
}); });
}); });
@@ -98,7 +98,12 @@ export function createCharacterNameCandidateLookup(deps: {
let signature: string | null = null; let signature: string | null = null;
let lastSignatureCheckAtMs = 0; let lastSignatureCheckAtMs = 0;
let formsByMediaId = new Map<number, string[]>(); let formsByMediaId = new Map<number, string[]>();
let refreshInFlight = false;
// Same stale-while-revalidate shape as the image lookup: the rebuild re-reads every cached
// snapshot, so it runs in the background while lookups keep serving the previous forms. The
// signature only advances once its rebuild has landed, so a failed or superseded rebuild is
// retried on the next signature check.
function refreshIfNeeded(): void { function refreshIfNeeded(): void {
if (!outputDir) { if (!outputDir) {
formsByMediaId = new Map<number, string[]>(); formsByMediaId = new Map<number, string[]>();
@@ -114,17 +119,26 @@ export function createCharacterNameCandidateLookup(deps: {
} }
lastSignatureCheckAtMs = nowMs; lastSignatureCheckAtMs = nowMs;
const nextSignature = getSnapshotDirectorySignature(outputDir); const nextSignature = getSnapshotDirectorySignature(outputDir);
if (nextSignature === signature) { if (nextSignature === signature || refreshInFlight) {
return; return;
} }
signature = nextSignature; refreshInFlight = true;
formsByMediaId = new Map<number, string[]>(); void (async () => {
for (const snapshot of readCachedSnapshots(outputDir)) { try {
const forms = collectSnapshotNameForms(snapshot); const snapshots = await readCachedSnapshots(outputDir);
if (forms.length > 0) { const nextFormsByMediaId = new Map<number, string[]>();
formsByMediaId.set(snapshot.mediaId, forms); for (const snapshot of snapshots) {
const forms = collectSnapshotNameForms(snapshot);
if (forms.length > 0) {
nextFormsByMediaId.set(snapshot.mediaId, forms);
}
}
formsByMediaId = nextFormsByMediaId;
signature = nextSignature;
} finally {
refreshInFlight = false;
} }
} })();
} }
return { return {
@@ -34,7 +34,7 @@ function createSnapshotWithoutImages(): CharacterDictionarySnapshot {
test('generateForCurrentMedia refreshes same-version snapshots missing images when inline images are enabled', async () => { test('generateForCurrentMedia refreshes same-version snapshots missing images when inline images are enabled', async () => {
const userDataPath = makeTempDir(); const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries'); const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages()); await writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
const originalFetch = globalThis.fetch; const originalFetch = globalThis.fetch;
const fetchUrls: string[] = []; const fetchUrls: string[] = [];
@@ -124,7 +124,7 @@ test('generateForCurrentMedia refreshes same-version snapshots missing images wh
test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable', async () => { test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable', async () => {
const userDataPath = makeTempDir(); const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries'); const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), { await writeSnapshot(getSnapshotPath(outputDir, 130298), {
...createSnapshotWithoutImages(), ...createSnapshotWithoutImages(),
nameSplitSource: 'heuristic', nameSplitSource: 'heuristic',
}); });
@@ -213,7 +213,7 @@ test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable'
test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is available', async () => { test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is available', async () => {
const userDataPath = makeTempDir(); const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries'); const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), { await writeSnapshot(getSnapshotPath(outputDir, 130298), {
...createSnapshotWithoutImages(), ...createSnapshotWithoutImages(),
nameSplitSource: 'mecab', nameSplitSource: 'mecab',
}); });
@@ -253,7 +253,7 @@ test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is availabl
test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is unavailable', async () => { test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is unavailable', async () => {
const userDataPath = makeTempDir(); const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries'); const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), { await writeSnapshot(getSnapshotPath(outputDir, 130298), {
...createSnapshotWithoutImages(), ...createSnapshotWithoutImages(),
nameSplitSource: 'heuristic', nameSplitSource: 'heuristic',
}); });
@@ -293,7 +293,7 @@ test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is una
test('generateForCurrentMedia keeps same-version snapshots without images when inline images are disabled', async () => { test('generateForCurrentMedia keeps same-version snapshots without images when inline images are disabled', async () => {
const userDataPath = makeTempDir(); const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries'); const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages()); await writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
const originalFetch = globalThis.fetch; const originalFetch = globalThis.fetch;
globalThis.fetch = (async (input: string | URL | Request) => { globalThis.fetch = (async (input: string | URL | Request) => {
@@ -42,7 +42,7 @@ function readStoredZipEntries(zipPath: string): Map<string, Buffer> {
return entries; return entries;
} }
test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', () => { test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', async () => {
const tempDir = makeTempDir(); const tempDir = makeTempDir();
const outputPath = path.join(tempDir, 'dictionary.zip'); const outputPath = path.join(tempDir, 'dictionary.zip');
const termEntries: CharacterDictionaryTermEntry[] = [ const termEntries: CharacterDictionaryTermEntry[] = [
@@ -62,7 +62,7 @@ test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', ()
); );
}) as typeof Buffer.concat; }) as typeof Buffer.concat;
const result = buildDictionaryZip( const result = await buildDictionaryZip(
outputPath, outputPath,
'Dictionary Title', 'Dictionary Title',
'Dictionary Description', 'Dictionary Description',
@@ -106,11 +106,11 @@ test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', ()
} }
}); });
test('readDictionaryZipRevision reads the built revision and rejects foreign archives', () => { test('readDictionaryZipRevision reads the built revision and rejects foreign archives', async () => {
const dir = makeTempDir(); const dir = makeTempDir();
try { try {
const zipPath = path.join(dir, 'merged.zip'); const zipPath = path.join(dir, 'merged.zip');
buildDictionaryZip( await buildDictionaryZip(
zipPath, zipPath,
'SubMiner Character Dictionary', 'SubMiner Character Dictionary',
'Character names', 'Character names',
+9 -5
View File
@@ -1,5 +1,5 @@
import * as path from 'path'; import * as path from 'path';
import { readStoredZipFirstFile, writeStoredZip } from '../../shared/stored-zip'; import { readStoredZipFirstFile, writeStoredZipAsync } from '../../shared/stored-zip';
import { ensureDir } from './fs-utils'; import { ensureDir } from './fs-utils';
import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } from './types'; import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } from './types';
@@ -48,14 +48,14 @@ export function readDictionaryZipRevision(zipPath: string): string | null {
} }
} }
export function buildDictionaryZip( export async function buildDictionaryZip(
outputPath: string, outputPath: string,
dictionaryTitle: string, dictionaryTitle: string,
description: string, description: string,
revision: string, revision: string,
termEntries: CharacterDictionaryTermEntry[], termEntries: CharacterDictionaryTermEntry[],
images: CharacterDictionarySnapshotImage[], images: CharacterDictionarySnapshotImage[],
): { zipPath: string; entryCount: number } { ): Promise<{ zipPath: string; entryCount: number }> {
ensureDir(path.dirname(outputPath)); ensureDir(path.dirname(outputPath));
function* zipFiles(): Iterable<{ name: string; data: Buffer }> { function* zipFiles(): Iterable<{ name: string; data: Buffer }> {
@@ -78,7 +78,11 @@ export function buildDictionaryZip(
}; };
} }
const entriesPerBank = 10_000; // Each bank is stringified in one shot, so the bank size sets the longest single block in the
// build. 10k entries measured ~38MB and ~135ms per bank on a real merged dictionary; 2k keeps
// every bank under the archive writer's yield budget at ~27ms. Yomitan reads any number of
// term_bank_N.json files, so this only changes how the terms are split across them.
const entriesPerBank = 2_000;
for (let i = 0; i < termEntries.length; i += entriesPerBank) { for (let i = 0; i < termEntries.length; i += entriesPerBank) {
yield { yield {
name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`, name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`,
@@ -87,6 +91,6 @@ export function buildDictionaryZip(
} }
} }
writeStoredZip(outputPath, zipFiles()); await writeStoredZipAsync(outputPath, zipFiles());
return { zipPath: outputPath, entryCount: termEntries.length }; return { zipPath: outputPath, entryCount: termEntries.length };
} }
+74
View File
@@ -0,0 +1,74 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { readStoredZipFirstFile, writeStoredZip, writeStoredZipAsync } from './stored-zip';
function makeTempDir(): string {
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-stored-zip-'));
}
function readEntries(zipPath: string): Map<string, Buffer> {
const archive = fs.readFileSync(zipPath);
const entries = new Map<string, Buffer>();
let cursor = 0;
while (cursor + 4 <= archive.length) {
const signature = archive.readUInt32LE(cursor);
if (signature === 0x02014b50 || signature === 0x06054b50) {
break;
}
assert.equal(signature, 0x04034b50, `unexpected local file header at offset ${cursor}`);
const size = archive.readUInt32LE(cursor + 18);
const nameLength = archive.readUInt16LE(cursor + 26);
const extraLength = archive.readUInt16LE(cursor + 28);
const nameStart = cursor + 30;
const dataStart = nameStart + nameLength + extraLength;
entries.set(
archive.subarray(nameStart, nameStart + nameLength).toString('utf8'),
Buffer.from(archive.subarray(dataStart, dataStart + size)),
);
cursor = dataStart + size;
}
return entries;
}
// The async writer yields on a byte budget between entries, so an entry that is itself larger than
// that budget is the case where the accounting could drift: the offsets, CRCs, and central
// directory all have to come out identical to the synchronous writer.
test('writeStoredZipAsync writes a correct archive when one entry exceeds the yield budget', async () => {
const dir = makeTempDir();
try {
// Comfortably past the writer's 8MB yield budget.
const oversized = Buffer.alloc(10 * 1024 * 1024);
for (let i = 0; i < oversized.length; i += 1) {
oversized[i] = i % 251;
}
const files = [
{ name: 'index.json', data: Buffer.from('{"revision":"rev-1"}', 'utf8') },
{ name: 'big.bin', data: oversized },
{ name: 'after.txt', data: Buffer.from('written after the oversized entry', 'utf8') },
];
const asyncPath = path.join(dir, 'async.zip');
const syncPath = path.join(dir, 'sync.zip');
const asyncResult = await writeStoredZipAsync(asyncPath, files);
const syncResult = writeStoredZip(syncPath, files);
assert.equal(asyncResult.entryCount, 3);
assert.deepEqual(asyncResult, syncResult);
// Byte-identical to the synchronous writer: yielding mid-archive changed no offset or CRC.
assert.ok(fs.readFileSync(asyncPath).equals(fs.readFileSync(syncPath)));
const entries = readEntries(asyncPath);
assert.deepEqual([...entries.keys()], ['index.json', 'big.bin', 'after.txt']);
assert.ok(entries.get('big.bin')!.equals(oversized));
assert.equal(entries.get('after.txt')!.toString('utf8'), 'written after the oversized entry');
// The trailing records still parse, which is what proves the archive is complete.
assert.equal(readStoredZipFirstFile(asyncPath)?.name, 'index.json');
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
+103 -47
View File
@@ -1,4 +1,5 @@
import * as fs from 'fs'; import * as fs from 'fs';
import * as zlib from 'zlib';
type ZipEntry = { type ZipEntry = {
name: string; name: string;
@@ -36,10 +37,17 @@ const CRC32_TABLE = (() => {
return table; return table;
})(); })();
// Native CRC32 (Node >= 20.15) runs at native throughput, which matters for the multi-hundred-MB
// dictionary archives; the table loop stays as a fallback for runtimes without it.
const nativeCrc32 = (zlib as { crc32?: (data: Uint8Array, value?: number) => number }).crc32;
function crc32(data: Buffer): number { function crc32(data: Buffer): number {
if (typeof nativeCrc32 === 'function') {
return nativeCrc32(data) >>> 0;
}
let crc = 0xffffffff; let crc = 0xffffffff;
for (const byte of data) { for (let i = 0; i < data.length; i += 1) {
crc = CRC32_TABLE[(crc ^ byte) & 0xff]! ^ (crc >>> 8); crc = CRC32_TABLE[(crc ^ data[i]!) & 0xff]! ^ (crc >>> 8);
} }
return (crc ^ 0xffffffff) >>> 0; return (crc ^ 0xffffffff) >>> 0;
} }
@@ -294,59 +302,70 @@ function writeBuffer(fd: number, buffer: Buffer): void {
} }
} }
type ZipWriteState = {
entries: ZipEntry[];
offset: number;
};
/** Appends one stored entry (local header + data) and returns the bytes written. */
function appendStoredZipFile(fd: number, state: ZipWriteState, file: StoredZipFile): number {
const fileName = Buffer.from(file.name, 'utf8');
const fileSize = file.data.length;
if (fileName.length > ZIP32_MAX_UINT16) {
throw new RangeError(`ZIP entry name too long: ${file.name}`);
}
if (fileSize > ZIP32_MAX_UINT32) {
throw new RangeError(`ZIP entry too large for ZIP32: ${file.name}`);
}
if (state.offset > ZIP32_MAX_UINT32) {
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
}
const fileCrc32 = crc32(file.data);
const localHeader = createLocalFileHeader(fileName, fileCrc32, fileSize);
const nextOffset = state.offset + localHeader.length + fileSize;
if (nextOffset > ZIP32_MAX_UINT32) {
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
}
writeBuffer(fd, localHeader);
writeBuffer(fd, file.data);
state.entries.push({
name: file.name,
crc32: fileCrc32,
size: fileSize,
localHeaderOffset: state.offset,
});
const written = nextOffset - state.offset;
state.offset = nextOffset;
return written;
}
function finishStoredZip(fd: number, state: ZipWriteState): void {
const centralStart = state.offset;
if (centralStart > ZIP32_MAX_UINT32) {
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
}
for (const entry of state.entries) {
const centralHeader = createCentralDirectoryHeader(entry);
writeBuffer(fd, centralHeader);
state.offset += centralHeader.length;
}
const centralSize = state.offset - centralStart;
writeBuffer(fd, createEndOfCentralDirectory(state.entries.length, centralSize, centralStart));
}
export function writeStoredZip( export function writeStoredZip(
outputPath: string, outputPath: string,
files: Iterable<StoredZipFile>, files: Iterable<StoredZipFile>,
): { entryCount: number } { ): { entryCount: number } {
const entries: ZipEntry[] = []; const state: ZipWriteState = { entries: [], offset: 0 };
let offset = 0;
const fd = fs.openSync(outputPath, 'w'); const fd = fs.openSync(outputPath, 'w');
try { try {
for (const file of files) { for (const file of files) {
const fileName = Buffer.from(file.name, 'utf8'); appendStoredZipFile(fd, state, file);
const fileSize = file.data.length;
if (fileName.length > ZIP32_MAX_UINT16) {
throw new RangeError(`ZIP entry name too long: ${file.name}`);
}
if (fileSize > ZIP32_MAX_UINT32) {
throw new RangeError(`ZIP entry too large for ZIP32: ${file.name}`);
}
if (offset > ZIP32_MAX_UINT32) {
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
}
const fileCrc32 = crc32(file.data);
const localHeader = createLocalFileHeader(fileName, fileCrc32, fileSize);
const nextOffset = offset + localHeader.length + fileSize;
if (nextOffset > ZIP32_MAX_UINT32) {
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
}
writeBuffer(fd, localHeader);
writeBuffer(fd, file.data);
entries.push({
name: file.name,
crc32: fileCrc32,
size: fileSize,
localHeaderOffset: offset,
});
if (nextOffset > ZIP32_MAX_UINT32) {
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
}
offset = nextOffset;
} }
finishStoredZip(fd, state);
const centralStart = offset;
if (centralStart > ZIP32_MAX_UINT32) {
throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
}
for (const entry of entries) {
const centralHeader = createCentralDirectoryHeader(entry);
writeBuffer(fd, centralHeader);
offset += centralHeader.length;
}
const centralSize = offset - centralStart;
writeBuffer(fd, createEndOfCentralDirectory(entries.length, centralSize, centralStart));
} catch (error) { } catch (error) {
fs.closeSync(fd); fs.closeSync(fd);
fs.rmSync(outputPath, { force: true }); fs.rmSync(outputPath, { force: true });
@@ -354,5 +373,42 @@ export function writeStoredZip(
} }
fs.closeSync(fd); fs.closeSync(fd);
return { entryCount: entries.length }; return { entryCount: state.entries.length };
}
// Yielding roughly every 8MB keeps individual event-loop blocks in the low tens of milliseconds
// while adding a negligible number of macrotask hops even for the largest merged dictionary.
const ASYNC_ZIP_YIELD_BYTE_BUDGET = 8 * 1024 * 1024;
/**
* Same archive as {@link writeStoredZip}, written without starving the event loop: entry
* generation, CRC, and writes proceed in byte-budgeted slices with a macrotask yield in between.
* Multi-hundred-MB dictionary archives previously blocked the main process long enough for the
* compositor to declare the app unresponsive.
*/
export async function writeStoredZipAsync(
outputPath: string,
files: Iterable<StoredZipFile>,
): Promise<{ entryCount: number }> {
const state: ZipWriteState = { entries: [], offset: 0 };
const fd = fs.openSync(outputPath, 'w');
try {
let bytesSinceYield = 0;
for (const file of files) {
bytesSinceYield += appendStoredZipFile(fd, state, file);
if (bytesSinceYield >= ASYNC_ZIP_YIELD_BYTE_BUDGET) {
bytesSinceYield = 0;
await new Promise<void>((resolve) => setImmediate(resolve));
}
}
finishStoredZip(fd, state);
} catch (error) {
fs.closeSync(fd);
fs.rmSync(outputPath, { force: true });
throw error;
}
fs.closeSync(fd);
return { entryCount: state.entries.length };
} }
@@ -98,10 +98,10 @@ export function DuplicateLineCleanup({ onClose, onCleaned }: DuplicateLineCleanu
<div className="space-y-4 px-5 py-4"> <div className="space-y-4 px-5 py-4">
<p className="text-xs leading-relaxed text-ctp-subtext0"> <p className="text-xs leading-relaxed text-ctp-subtext0">
Typeset subtitles karaoke openings, animated signs are authored as one event per Karaoke openings and animated signs are typeset as one subtitle event per animation
animation frame, and older versions counted every frame as its own line. This finds frame, and older versions counted every frame as a line. This collapses those runs back
those runs and collapses each one back to a single line, giving back the word and kanji to one line and drops the word and kanji counts they added. Repeated dialogue is left
counts they inflated. Ordinary repeated dialogue is left alone. alone.
</p> </p>
<div> <div>
@@ -192,8 +192,8 @@ export function DuplicateLineCleanup({ onClose, onCleaned }: DuplicateLineCleanu
</button> </button>
</div> </div>
<p className="text-[11px] text-ctp-overlay1"> <p className="text-[11px] text-ctp-overlay1">
Scan first: cleanup removes rows and cannot be undone. Session watch time and lines-seen Scan first: cleanup deletes rows and can't be undone. Watch time and lines-seen totals
totals are left untouched. stay as they are.
</p> </p>
</div> </div>
</div> </div>