fix(dictionary): stop large character dictionaries from timing out (#189)

2026-09-24 17:16:21 -07:00 · 2026-08-11 18:40:18 -07:00
parent 7b0fbdf254
commit ee25536d90
16 changed files with 1147 additions and 108 deletions
@@ -3,6 +3,7 @@ import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
 import test from 'node:test';
+import { buildDictionaryZip } from '../character-dictionary-runtime/zip';
 import {
  createCharacterDictionaryAutoSyncRuntimeService,
  getCharacterDictionaryManagerSnapshot,
@@ -14,6 +15,14 @@ function makeTempDir(): string {
  return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-char-dict-auto-sync-'));
 }

+async function waitUntil(predicate: () => boolean, label: string): Promise<void> {
+  for (let attempt = 0; attempt < 500; attempt += 1) {
+    if (predicate()) return;
+    await new Promise((resolve) => setTimeout(resolve, 0));
+  }
+  throw new Error(`Timed out waiting for ${label}`);
+}
+
 function createDeferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
  let resolve!: (value: T) => void;
  const promise = new Promise<T>((nextResolve) => {
@@ -187,7 +196,7 @@ test('auto sync imports merged dictionary and persists MRU state', async () => {
    '[dictionary:auto-sync] syncing current anime snapshot',
    '[dictionary:auto-sync] active AniList media set: 130298 - The Eminence in Shadow',
    '[dictionary:auto-sync] rebuilding merged dictionary for active anime set',
-    '[dictionary:auto-sync] importing merged dictionary: /tmp/subminer-character-dictionary.zip',
+    '[dictionary:auto-sync] importing merged dictionary: /tmp/subminer-character-dictionary.zip (timeout 120000ms)',
    '[dictionary:auto-sync] applying Yomitan settings for SubMiner Character Dictionary',
    '[dictionary:auto-sync] synced AniList 130298: SubMiner Character Dictionary (2544 entries)',
  ]);
@@ -367,7 +376,14 @@ test('auto sync reimports existing merged zip without rebuilding on unchanged re
  const userDataPath = makeTempDir();
  const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
  fs.mkdirSync(dictionariesDir, { recursive: true });
-  fs.writeFileSync(path.join(dictionariesDir, 'merged.zip'), 'cached-zip', 'utf8');
+  buildDictionaryZip(
+    path.join(dictionariesDir, 'merged.zip'),
+    'SubMiner Character Dictionary',
+    'Character names',
+    'rev-7',
+    [{ term: 'フリーレン', reading: 'フリーレン', role: 'main', glossary: [] } as never],
+    [],
+  );
  const mergedBuilds: number[][] = [];
  const imports: string[] = [];
  let importedRevision: string | null = null;
@@ -958,11 +974,9 @@ test('auto sync emits building while merged dictionary generation is in flight',
  });

  const syncPromise = runtime.runSyncNow();
-  await Promise.resolve();
-
-  assert.equal(
-    events.some((event) => event.phase === 'building'),
-    true,
+  await waitUntil(
+    () => events.some((event) => event.phase === 'building'),
+    'the building status event',
  );

  buildDeferred.resolve({
@@ -1029,8 +1043,7 @@ test('auto sync waits for tokenization-ready gate before Yomitan mutations', asy
  });

  const syncPromise = runtime.runSyncNow();
-  await Promise.resolve();
-  await Promise.resolve();
+  await waitUntil(() => calls.includes('wait'), 'the tokenization-ready gate');

  assert.deepEqual(calls, ['build', 'wait']);

@@ -1039,3 +1052,395 @@ test('auto sync waits for tokenization-ready gate before Yomitan mutations', asy

  assert.deepEqual(calls, ['build', 'wait', 'info', 'import', 'settings']);
 });
+
+test('auto sync scales the import timeout with the merged dictionary size', async () => {
+  const userDataPath = makeTempDir();
+  const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
+  fs.mkdirSync(dictionariesDir, { recursive: true });
+  const zipPath = path.join(dictionariesDir, 'merged.zip');
+  // 2 MB of merged dictionary buys ~12s of import budget on top of the base.
+  fs.writeFileSync(zipPath, Buffer.alloc(2 * 1024 * 1024));
+  const events: Array<{ phase: string; message: string }> = [];
+  let importedRevision: string | null = null;
+
+  const runtime = createCharacterDictionaryAutoSyncRuntimeService({
+    userDataPath,
+    getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
+    getOrCreateCurrentSnapshot: async () => ({
+      mediaId: 21,
+      mediaTitle: 'ONE PIECE',
+      entryCount: 4000,
+      fromCache: false,
+      updatedAt: 1000,
+    }),
+    buildMergedDictionary: async () => ({
+      zipPath,
+      revision: 'rev-21',
+      dictionaryTitle: 'SubMiner Character Dictionary',
+      entryCount: 4000,
+    }),
+    getYomitanDictionaryInfo: async () =>
+      importedRevision
+        ? [{ title: 'SubMiner Character Dictionary', revision: importedRevision }]
+        : [],
+    importYomitanDictionary: async () => {
+      // Far longer than the quick-operation budget, well inside the size-scaled one.
+      await new Promise((resolve) => setTimeout(resolve, 400));
+      importedRevision = 'rev-21';
+      return true;
+    },
+    deleteYomitanDictionary: async () => true,
+    upsertYomitanDictionarySettings: async () => true,
+    now: () => 1000,
+    // Comfortable for the stubs that resolve immediately, still far under the import's 400ms.
+    operationTimeoutMs: 100,
+    dictionaryImportTimeoutBaseMs: 20,
+    onSyncStatus: (event) => {
+      events.push({ phase: event.phase, message: event.message });
+    },
+  });
+
+  await runtime.runSyncNow();
+
+  assert.equal(
+    events.some((event) => event.phase === 'failed'),
+    false,
+  );
+  assert.deepEqual(events.at(-1), {
+    phase: 'ready',
+    message: 'Character dictionary ready for ONE PIECE',
+  });
+});
+
+test('auto sync reports the scaled budget when an import really does hang', async () => {
+  const userDataPath = makeTempDir();
+  const events: Array<{ phase: string; message: string }> = [];
+
+  const runtime = createCharacterDictionaryAutoSyncRuntimeService({
+    userDataPath,
+    getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
+    getOrCreateCurrentSnapshot: async () => ({
+      mediaId: 21,
+      mediaTitle: 'ONE PIECE',
+      entryCount: 4000,
+      fromCache: false,
+      updatedAt: 1000,
+    }),
+    buildMergedDictionary: async () => ({
+      zipPath: path.join(userDataPath, 'character-dictionaries', 'missing.zip'),
+      revision: 'rev-21',
+      dictionaryTitle: 'SubMiner Character Dictionary',
+      entryCount: 4000,
+    }),
+    getYomitanDictionaryInfo: async () => [],
+    importYomitanDictionary: () => new Promise<boolean>(() => {}),
+    deleteYomitanDictionary: async () => true,
+    upsertYomitanDictionarySettings: async () => true,
+    now: () => 1000,
+    dictionaryImportTimeoutBaseMs: 20,
+    onSyncStatus: (event) => {
+      events.push({ phase: event.phase, message: event.message });
+    },
+  });
+
+  await assert.rejects(
+    runtime.runSyncNow(),
+    /importYomitanDictionary\(missing\.zip\) timed out after 20ms/,
+  );
+  assert.equal(events.at(-1)?.phase, 'failed');
+});
+
+test('auto sync ticks the importing notification while the import runs', async () => {
+  const userDataPath = makeTempDir();
+  const events: Array<{ phase: string; message: string }> = [];
+  const scheduled: Array<() => void> = [];
+  const importDeferred = createDeferred<boolean>();
+  let clock = 1000;
+
+  const runtime = createCharacterDictionaryAutoSyncRuntimeService({
+    userDataPath,
+    getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
+    getOrCreateCurrentSnapshot: async () => ({
+      mediaId: 21,
+      mediaTitle: 'ONE PIECE',
+      entryCount: 4000,
+      fromCache: false,
+      updatedAt: 1000,
+    }),
+    buildMergedDictionary: async () => ({
+      zipPath: '/tmp/merged.zip',
+      revision: 'rev-21',
+      dictionaryTitle: 'SubMiner Character Dictionary',
+      entryCount: 4000,
+    }),
+    getYomitanDictionaryInfo: async () => [],
+    importYomitanDictionary: () => importDeferred.promise,
+    deleteYomitanDictionary: async () => true,
+    upsertYomitanDictionarySettings: async () => true,
+    now: () => clock,
+    schedule: (fn) => {
+      scheduled.push(fn);
+      return 0 as unknown as ReturnType<typeof setTimeout>;
+    },
+    clearSchedule: () => undefined,
+    onSyncStatus: (event) => {
+      events.push({ phase: event.phase, message: event.message });
+    },
+  });
+
+  const syncPromise = runtime.runSyncNow();
+  await waitUntil(
+    () => events.some((event) => event.phase === 'importing'),
+    'the importing status event',
+  );
+
+  clock = 1000 + 65_000;
+  // The importing heartbeat is the most recently scheduled tick.
+  scheduled.at(-1)!();
+
+  assert.deepEqual(events.at(-1), {
+    phase: 'importing',
+    message: 'Importing character dictionary for ONE PIECE (1m 05s)...',
+  });
+
+  importDeferred.resolve(true);
+  await syncPromise;
+  assert.equal(events.at(-1)?.phase, 'ready');
+});
+
+test('auto sync reports character and image counts while generating a snapshot', async () => {
+  const userDataPath = makeTempDir();
+  const events: Array<{ phase: string; message: string }> = [];
+  let clock = 1000;
+  let importedRevision: string | null = null;
+
+  const runtime = createCharacterDictionaryAutoSyncRuntimeService({
+    userDataPath,
+    getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
+    getOrCreateCurrentSnapshot: async (_targetPath, progress) => {
+      progress?.onGenerating?.({ mediaId: 21, mediaTitle: 'ONE PIECE' });
+      progress?.onGenerateProgress?.({
+        mediaId: 21,
+        mediaTitle: 'ONE PIECE',
+        stage: 'characters',
+        completed: 50,
+        total: null,
+        page: 12,
+      });
+      // Same stage, same clock tick: throttled away so a 33-page fetch cannot spam the overlay.
+      progress?.onGenerateProgress?.({
+        mediaId: 21,
+        mediaTitle: 'ONE PIECE',
+        stage: 'characters',
+        completed: 100,
+        total: null,
+        page: 13,
+      });
+      // A stage change always reports, throttle window or not.
+      progress?.onGenerateProgress?.({
+        mediaId: 21,
+        mediaTitle: 'ONE PIECE',
+        stage: 'images',
+        completed: 1,
+        total: 1220,
+      });
+      clock += 2000;
+      progress?.onGenerateProgress?.({
+        mediaId: 21,
+        mediaTitle: 'ONE PIECE',
+        stage: 'images',
+        completed: 240,
+        total: 1220,
+      });
+      clock += 6000;
+      progress?.onGenerateProgress?.({
+        mediaId: 21,
+        mediaTitle: 'ONE PIECE',
+        stage: 'names',
+        completed: 800,
+        total: 1220,
+      });
+      progress?.onGenerateProgress?.({
+        mediaId: 21,
+        mediaTitle: 'ONE PIECE',
+        stage: 'saving',
+        completed: 0,
+        total: null,
+      });
+      return {
+        mediaId: 21,
+        mediaTitle: 'ONE PIECE',
+        entryCount: 4000,
+        fromCache: false,
+        updatedAt: 1000,
+      };
+    },
+    buildMergedDictionary: async () => ({
+      zipPath: '/tmp/merged.zip',
+      revision: 'rev-21',
+      dictionaryTitle: 'SubMiner Character Dictionary',
+      entryCount: 4000,
+    }),
+    getYomitanDictionaryInfo: async () =>
+      importedRevision
+        ? [{ title: 'SubMiner Character Dictionary', revision: importedRevision }]
+        : [],
+    importYomitanDictionary: async () => {
+      importedRevision = 'rev-21';
+      return true;
+    },
+    deleteYomitanDictionary: async () => true,
+    upsertYomitanDictionarySettings: async () => true,
+    now: () => clock,
+    onSyncStatus: (event) => {
+      events.push({ phase: event.phase, message: event.message });
+    },
+  });
+
+  await runtime.runSyncNow();
+
+  assert.deepEqual(
+    events.filter((event) => event.phase === 'generating').map((event) => event.message),
+    [
+      'Generating character dictionary for ONE PIECE...',
+      'Generating character dictionary for ONE PIECE (page 12, 50 characters)...',
+      'Generating character dictionary for ONE PIECE (image 1/1220)...',
+      'Generating character dictionary for ONE PIECE (image 240/1220, ~10s left)...',
+      'Generating character dictionary for ONE PIECE (name 800/1220 · 8s)...',
+      'Generating character dictionary for ONE PIECE (saving snapshot · 8s)...',
+    ],
+  );
+});
+
+test('auto sync keeps the generating clock ticking when a stage stalls', async () => {
+  const userDataPath = makeTempDir();
+  const events: Array<{ phase: string; message: string }> = [];
+  const scheduled: Array<() => void> = [];
+  const snapshotDeferred = createDeferred<{
+    mediaId: number;
+    mediaTitle: string;
+    entryCount: number;
+    fromCache: boolean;
+    updatedAt: number;
+  }>();
+  let clock = 1000;
+
+  const runtime = createCharacterDictionaryAutoSyncRuntimeService({
+    userDataPath,
+    getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
+    getOrCreateCurrentSnapshot: async (_targetPath, progress) => {
+      progress?.onGenerating?.({ mediaId: 21, mediaTitle: 'ONE PIECE' });
+      progress?.onGenerateProgress?.({
+        mediaId: 21,
+        mediaTitle: 'ONE PIECE',
+        stage: 'images',
+        completed: 240,
+        total: 1220,
+      });
+      return await snapshotDeferred.promise;
+    },
+    buildMergedDictionary: async () => ({
+      zipPath: '/tmp/merged.zip',
+      revision: 'rev-21',
+      dictionaryTitle: 'SubMiner Character Dictionary',
+      entryCount: 4000,
+    }),
+    getYomitanDictionaryInfo: async () => [],
+    importYomitanDictionary: async () => true,
+    deleteYomitanDictionary: async () => true,
+    upsertYomitanDictionarySettings: async () => true,
+    now: () => clock,
+    schedule: (fn) => {
+      scheduled.push(fn);
+      return 0 as unknown as ReturnType<typeof setTimeout>;
+    },
+    clearSchedule: () => undefined,
+    onSyncStatus: (event) => {
+      events.push({ phase: event.phase, message: event.message });
+    },
+  });
+
+  const syncPromise = runtime.runSyncNow();
+  await waitUntil(() => scheduled.length > 0, 'the generating heartbeat');
+
+  // No further progress arrives: only the clock moves.
+  clock += 95_000;
+  scheduled.at(-1)!();
+
+  assert.deepEqual(events.at(-1), {
+    phase: 'generating',
+    message: 'Generating character dictionary for ONE PIECE (image 240/1220 · 1m 35s)...',
+  });
+
+  snapshotDeferred.resolve({
+    mediaId: 21,
+    mediaTitle: 'ONE PIECE',
+    entryCount: 4000,
+    fromCache: false,
+    updatedAt: 1000,
+  });
+  await syncPromise;
+  assert.equal(events.at(-1)?.phase, 'ready');
+});
+
+test('auto sync rebuilds instead of importing a cached merged ZIP with a mismatched revision', async () => {
+  const userDataPath = makeTempDir();
+  const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
+  fs.mkdirSync(dictionariesDir, { recursive: true });
+  const statePath = path.join(dictionariesDir, 'auto-sync-state.json');
+  fs.writeFileSync(
+    statePath,
+    JSON.stringify({
+      activeMediaIds: ['7 - Frieren'],
+      mergedRevision: 'rev-7',
+      mergedDictionaryTitle: 'SubMiner Character Dictionary',
+    }),
+    'utf8',
+  );
+  // Left over from an interrupted run: the archive on disk is not the revision state recorded.
+  buildDictionaryZip(
+    path.join(dictionariesDir, 'merged.zip'),
+    'SubMiner Character Dictionary',
+    'Character names',
+    'rev-stale',
+    [{ term: 'フリーレン', reading: 'フリーレン', role: 'main', glossary: [] } as never],
+    [],
+  );
+  const mergedBuilds: number[][] = [];
+  const imports: string[] = [];
+
+  const runtime = createCharacterDictionaryAutoSyncRuntimeService({
+    userDataPath,
+    getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
+    getOrCreateCurrentSnapshot: async () => ({
+      mediaId: 7,
+      mediaTitle: 'Frieren',
+      entryCount: 100,
+      fromCache: true,
+      updatedAt: 1000,
+    }),
+    buildMergedDictionary: async (mediaIds) => {
+      mergedBuilds.push([...mediaIds]);
+      return {
+        zipPath: '/tmp/rebuilt-merged.zip',
+        revision: 'rev-7',
+        dictionaryTitle: 'SubMiner Character Dictionary',
+        entryCount: 100,
+      };
+    },
+    // Yomitan does not have the dictionary, so the sync has to import despite the cached state.
+    getYomitanDictionaryInfo: async () => [],
+    importYomitanDictionary: async (zipPath) => {
+      imports.push(zipPath);
+      return true;
+    },
+    deleteYomitanDictionary: async () => true,
+    upsertYomitanDictionarySettings: async () => true,
+    now: () => 1000,
+  });
+
+  await runtime.runSyncNow();
+
+  assert.deepEqual(mergedBuilds, [[7]]);
+  assert.deepEqual(imports, ['/tmp/rebuilt-merged.zip']);
+});