fix(dictionary): stop large character dictionaries from timing out (#189)

src/main/character-dictionary-runtime/fetch.ts

@@ -278,7 +278,7 @@ export async function fetchAniListMediaCandidateById(
 export async function fetchCharactersForMedia(
   mediaId: number,
   beforeRequest?: () => Promise<void>,
-  onPageFetched?: (page: number) => void,
+  onPageFetched?: (page: number, charactersSoFar: number) => void,
 ): Promise<{
   mediaTitle: string;
   characters: CharacterRecord[];
@@ -345,7 +345,6 @@ export async function fetchCharactersForMedia(
       },
       beforeRequest,
     );
-    onPageFetched?.(page);
 
     const media = data.Media;
     if (!media) {
@@ -415,6 +414,8 @@ export async function fetchCharactersForMedia(
       });
     }
 
+    onPageFetched?.(page, characters.length);
+
     const hasNextPage = Boolean(media.characters?.pageInfo?.hasNextPage);
     if (!hasNextPage) {
       break;

src/main/character-dictionary-runtime/name-split-resolver.ts

@@ -86,8 +86,10 @@ export async function resolveJapaneseNameSplits(
   characters: CharacterRecord[],
   tokenize: NameSplitTokenizer,
   logWarn?: (message: string) => void,
+  onCharacterResolved?: (completed: number, total: number) => void,
 ): Promise<Map<string, ResolvedNameSplit>> {
   const splits = new Map<string, ResolvedNameSplit>();
+  let resolvedCharacters = 0;
   for (const character of characters) {
     const familyHintReading = buildReadingFromHint(character.lastNameHint?.trim() || '');
     const givenHintReading = buildReadingFromHint(character.firstNameHint?.trim() || '');
@@ -113,6 +115,8 @@ export async function resolveJapaneseNameSplits(
         splits.set(name, { family, given });
       }
     }
+    resolvedCharacters += 1;
+    onCharacterResolved?.(resolvedCharacters, characters.length);
   }
   return splits;
 }

src/main/character-dictionary-runtime/types.ts

@@ -122,9 +122,24 @@ export type CharacterDictionarySnapshotProgress = {
   mediaTitle: string;
 };
 
+export type CharacterDictionarySnapshotStage = 'characters' | 'images' | 'names' | 'saving';
+
+/**
+ * Fine-grained generation progress. `total` is null while the work size is still unknown (AniList
+ * paginates characters, so the character count only settles on the last page).
+ */
+export type CharacterDictionarySnapshotStageProgress = CharacterDictionarySnapshotProgress & {
+  stage: CharacterDictionarySnapshotStage;
+  completed: number;
+  total: number | null;
+  /** AniList page currently being downloaded; only set during the `characters` stage. */
+  page?: number;
+};
+
 export type CharacterDictionarySnapshotProgressCallbacks = {
   onChecking?: (progress: CharacterDictionarySnapshotProgress) => void;
   onGenerating?: (progress: CharacterDictionarySnapshotProgress) => void;
+  onGenerateProgress?: (progress: CharacterDictionarySnapshotStageProgress) => void;
 };
 
 export type MergedCharacterDictionaryBuildResult = {

src/main/character-dictionary-runtime/zip.test.ts

@@ -3,7 +3,7 @@ import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import test from 'node:test';
-import { buildDictionaryZip } from './zip';
+import { buildDictionaryZip, readDictionaryZipRevision } from './zip';
 import type { CharacterDictionaryTermEntry } from './types';
 
 function makeTempDir(): string {
@@ -105,3 +105,63 @@ test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', ()
     cleanupDir(tempDir);
   }
 });
+
+test('readDictionaryZipRevision reads the built revision and rejects foreign archives', () => {
+  const dir = makeTempDir();
+  try {
+    const zipPath = path.join(dir, 'merged.zip');
+    buildDictionaryZip(
+      zipPath,
+      'SubMiner Character Dictionary',
+      'Character names',
+      'rev-42',
+      [
+        {
+          term: 'ルフィ',
+          reading: 'ルフィ',
+          role: 'main',
+          glossary: [],
+        } as unknown as CharacterDictionaryTermEntry,
+      ],
+      [],
+    );
+
+    assert.equal(readDictionaryZipRevision(zipPath), 'rev-42');
+    assert.equal(readDictionaryZipRevision(path.join(dir, 'missing.zip')), null);
+
+    const archive = fs.readFileSync(zipPath);
+    const truncatedPath = path.join(dir, 'truncated.zip');
+    fs.writeFileSync(truncatedPath, archive.subarray(0, 40));
+    assert.equal(readDictionaryZipRevision(truncatedPath), null);
+
+    // An archive cut short after index.json still holds a readable revision, but importing it
+    // would hand Yomitan a half-written file: the missing end-of-central-directory record has to
+    // reject it. One byte off the end is enough to make the record incomplete.
+    for (const missingBytes of [1, 22, archive.length - 200]) {
+      const cutPath = path.join(dir, `cut-${missingBytes}.zip`);
+      fs.writeFileSync(cutPath, archive.subarray(0, archive.length - missingBytes));
+      assert.equal(readDictionaryZipRevision(cutPath), null, `cut of ${missingBytes} bytes`);
+    }
+
+    // Same size, corrupt directory: a record overwritten in place has to be rejected too.
+    const centralStart = archive.readUInt32LE(archive.length - 22 + 16);
+    const brokenSignaturePath = path.join(dir, 'broken-signature.zip');
+    const brokenSignature = Buffer.from(archive);
+    brokenSignature.writeUInt32LE(0xdeadbeef, centralStart);
+    fs.writeFileSync(brokenSignaturePath, brokenSignature);
+    assert.equal(readDictionaryZipRevision(brokenSignaturePath), null);
+
+    const brokenLengthPath = path.join(dir, 'broken-length.zip');
+    const brokenLength = Buffer.from(archive);
+    // Name length that runs the walk past the end of the directory.
+    brokenLength.writeUInt16LE(0xffff, centralStart + 28);
+    fs.writeFileSync(brokenLengthPath, brokenLength);
+    assert.equal(readDictionaryZipRevision(brokenLengthPath), null);
+
+    const foreignPath = path.join(dir, 'foreign.zip');
+    fs.writeFileSync(foreignPath, Buffer.from('not a zip at all', 'utf8'));
+    assert.equal(readDictionaryZipRevision(foreignPath), null);
+  } finally {
+    cleanupDir(dir);
+  }
+});

src/main/character-dictionary-runtime/zip.ts

@@ -1,5 +1,5 @@
 import * as path from 'path';
-import { writeStoredZip } from '../../shared/stored-zip';
+import { readStoredZipFirstFile, writeStoredZip } from '../../shared/stored-zip';
 import { ensureDir } from './fs-utils';
 import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } from './types';
 
@@ -31,6 +31,23 @@ function createTagBank(): Array<[string, string, number, string, number]> {
   ];
 }
 
+/**
+ * Revision recorded inside a built dictionary ZIP, or null when the archive is missing, truncated,
+ * or not one of ours. `index.json` is always the first entry written by {@link buildDictionaryZip}.
+ */
+export function readDictionaryZipRevision(zipPath: string): string | null {
+  const firstFile = readStoredZipFirstFile(zipPath);
+  if (!firstFile || firstFile.name !== 'index.json') {
+    return null;
+  }
+  try {
+    const index = JSON.parse(firstFile.data.toString('utf8')) as { revision?: unknown };
+    return typeof index.revision === 'string' && index.revision.length > 0 ? index.revision : null;
+  } catch {
+    return null;
+  }
+}
+
 export function buildDictionaryZip(
   outputPath: string,
   dictionaryTitle: string,