From da2c50b89fdde7cab01a88577936cf5b55a5076f Mon Sep 17 00:00:00 2001 From: sudacode Date: Fri, 7 Aug 2026 00:26:51 -0700 Subject: [PATCH] fix(dictionary): stop large character dictionaries from timing out - Size the Yomitan import/delete timeout off the merged ZIP (2min + 6s/MB, capped 30min) instead of the flat 7s query budget, so a big series like One Piece no longer fails mid-import - Report generation progress by stage (AniList page/character count, image N/total with ETA, name-split N/total, saving) instead of one static "Generating..." message - Add an elapsed-time heartbeat to generating/importing notifications, ticking every 5s even when a stage stalls, so a slow step reads as busy rather than hung --- changes/character-dictionary-large-imports.md | 7 + src/main/character-dictionary-runtime.test.ts | 2 +- src/main/character-dictionary-runtime.ts | 52 ++- .../character-dictionary-runtime/fetch.ts | 5 +- .../name-split-resolver.ts | 4 + .../character-dictionary-runtime/types.ts | 15 + .../character-dictionary-auto-sync.test.ts | 350 +++++++++++++++++- .../runtime/character-dictionary-auto-sync.ts | 346 ++++++++++++++--- 8 files changed, 709 insertions(+), 72 deletions(-) create mode 100644 changes/character-dictionary-large-imports.md diff --git a/changes/character-dictionary-large-imports.md b/changes/character-dictionary-large-imports.md new file mode 100644 index 00000000..4fe4e511 --- /dev/null +++ b/changes/character-dictionary-large-imports.md @@ -0,0 +1,7 @@ +type: fixed +area: character dictionary + +- A large character dictionary no longer fails to install. The Yomitan import (and the delete that precedes it) used a flat 7 second budget, so a series like One Piece failed with `importYomitanDictionary(merged.zip) timed out after 7000ms` while the import was still healthy. The budget is now sized from the merged ZIP (2 minutes plus 6 seconds per MB, capped at 30 minutes); the quick queries keep the 7 second budget. +- The "Generating character dictionary" notification now reports what it is doing instead of one static message for the whole run: the AniList page and character count while characters download (`page 12, 587 characters`), then `image 240/1220, ~4m left` per downloaded character/voice actor image, then `name 800/1220` for MeCab name splits and `saving snapshot` at the end. Updates are throttled to one per second, and every stage change and final item always reports. +- Every long phase also carries an elapsed clock (`· 1m 35s`), refreshed every 5 seconds even when nothing else moves, so a stalled step is visibly different from a frozen app. Long imports tick the same way. +- Image downloads are also logged every 100 files, and the import logs the timeout it computed. diff --git a/src/main/character-dictionary-runtime.test.ts b/src/main/character-dictionary-runtime.test.ts index 077835fd..13f01446 100644 --- a/src/main/character-dictionary-runtime.test.ts +++ b/src/main/character-dictionary-runtime.test.ts @@ -1903,7 +1903,7 @@ test('generateForCurrentMedia logs progress while resolving and rebuilding snaps '[dictionary] current anime guess: The Eminence in Shadow (episode 5)', '[dictionary] AniList match: The Eminence in Shadow -> AniList 130298', '[dictionary] snapshot miss for AniList 130298, fetching characters', - '[dictionary] downloaded AniList character page 1 for AniList 130298', + '[dictionary] downloaded AniList character page 1 for AniList 130298 (1 characters)', '[dictionary] downloading 1 images for AniList 130298', '[dictionary] stored snapshot for AniList 130298: 16 terms', '[dictionary] building ZIP for AniList 130298', diff --git a/src/main/character-dictionary-runtime.ts b/src/main/character-dictionary-runtime.ts index cad02577..56b71ecb 100644 --- a/src/main/character-dictionary-runtime.ts +++ b/src/main/character-dictionary-runtime.ts @@ -64,6 +64,7 @@ export type { CharacterDictionarySnapshotProgress, CharacterDictionarySnapshotProgressCallbacks, CharacterDictionarySnapshotResult, + CharacterDictionarySnapshotStageProgress, MergedCharacterDictionaryBuildResult, } from './character-dictionary-runtime/types'; @@ -363,19 +364,28 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar deps.logInfo?.(`[dictionary] snapshot stale for AniList ${mediaId}: ${refreshReason}`); } + const progressMediaTitle = mediaTitleHint || `AniList ${mediaId}`; progress?.onGenerating?.({ mediaId, - mediaTitle: mediaTitleHint || `AniList ${mediaId}`, + mediaTitle: progressMediaTitle, }); deps.logInfo?.(`[dictionary] snapshot miss for AniList ${mediaId}, fetching characters`); const { mediaTitle: fetchedMediaTitle, characters } = await fetchCharactersForMedia( mediaId, beforeRequest, - (page) => { + (page, charactersSoFar) => { deps.logInfo?.( - `[dictionary] downloaded AniList character page ${page} for AniList ${mediaId}`, + `[dictionary] downloaded AniList character page ${page} for AniList ${mediaId} (${charactersSoFar} characters)`, ); + progress?.onGenerateProgress?.({ + mediaId, + mediaTitle: progressMediaTitle, + stage: 'characters', + completed: charactersSoFar, + total: null, + page, + }); }, ); if (characters.length === 0) { @@ -403,12 +413,26 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar ); } let hasAttemptedImageDownload = false; + let attemptedImageCount = 0; for (const entry of allImageUrls) { if (hasAttemptedImageDownload) { await sleepMs(CHARACTER_IMAGE_DOWNLOAD_DELAY_MS); } hasAttemptedImageDownload = true; const image = await downloadCharacterImage(entry.url, entry.id); + attemptedImageCount += 1; + progress?.onGenerateProgress?.({ + mediaId, + mediaTitle: progressMediaTitle, + stage: 'images', + completed: attemptedImageCount, + total: allImageUrls.length, + }); + if (attemptedImageCount % 100 === 0) { + deps.logInfo?.( + `[dictionary] downloaded ${attemptedImageCount}/${allImageUrls.length} images for AniList ${mediaId}`, + ); + } if (!image) continue; if (entry.kind === 'character') { imagesByCharacterId.set(entry.id, { @@ -425,11 +449,31 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar const nameSplitTokenizerAvailable = isNameSplitTokenizerAvailable(); const resolvedNameSplits = nameSplitTokenizerAvailable - ? await resolveJapaneseNameSplits(characters, deps.tokenizeJapaneseName!, deps.logWarn) + ? await resolveJapaneseNameSplits( + characters, + deps.tokenizeJapaneseName!, + deps.logWarn, + (completed, total) => { + progress?.onGenerateProgress?.({ + mediaId, + mediaTitle: progressMediaTitle, + stage: 'names', + completed, + total, + }); + }, + ) : undefined; const nameSplitSource = resolvedNameSplits && resolvedNameSplits.size > 0 ? 'mecab' : 'heuristic'; + progress?.onGenerateProgress?.({ + mediaId, + mediaTitle: progressMediaTitle, + stage: 'saving', + completed: 0, + total: null, + }); const snapshot = buildSnapshotFromCharacters( mediaId, fetchedMediaTitle || mediaTitleHint || `AniList ${mediaId}`, diff --git a/src/main/character-dictionary-runtime/fetch.ts b/src/main/character-dictionary-runtime/fetch.ts index fe33c0f0..f7a28a44 100644 --- a/src/main/character-dictionary-runtime/fetch.ts +++ b/src/main/character-dictionary-runtime/fetch.ts @@ -278,7 +278,7 @@ export async function fetchAniListMediaCandidateById( export async function fetchCharactersForMedia( mediaId: number, beforeRequest?: () => Promise, - onPageFetched?: (page: number) => void, + onPageFetched?: (page: number, charactersSoFar: number) => void, ): Promise<{ mediaTitle: string; characters: CharacterRecord[]; @@ -345,7 +345,6 @@ export async function fetchCharactersForMedia( }, beforeRequest, ); - onPageFetched?.(page); const media = data.Media; if (!media) { @@ -415,6 +414,8 @@ export async function fetchCharactersForMedia( }); } + onPageFetched?.(page, characters.length); + const hasNextPage = Boolean(media.characters?.pageInfo?.hasNextPage); if (!hasNextPage) { break; diff --git a/src/main/character-dictionary-runtime/name-split-resolver.ts b/src/main/character-dictionary-runtime/name-split-resolver.ts index 63486a3e..a1acedda 100644 --- a/src/main/character-dictionary-runtime/name-split-resolver.ts +++ b/src/main/character-dictionary-runtime/name-split-resolver.ts @@ -86,8 +86,10 @@ export async function resolveJapaneseNameSplits( characters: CharacterRecord[], tokenize: NameSplitTokenizer, logWarn?: (message: string) => void, + onCharacterResolved?: (completed: number, total: number) => void, ): Promise> { const splits = new Map(); + let resolvedCharacters = 0; for (const character of characters) { const familyHintReading = buildReadingFromHint(character.lastNameHint?.trim() || ''); const givenHintReading = buildReadingFromHint(character.firstNameHint?.trim() || ''); @@ -113,6 +115,8 @@ export async function resolveJapaneseNameSplits( splits.set(name, { family, given }); } } + resolvedCharacters += 1; + onCharacterResolved?.(resolvedCharacters, characters.length); } return splits; } diff --git a/src/main/character-dictionary-runtime/types.ts b/src/main/character-dictionary-runtime/types.ts index 6c668604..8ca37553 100644 --- a/src/main/character-dictionary-runtime/types.ts +++ b/src/main/character-dictionary-runtime/types.ts @@ -122,9 +122,24 @@ export type CharacterDictionarySnapshotProgress = { mediaTitle: string; }; +export type CharacterDictionarySnapshotStage = 'characters' | 'images' | 'names' | 'saving'; + +/** + * Fine-grained generation progress. `total` is null while the work size is still unknown (AniList + * paginates characters, so the character count only settles on the last page). + */ +export type CharacterDictionarySnapshotStageProgress = CharacterDictionarySnapshotProgress & { + stage: CharacterDictionarySnapshotStage; + completed: number; + total: number | null; + /** AniList page currently being downloaded; only set during the `characters` stage. */ + page?: number; +}; + export type CharacterDictionarySnapshotProgressCallbacks = { onChecking?: (progress: CharacterDictionarySnapshotProgress) => void; onGenerating?: (progress: CharacterDictionarySnapshotProgress) => void; + onGenerateProgress?: (progress: CharacterDictionarySnapshotStageProgress) => void; }; export type MergedCharacterDictionaryBuildResult = { diff --git a/src/main/runtime/character-dictionary-auto-sync.test.ts b/src/main/runtime/character-dictionary-auto-sync.test.ts index f6302e0e..329af5dd 100644 --- a/src/main/runtime/character-dictionary-auto-sync.test.ts +++ b/src/main/runtime/character-dictionary-auto-sync.test.ts @@ -14,6 +14,14 @@ function makeTempDir(): string { return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-char-dict-auto-sync-')); } +async function waitUntil(predicate: () => boolean, label: string): Promise { + for (let attempt = 0; attempt < 500; attempt += 1) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 0)); + } + throw new Error(`Timed out waiting for ${label}`); +} + function createDeferred(): { promise: Promise; resolve: (value: T) => void } { let resolve!: (value: T) => void; const promise = new Promise((nextResolve) => { @@ -187,7 +195,7 @@ test('auto sync imports merged dictionary and persists MRU state', async () => { '[dictionary:auto-sync] syncing current anime snapshot', '[dictionary:auto-sync] active AniList media set: 130298 - The Eminence in Shadow', '[dictionary:auto-sync] rebuilding merged dictionary for active anime set', - '[dictionary:auto-sync] importing merged dictionary: /tmp/subminer-character-dictionary.zip', + '[dictionary:auto-sync] importing merged dictionary: /tmp/subminer-character-dictionary.zip (timeout 120000ms)', '[dictionary:auto-sync] applying Yomitan settings for SubMiner Character Dictionary', '[dictionary:auto-sync] synced AniList 130298: SubMiner Character Dictionary (2544 entries)', ]); @@ -958,11 +966,9 @@ test('auto sync emits building while merged dictionary generation is in flight', }); const syncPromise = runtime.runSyncNow(); - await Promise.resolve(); - - assert.equal( - events.some((event) => event.phase === 'building'), - true, + await waitUntil( + () => events.some((event) => event.phase === 'building'), + 'the building status event', ); buildDeferred.resolve({ @@ -1029,8 +1035,7 @@ test('auto sync waits for tokenization-ready gate before Yomitan mutations', asy }); const syncPromise = runtime.runSyncNow(); - await Promise.resolve(); - await Promise.resolve(); + await waitUntil(() => calls.includes('wait'), 'the tokenization-ready gate'); assert.deepEqual(calls, ['build', 'wait']); @@ -1039,3 +1044,332 @@ test('auto sync waits for tokenization-ready gate before Yomitan mutations', asy assert.deepEqual(calls, ['build', 'wait', 'info', 'import', 'settings']); }); + +test('auto sync scales the import timeout with the merged dictionary size', async () => { + const userDataPath = makeTempDir(); + const dictionariesDir = path.join(userDataPath, 'character-dictionaries'); + fs.mkdirSync(dictionariesDir, { recursive: true }); + const zipPath = path.join(dictionariesDir, 'merged.zip'); + // 2 MB of merged dictionary buys ~12s of import budget on top of the base. + fs.writeFileSync(zipPath, Buffer.alloc(2 * 1024 * 1024)); + const events: Array<{ phase: string; message: string }> = []; + let importedRevision: string | null = null; + + const runtime = createCharacterDictionaryAutoSyncRuntimeService({ + userDataPath, + getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }), + getOrCreateCurrentSnapshot: async () => ({ + mediaId: 21, + mediaTitle: 'ONE PIECE', + entryCount: 4000, + fromCache: false, + updatedAt: 1000, + }), + buildMergedDictionary: async () => ({ + zipPath, + revision: 'rev-21', + dictionaryTitle: 'SubMiner Character Dictionary', + entryCount: 4000, + }), + getYomitanDictionaryInfo: async () => + importedRevision + ? [{ title: 'SubMiner Character Dictionary', revision: importedRevision }] + : [], + importYomitanDictionary: async () => { + // Far longer than the quick-operation budget, well inside the size-scaled one. + await new Promise((resolve) => setTimeout(resolve, 60)); + importedRevision = 'rev-21'; + return true; + }, + deleteYomitanDictionary: async () => true, + upsertYomitanDictionarySettings: async () => true, + now: () => 1000, + operationTimeoutMs: 5, + dictionaryImportTimeoutBaseMs: 20, + onSyncStatus: (event) => { + events.push({ phase: event.phase, message: event.message }); + }, + }); + + await runtime.runSyncNow(); + + assert.equal( + events.some((event) => event.phase === 'failed'), + false, + ); + assert.deepEqual(events.at(-1), { + phase: 'ready', + message: 'Character dictionary ready for ONE PIECE', + }); +}); + +test('auto sync reports the scaled budget when an import really does hang', async () => { + const userDataPath = makeTempDir(); + const events: Array<{ phase: string; message: string }> = []; + + const runtime = createCharacterDictionaryAutoSyncRuntimeService({ + userDataPath, + getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }), + getOrCreateCurrentSnapshot: async () => ({ + mediaId: 21, + mediaTitle: 'ONE PIECE', + entryCount: 4000, + fromCache: false, + updatedAt: 1000, + }), + buildMergedDictionary: async () => ({ + zipPath: path.join(userDataPath, 'character-dictionaries', 'missing.zip'), + revision: 'rev-21', + dictionaryTitle: 'SubMiner Character Dictionary', + entryCount: 4000, + }), + getYomitanDictionaryInfo: async () => [], + importYomitanDictionary: () => new Promise(() => {}), + deleteYomitanDictionary: async () => true, + upsertYomitanDictionarySettings: async () => true, + now: () => 1000, + dictionaryImportTimeoutBaseMs: 20, + onSyncStatus: (event) => { + events.push({ phase: event.phase, message: event.message }); + }, + }); + + await assert.rejects( + runtime.runSyncNow(), + /importYomitanDictionary\(missing\.zip\) timed out after 20ms/, + ); + assert.equal(events.at(-1)?.phase, 'failed'); +}); + +test('auto sync ticks the importing notification while the import runs', async () => { + const userDataPath = makeTempDir(); + const events: Array<{ phase: string; message: string }> = []; + const scheduled: Array<() => void> = []; + const importDeferred = createDeferred(); + let clock = 1000; + + const runtime = createCharacterDictionaryAutoSyncRuntimeService({ + userDataPath, + getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }), + getOrCreateCurrentSnapshot: async () => ({ + mediaId: 21, + mediaTitle: 'ONE PIECE', + entryCount: 4000, + fromCache: false, + updatedAt: 1000, + }), + buildMergedDictionary: async () => ({ + zipPath: '/tmp/merged.zip', + revision: 'rev-21', + dictionaryTitle: 'SubMiner Character Dictionary', + entryCount: 4000, + }), + getYomitanDictionaryInfo: async () => [], + importYomitanDictionary: () => importDeferred.promise, + deleteYomitanDictionary: async () => true, + upsertYomitanDictionarySettings: async () => true, + now: () => clock, + schedule: (fn) => { + scheduled.push(fn); + return 0 as unknown as ReturnType; + }, + clearSchedule: () => undefined, + onSyncStatus: (event) => { + events.push({ phase: event.phase, message: event.message }); + }, + }); + + const syncPromise = runtime.runSyncNow(); + await waitUntil( + () => events.some((event) => event.phase === 'importing'), + 'the importing status event', + ); + + clock = 1000 + 65_000; + // The importing heartbeat is the most recently scheduled tick. + scheduled.at(-1)!(); + + assert.deepEqual(events.at(-1), { + phase: 'importing', + message: 'Importing character dictionary for ONE PIECE (1m 05s)...', + }); + + importDeferred.resolve(true); + await syncPromise; + assert.equal(events.at(-1)?.phase, 'ready'); +}); + +test('auto sync reports character and image counts while generating a snapshot', async () => { + const userDataPath = makeTempDir(); + const events: Array<{ phase: string; message: string }> = []; + let clock = 1000; + let importedRevision: string | null = null; + + const runtime = createCharacterDictionaryAutoSyncRuntimeService({ + userDataPath, + getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }), + getOrCreateCurrentSnapshot: async (_targetPath, progress) => { + progress?.onGenerating?.({ mediaId: 21, mediaTitle: 'ONE PIECE' }); + progress?.onGenerateProgress?.({ + mediaId: 21, + mediaTitle: 'ONE PIECE', + stage: 'characters', + completed: 50, + total: null, + page: 12, + }); + // Same stage, same clock tick: throttled away so a 33-page fetch cannot spam the overlay. + progress?.onGenerateProgress?.({ + mediaId: 21, + mediaTitle: 'ONE PIECE', + stage: 'characters', + completed: 100, + total: null, + page: 13, + }); + // A stage change always reports, throttle window or not. + progress?.onGenerateProgress?.({ + mediaId: 21, + mediaTitle: 'ONE PIECE', + stage: 'images', + completed: 1, + total: 1220, + }); + clock += 2000; + progress?.onGenerateProgress?.({ + mediaId: 21, + mediaTitle: 'ONE PIECE', + stage: 'images', + completed: 240, + total: 1220, + }); + clock += 6000; + progress?.onGenerateProgress?.({ + mediaId: 21, + mediaTitle: 'ONE PIECE', + stage: 'names', + completed: 800, + total: 1220, + }); + progress?.onGenerateProgress?.({ + mediaId: 21, + mediaTitle: 'ONE PIECE', + stage: 'saving', + completed: 0, + total: null, + }); + return { + mediaId: 21, + mediaTitle: 'ONE PIECE', + entryCount: 4000, + fromCache: false, + updatedAt: 1000, + }; + }, + buildMergedDictionary: async () => ({ + zipPath: '/tmp/merged.zip', + revision: 'rev-21', + dictionaryTitle: 'SubMiner Character Dictionary', + entryCount: 4000, + }), + getYomitanDictionaryInfo: async () => + importedRevision + ? [{ title: 'SubMiner Character Dictionary', revision: importedRevision }] + : [], + importYomitanDictionary: async () => { + importedRevision = 'rev-21'; + return true; + }, + deleteYomitanDictionary: async () => true, + upsertYomitanDictionarySettings: async () => true, + now: () => clock, + onSyncStatus: (event) => { + events.push({ phase: event.phase, message: event.message }); + }, + }); + + await runtime.runSyncNow(); + + assert.deepEqual( + events.filter((event) => event.phase === 'generating').map((event) => event.message), + [ + 'Generating character dictionary for ONE PIECE...', + 'Generating character dictionary for ONE PIECE (page 12, 50 characters)...', + 'Generating character dictionary for ONE PIECE (image 1/1220)...', + 'Generating character dictionary for ONE PIECE (image 240/1220, ~10s left)...', + 'Generating character dictionary for ONE PIECE (name 800/1220 · 8s)...', + 'Generating character dictionary for ONE PIECE (saving snapshot · 8s)...', + ], + ); +}); + +test('auto sync keeps the generating clock ticking when a stage stalls', async () => { + const userDataPath = makeTempDir(); + const events: Array<{ phase: string; message: string }> = []; + const scheduled: Array<() => void> = []; + const snapshotDeferred = createDeferred<{ + mediaId: number; + mediaTitle: string; + entryCount: number; + fromCache: boolean; + updatedAt: number; + }>(); + let clock = 1000; + + const runtime = createCharacterDictionaryAutoSyncRuntimeService({ + userDataPath, + getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }), + getOrCreateCurrentSnapshot: async (_targetPath, progress) => { + progress?.onGenerating?.({ mediaId: 21, mediaTitle: 'ONE PIECE' }); + progress?.onGenerateProgress?.({ + mediaId: 21, + mediaTitle: 'ONE PIECE', + stage: 'images', + completed: 240, + total: 1220, + }); + return await snapshotDeferred.promise; + }, + buildMergedDictionary: async () => ({ + zipPath: '/tmp/merged.zip', + revision: 'rev-21', + dictionaryTitle: 'SubMiner Character Dictionary', + entryCount: 4000, + }), + getYomitanDictionaryInfo: async () => [], + importYomitanDictionary: async () => true, + deleteYomitanDictionary: async () => true, + upsertYomitanDictionarySettings: async () => true, + now: () => clock, + schedule: (fn) => { + scheduled.push(fn); + return 0 as unknown as ReturnType; + }, + clearSchedule: () => undefined, + onSyncStatus: (event) => { + events.push({ phase: event.phase, message: event.message }); + }, + }); + + const syncPromise = runtime.runSyncNow(); + await waitUntil(() => scheduled.length > 0, 'the generating heartbeat'); + + // No further progress arrives: only the clock moves. + clock += 95_000; + scheduled.at(-1)!(); + + assert.deepEqual(events.at(-1), { + phase: 'generating', + message: 'Generating character dictionary for ONE PIECE (image 240/1220 · 1m 35s)...', + }); + + snapshotDeferred.resolve({ + mediaId: 21, + mediaTitle: 'ONE PIECE', + entryCount: 4000, + fromCache: false, + updatedAt: 1000, + }); + await syncPromise; + assert.equal(events.at(-1)?.phase, 'ready'); +}); diff --git a/src/main/runtime/character-dictionary-auto-sync.ts b/src/main/runtime/character-dictionary-auto-sync.ts index a51e8d91..7801128a 100644 --- a/src/main/runtime/character-dictionary-auto-sync.ts +++ b/src/main/runtime/character-dictionary-auto-sync.ts @@ -5,9 +5,14 @@ import type { AnilistCharacterDictionaryProfileScope } from '../../types'; import type { CharacterDictionarySnapshotProgressCallbacks, CharacterDictionarySnapshotResult, + CharacterDictionarySnapshotStageProgress, MergedCharacterDictionaryBuildResult, } from '../character-dictionary-runtime'; +const DEFAULT_IMPORT_TIMEOUT_BASE_MS = 120_000; +const IMPORT_TIMEOUT_MS_PER_MB = 6_000; +const IMPORT_TIMEOUT_MAX_MS = 1_800_000; + type AutoSyncMediaEntry = { mediaId: number; label: string; @@ -72,7 +77,15 @@ export interface CharacterDictionaryAutoSyncRuntimeDeps { now: () => number; schedule?: (fn: () => void, delayMs: number) => ReturnType; clearSchedule?: (timer: ReturnType) => void; + /** Budget for the quick Yomitan queries (dictionary info, settings upsert). */ operationTimeoutMs?: number; + /** + * Base budget for the slow Yomitan mutations (delete + import). The effective budget grows with + * the merged ZIP size, because importing a large dictionary can take several minutes. + */ + dictionaryImportTimeoutBaseMs?: number; + heartbeatMs?: number; + progressThrottleMs?: number; logInfo?: (message: string) => void; logWarn?: (message: string) => void; onSyncStatus?: (event: CharacterDictionaryAutoSyncStatusEvent) => void; @@ -353,12 +366,70 @@ function buildCheckingMessage(mediaTitle: string): string { return `Checking character dictionary for ${mediaTitle}...`; } -function buildGeneratingMessage(mediaTitle: string): string { - return `Generating character dictionary for ${mediaTitle}...`; +function buildGeneratingMessage(mediaTitle: string, detail?: string): string { + return detail + ? `Generating character dictionary for ${mediaTitle} (${detail})...` + : `Generating character dictionary for ${mediaTitle}...`; } -function buildImportingMessage(mediaTitle: string): string { - return `Importing character dictionary for ${mediaTitle}...`; +function formatCharacterDictionaryProgressDetail( + progress: CharacterDictionarySnapshotStageProgress, + remainingMs: number | null, +): string { + if (progress.stage === 'saving') { + return 'saving snapshot'; + } + if (progress.stage === 'names') { + return progress.total !== null && progress.total > 0 + ? `name ${progress.completed}/${progress.total}` + : `${progress.completed} names`; + } + if (progress.stage === 'images') { + const counted = + progress.total !== null && progress.total > 0 + ? `image ${progress.completed}/${progress.total}` + : `${progress.completed} images`; + return remainingMs !== null + ? `${counted}, ~${formatRemainingDuration(remainingMs)} left` + : counted; + } + const page = typeof progress.page === 'number' ? `page ${progress.page}, ` : ''; + return `${page}${progress.completed} characters`; +} + +function formatElapsedDuration(elapsedMs: number): string { + const totalSeconds = Math.max(0, Math.floor(elapsedMs / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return minutes > 0 ? `${minutes}m ${String(seconds).padStart(2, '0')}s` : `${seconds}s`; +} + +/** Coarser than the elapsed clock: an estimate that ticks every second reads as precision it lacks. */ +function formatRemainingDuration(remainingMs: number): string { + const totalSeconds = Math.max(1, Math.round(remainingMs / 1000)); + if (totalSeconds >= 60) { + return `${Math.max(1, Math.round(totalSeconds / 60))}m`; + } + return `${Math.max(5, Math.ceil(totalSeconds / 5) * 5)}s`; +} + +/** + * The elapsed clock is the part that proves the app is alive: a stalled network fetch freezes the + * counts, but the clock keeps moving. + */ +function joinGeneratingDetail(detail: string | null, elapsedMs: number): string | undefined { + const parts = [detail, elapsedMs >= 5_000 ? formatElapsedDuration(elapsedMs) : null].filter( + (part): part is string => typeof part === 'string' && part.length > 0, + ); + return parts.length > 0 ? parts.join(' · ') : undefined; +} + +function buildImportingMessage(mediaTitle: string, elapsedMs?: number): string { + const elapsed = + typeof elapsedMs === 'number' && elapsedMs >= 1000 + ? ` (${formatElapsedDuration(elapsedMs)})` + : ''; + return `Importing character dictionary for ${mediaTitle}${elapsed}...`; } function buildBuildingMessage(mediaTitle: string): string { @@ -389,21 +460,31 @@ export function createCharacterDictionaryAutoSyncRuntimeService( const clearSchedule = deps.clearSchedule ?? ((timer) => clearTimeout(timer)); const debounceMs = 800; const operationTimeoutMs = Math.max(1, Math.floor(deps.operationTimeoutMs ?? 7_000)); + const dictionaryImportTimeoutBaseMs = Math.max( + 1, + Math.floor(deps.dictionaryImportTimeoutBaseMs ?? DEFAULT_IMPORT_TIMEOUT_BASE_MS), + ); + const heartbeatMs = Math.max(1, Math.floor(deps.heartbeatMs ?? 5_000)); + const progressThrottleMs = Math.max(0, Math.floor(deps.progressThrottleMs ?? 1_000)); let debounceTimer: ReturnType | null = null; let syncInFlight = false; let runQueued = false; let activeCurrentMediaId: number | null = null; - const withOperationTimeout = async (label: string, promise: Promise): Promise => { + const withTimeout = async ( + label: string, + promise: Promise, + timeoutMs: number, + ): Promise => { let timer: ReturnType | null = null; try { return await Promise.race([ promise, new Promise((_resolve, reject) => { timer = setTimeout(() => { - reject(new Error(`${label} timed out after ${operationTimeoutMs}ms`)); - }, operationTimeoutMs); + reject(new Error(`${label} timed out after ${timeoutMs}ms`)); + }, timeoutMs); }), ]); } finally { @@ -413,6 +494,57 @@ export function createCharacterDictionaryAutoSyncRuntimeService( } }; + const withOperationTimeout = (label: string, promise: Promise): Promise => + withTimeout(label, promise, operationTimeoutMs); + + /** + * Importing a merged dictionary means Yomitan writing every term and image into IndexedDB, which + * scales with the ZIP: a single-season dictionary lands in seconds, One Piece takes minutes. Size + * the budget off the archive instead of failing a healthy import on a flat deadline. + */ + const resolveImportTimeoutMs = (zipPath: string | null): number => { + let bytes = 0; + if (zipPath) { + try { + bytes = fs.statSync(zipPath).size; + } catch { + bytes = 0; + } + } + const sizeAllowanceMs = (bytes / (1024 * 1024)) * IMPORT_TIMEOUT_MS_PER_MB; + return Math.min( + IMPORT_TIMEOUT_MAX_MS, + Math.max( + dictionaryImportTimeoutBaseMs, + Math.round(dictionaryImportTimeoutBaseMs + sizeAllowanceMs), + ), + ); + }; + + /** Keeps the persistent notification ticking so a multi-minute import never looks hung. */ + const withHeartbeat = async ( + run: () => Promise, + onTick: (elapsedMs: number) => void, + ): Promise => { + const startedAt = deps.now(); + let stopped = false; + let timer: ReturnType | null = null; + const tick = (): void => { + if (stopped) return; + onTick(deps.now() - startedAt); + timer = schedule(tick, heartbeatMs); + }; + timer = schedule(tick, heartbeatMs); + try { + return await run(); + } finally { + stopped = true; + if (timer !== null) { + clearSchedule(timer); + } + } + }; + const runSyncOnce = async (): Promise => { const config = deps.getConfig(); if (!config.enabled) { @@ -422,33 +554,112 @@ export function createCharacterDictionaryAutoSyncRuntimeService( let currentMediaId: number | undefined; let currentMediaTitle: string | null = null; + let lastProgressAt = Number.NEGATIVE_INFINITY; + let lastProgressStage: CharacterDictionarySnapshotStageProgress['stage'] | null = null; + let generating: { + mediaId: number; + mediaTitle: string; + startedAt: number; + detail: string | null; + } | null = null; + let imageRate: { startedAt: number; startCompleted: number } | null = null; + + const emitGeneratingStatus = (): void => { + if (!generating) { + return; + } + deps.onSyncStatus?.({ + phase: 'generating', + mediaId: generating.mediaId, + mediaTitle: generating.mediaTitle, + message: buildGeneratingMessage( + generating.mediaTitle, + joinGeneratingDetail(generating.detail, deps.now() - generating.startedAt), + ), + }); + }; + + // Image downloads are serial and evenly paced, so the observed rate predicts the tail well. + const estimateRemainingImageMs = ( + progress: CharacterDictionarySnapshotStageProgress, + nowMs: number, + ): number | null => { + if (progress.stage !== 'images' || progress.total === null || imageRate === null) { + return null; + } + const done = progress.completed - imageRate.startCompleted; + const elapsedMs = nowMs - imageRate.startedAt; + const remaining = progress.total - progress.completed; + if (done <= 0 || elapsedMs <= 0 || remaining <= 0) { + return null; + } + return Math.round((elapsedMs / done) * remaining); + }; try { deps.logInfo?.('[dictionary:auto-sync] syncing current anime snapshot'); - const snapshot = await deps.getOrCreateCurrentSnapshot(undefined, { - onChecking: ({ mediaId, mediaTitle }) => { - currentMediaId = mediaId; - currentMediaTitle = mediaTitle; - activeCurrentMediaId = mediaId; - deps.onSyncStatus?.({ - phase: 'checking', - mediaId, - mediaTitle, - message: buildCheckingMessage(mediaTitle), - }); - }, - onGenerating: ({ mediaId, mediaTitle }) => { - currentMediaId = mediaId; - currentMediaTitle = mediaTitle; - activeCurrentMediaId = mediaId; - deps.onSyncStatus?.({ - phase: 'generating', - mediaId, - mediaTitle, - message: buildGeneratingMessage(mediaTitle), - }); - }, - }); + const snapshot = await withHeartbeat( + () => + deps.getOrCreateCurrentSnapshot(undefined, { + onChecking: ({ mediaId, mediaTitle }) => { + currentMediaId = mediaId; + currentMediaTitle = mediaTitle; + activeCurrentMediaId = mediaId; + deps.onSyncStatus?.({ + phase: 'checking', + mediaId, + mediaTitle, + message: buildCheckingMessage(mediaTitle), + }); + }, + onGenerating: ({ mediaId, mediaTitle }) => { + currentMediaId = mediaId; + currentMediaTitle = mediaTitle; + activeCurrentMediaId = mediaId; + lastProgressAt = Number.NEGATIVE_INFINITY; + lastProgressStage = null; + imageRate = null; + generating = { mediaId, mediaTitle, startedAt: deps.now(), detail: null }; + emitGeneratingStatus(); + }, + // Long-running work (AniList character pages, then one image download per character + // and voice actor, then MeCab name splits) reports counts so the notification shows + // movement instead of a frozen "Generating..." for the minutes a large series takes. + onGenerateProgress: (progress) => { + const nowMs = deps.now(); + if (!generating) { + generating = { + mediaId: progress.mediaId, + mediaTitle: progress.mediaTitle, + startedAt: nowMs, + detail: null, + }; + } + if (progress.stage === 'images' && imageRate === null) { + imageRate = { startedAt: nowMs, startCompleted: progress.completed }; + } + // Stage changes and the last item of a stage always report; the throttle only thins + // out the run of identical-looking updates in between. + const isFinal = progress.total !== null && progress.completed >= progress.total; + const isStageChange = progress.stage !== lastProgressStage; + if (!isFinal && !isStageChange && nowMs - lastProgressAt < progressThrottleMs) { + return; + } + lastProgressAt = nowMs; + lastProgressStage = progress.stage; + generating.mediaId = progress.mediaId; + generating.mediaTitle = progress.mediaTitle; + generating.detail = formatCharacterDictionaryProgressDetail( + progress, + estimateRemainingImageMs(progress, nowMs), + ); + emitGeneratingStatus(); + }, + }), + // Ticks even when a step stalls, so the message keeps moving while the counts do not. + () => emitGeneratingStatus(), + ); + generating = null; currentMediaId = snapshot.mediaId; currentMediaTitle = snapshot.mediaTitle; activeCurrentMediaId = snapshot.mediaId; @@ -531,33 +742,54 @@ export function createCharacterDictionaryAutoSyncRuntimeService( mediaTitle: snapshot.mediaTitle, message: buildImportingMessage(snapshot.mediaTitle), }); - if (existing !== null) { - await withOperationTimeout( - `deleteYomitanDictionary(${dictionaryTitle})`, - deps.deleteYomitanDictionary(dictionaryTitle), - ); - } - if (merged === null) { - const existingMergedZipPath = path.join(dictionariesDir, 'merged.zip'); - if (fs.existsSync(existingMergedZipPath)) { - merged = { - zipPath: existingMergedZipPath, - revision, - dictionaryTitle, - entryCount: snapshot.entryCount, - }; - } else { - merged = await deps.buildMergedDictionary(nextActiveMediaIdValues); - } - } - deps.logInfo?.(`[dictionary:auto-sync] importing merged dictionary: ${merged.zipPath}`); - const imported = await withOperationTimeout( - `importYomitanDictionary(${path.basename(merged.zipPath)})`, - deps.importYomitanDictionary(merged.zipPath), + await withHeartbeat( + async () => { + const importTimeoutMs = resolveImportTimeoutMs( + merged?.zipPath ?? path.join(dictionariesDir, 'merged.zip'), + ); + if (existing !== null) { + await withTimeout( + `deleteYomitanDictionary(${dictionaryTitle})`, + deps.deleteYomitanDictionary(dictionaryTitle), + importTimeoutMs, + ); + } + if (merged === null) { + const existingMergedZipPath = path.join(dictionariesDir, 'merged.zip'); + if (fs.existsSync(existingMergedZipPath)) { + merged = { + zipPath: existingMergedZipPath, + revision, + dictionaryTitle, + entryCount: snapshot.entryCount, + }; + } else { + merged = await deps.buildMergedDictionary(nextActiveMediaIdValues); + } + } + const mergedZipPath = merged.zipPath; + const mergedImportTimeoutMs = resolveImportTimeoutMs(mergedZipPath); + deps.logInfo?.( + `[dictionary:auto-sync] importing merged dictionary: ${mergedZipPath} (timeout ${mergedImportTimeoutMs}ms)`, + ); + const imported = await withTimeout( + `importYomitanDictionary(${path.basename(mergedZipPath)})`, + deps.importYomitanDictionary(mergedZipPath), + mergedImportTimeoutMs, + ); + if (!imported) { + throw new Error(`Failed to import dictionary ZIP: ${merged.zipPath}`); + } + }, + (elapsedMs) => { + deps.onSyncStatus?.({ + phase: 'importing', + mediaId: snapshot.mediaId, + mediaTitle: snapshot.mediaTitle, + message: buildImportingMessage(snapshot.mediaTitle, elapsedMs), + }); + }, ); - if (!imported) { - throw new Error(`Failed to import dictionary ZIP: ${merged.zipPath}`); - } changed = true; }