From 20797772f8ace8d342ea5233ffd872a44287d80b Mon Sep 17 00:00:00 2001 From: sudacode Date: Wed, 19 Aug 2026 01:49:15 -0700 Subject: [PATCH] fix(character-dictionary): cache completed MeCab refreshes (#212) --- changes/character-dictionary-mecab-cache.md | 5 ++ src/main.ts | 16 +++-- src/main/character-dictionary-runtime.ts | 6 +- .../image-lookup.test.ts | 60 +++++++++++++++++++ .../image-lookup.ts | 23 +++++++ .../name-split-resolver.test.ts | 18 ++++-- .../name-split-resolver.ts | 16 ++++- .../snapshot-refresh.test.ts | 59 +++++++++++++----- src/main/main-wiring.test.ts | 18 +++++- 9 files changed, 186 insertions(+), 35 deletions(-) create mode 100644 changes/character-dictionary-mecab-cache.md diff --git a/changes/character-dictionary-mecab-cache.md b/changes/character-dictionary-mecab-cache.md new file mode 100644 index 00000000..5426df89 --- /dev/null +++ b/changes/character-dictionary-mecab-cache.md @@ -0,0 +1,5 @@ +type: fixed +area: character dictionary + +- Reuse character dictionaries after MeCab completes without finding any name splits instead of regenerating character data and portraits on every launch. +- Restore inline character portraits when a cached portrait index finishes loading after subtitles have already been tokenized. diff --git a/src/main.ts b/src/main.ts index 423d553e..45de28b3 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2623,6 +2623,12 @@ const characterDictionaryAutoSyncRuntime = createCharacterDictionaryAutoSyncRunt const characterDictionaryImageLookup = createCharacterDictionaryImageLookup({ userDataPath: USER_DATA_PATH, getCurrentMediaId: () => characterDictionaryAutoSyncRuntime.getCurrentMediaId(), + onIndexReady: () => refreshCurrentSubtitleAnnotations(), + onIndexReadyError: (error) => + logger.warn( + 'Failed to refresh subtitle annotations after character portrait index became ready.', + error, + ), }); // Lets the Yomitan scan runtime skip name lookups at positions where no @@ -4047,7 +4053,7 @@ const recordTrackedCardsMined = (count: number, noteIds?: number[]): void => { ensureImmersionTrackerStarted(); appState.immersionTracker?.recordCardsMined(count, noteIds); }; -const refreshCurrentSubtitleAfterKnownWordUpdate = (): void => { +function refreshCurrentSubtitleAnnotations(): void { const hasCurrentSubtitle = appState.currentSubText.trim().length > 0; if (hasCurrentSubtitle) { subtitlePrefetchService?.pause(); @@ -4058,7 +4064,7 @@ const refreshCurrentSubtitleAfterKnownWordUpdate = (): void => { // Idle controller: no settle is coming to release the pause above. subtitlePrefetchService?.resume(); } -}; +} let hasAttemptedImmersionTrackerStartup = false; const ensureImmersionTrackerStarted = (): void => { if (hasAttemptedImmersionTrackerStartup || appState.immersionTracker) { @@ -5081,9 +5087,7 @@ function initializeOverlayRuntime(): void { overlayModalRuntime.primeModalWindow(); } appState.ankiIntegration?.setRecordCardsMinedCallback(recordTrackedCardsMined); - appState.ankiIntegration?.setKnownWordCacheUpdatedCallback( - refreshCurrentSubtitleAfterKnownWordUpdate, - ); + appState.ankiIntegration?.setKnownWordCacheUpdatedCallback(refreshCurrentSubtitleAnnotations); appState.ankiIntegration?.setSubtitleMiningContextConsumer(consumePendingSubtitleMiningContext); syncOverlayMpvSubtitleSuppression(); } @@ -5876,7 +5880,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({ appState.ankiIntegration = integration; appState.ankiIntegration?.setRecordCardsMinedCallback(recordTrackedCardsMined); appState.ankiIntegration?.setKnownWordCacheUpdatedCallback( - refreshCurrentSubtitleAfterKnownWordUpdate, + refreshCurrentSubtitleAnnotations, ); appState.ankiIntegration?.setSubtitleMiningContextConsumer( consumePendingSubtitleMiningContext, diff --git a/src/main/character-dictionary-runtime.ts b/src/main/character-dictionary-runtime.ts index c362d80b..1186775b 100644 --- a/src/main/character-dictionary-runtime.ts +++ b/src/main/character-dictionary-runtime.ts @@ -450,7 +450,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar } const nameSplitTokenizerAvailable = isNameSplitTokenizerAvailable(); - const resolvedNameSplits = nameSplitTokenizerAvailable + const nameSplitResolution = nameSplitTokenizerAvailable ? await resolveJapaneseNameSplits( characters, deps.tokenizeJapaneseName!, @@ -466,8 +466,8 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar }, ) : undefined; - const nameSplitSource = - resolvedNameSplits && resolvedNameSplits.size > 0 ? 'mecab' : 'heuristic'; + const resolvedNameSplits = nameSplitResolution?.splits; + const nameSplitSource = nameSplitResolution?.kind === 'complete' ? 'mecab' : 'heuristic'; progress?.onGenerateProgress?.({ mediaId, diff --git a/src/main/character-dictionary-runtime/image-lookup.test.ts b/src/main/character-dictionary-runtime/image-lookup.test.ts index aadaf39a..c5b84f75 100644 --- a/src/main/character-dictionary-runtime/image-lookup.test.ts +++ b/src/main/character-dictionary-runtime/image-lookup.test.ts @@ -198,6 +198,66 @@ test('createCharacterDictionaryImageLookup can scope duplicate names to the curr assert.equal(scoped.alt, 'Kazuma'); }); +test('createCharacterDictionaryImageLookup reports and retries a failed index-ready callback', async () => { + const outputDir = makeTempDir(); + const snapshot: CharacterDictionarySnapshot = { + formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION, + mediaId: 21858, + mediaTitle: 'Little Witch Academia', + entryCount: 1, + updatedAt: 1_700_000_000_000, + termEntries: [ + [ + 'ダイアナ', + 'だいあな', + 'name primary', + '', + 75, + [ + { + type: 'structured-content', + content: { + tag: 'img', + path: 'img/m21858-c81709.png', + alt: 'ダイアナ・キャベンディッシュ', + }, + }, + ], + 0, + '', + ], + ], + images: [{ path: 'img/m21858-c81709.png', dataBase64: PNG_1X1_BASE64 }], + }; + await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot); + const callbackError = new Error('annotation refresh failed'); + const reportingError = new Error('error reporter failed'); + let readyCount = 0; + const reportedErrors: unknown[] = []; + const lookup = createCharacterDictionaryImageLookup({ + outputDir, + onIndexReady: () => { + readyCount += 1; + if (readyCount === 1) { + throw callbackError; + } + }, + onIndexReadyError: (error) => { + reportedErrors.push(error); + throw reportingError; + }, + }); + + assert.equal(lookup.get('ダイアナ', snapshot.mediaId), null); + await waitForRefresh(() => (reportedErrors.length === 1 ? true : null)); + assert.ok(lookup.get('ダイアナ', snapshot.mediaId)); + + assert.equal(readyCount, 2); + assert.deepEqual(reportedErrors, [callbackError]); + lookup.get('ダイアナ', snapshot.mediaId); + assert.equal(readyCount, 2); +}); + test('createCharacterDictionaryImageLookup does not fall back globally on scoped miss', async () => { const outputDir = makeTempDir(); const snapshot: CharacterDictionarySnapshot = { diff --git a/src/main/character-dictionary-runtime/image-lookup.ts b/src/main/character-dictionary-runtime/image-lookup.ts index d21e3020..e10e1e5c 100644 --- a/src/main/character-dictionary-runtime/image-lookup.ts +++ b/src/main/character-dictionary-runtime/image-lookup.ts @@ -218,6 +218,8 @@ export function createCharacterDictionaryImageLookup(deps: { userDataPath?: string; outputDir?: string; getCurrentMediaId?: () => number | null | undefined; + onIndexReady?: () => void; + onIndexReadyError?: (error: unknown) => void; }): { get: (term: string, mediaId?: number | null) => CharacterNameImage | null; invalidate: () => void; @@ -229,6 +231,24 @@ export function createCharacterDictionaryImageLookup(deps: { let index = new Map(); let indexByMediaId = new Map>(); let refreshInFlight = false; + let indexReadyDeliveryPending = false; + + function deliverIndexReadyIfPending(): void { + if (!indexReadyDeliveryPending || !deps.onIndexReady) { + return; + } + indexReadyDeliveryPending = false; + try { + deps.onIndexReady(); + } catch (error) { + indexReadyDeliveryPending = true; + try { + deps.onIndexReadyError?.(error); + } catch { + // Error reporting must not reject the detached index refresh task. + } + } + } // Rebuilding means re-reading every cached snapshot (potentially GBs of JSON), which used to run // synchronously inside a lookup and froze the whole app right after a snapshot changed. Lookups @@ -241,6 +261,7 @@ export function createCharacterDictionaryImageLookup(deps: { signature = ''; return; } + deliverIndexReadyIfPending(); const nextSignature = getSnapshotDirectorySignature(outputDir); if (nextSignature === signature || refreshInFlight) { return; @@ -262,6 +283,8 @@ export function createCharacterDictionaryImageLookup(deps: { index = nextIndex; indexByMediaId = nextIndexByMediaId; signature = nextSignature; + indexReadyDeliveryPending = deps.onIndexReady !== undefined; + deliverIndexReadyIfPending(); } finally { refreshInFlight = false; } diff --git a/src/main/character-dictionary-runtime/name-split-resolver.test.ts b/src/main/character-dictionary-runtime/name-split-resolver.test.ts index 84f70c69..9f442ecd 100644 --- a/src/main/character-dictionary-runtime/name-split-resolver.test.ts +++ b/src/main/character-dictionary-runtime/name-split-resolver.test.ts @@ -43,7 +43,8 @@ test('resolveJapaneseNameSplits splits a single-kanji surname via person-name PO }), ); - assert.deepEqual(splits.get('東紫乃'), { family: '東', given: '紫乃' }); + assert.equal(splits.kind, 'complete'); + assert.deepEqual(splits.splits.get('東紫乃'), { family: '東', given: '紫乃' }); }); test('resolveJapaneseNameSplits corrects a hint-length-misleading surname boundary', async () => { @@ -64,7 +65,8 @@ test('resolveJapaneseNameSplits corrects a hint-length-misleading surname bounda }), ); - assert.deepEqual(splits.get('渡辺真奈美'), { family: '渡辺', given: '真奈美' }); + assert.equal(splits.kind, 'complete'); + assert.deepEqual(splits.splits.get('渡辺真奈美'), { family: '渡辺', given: '真奈美' }); }); test('resolveJapaneseNameSplits falls back to hint readings when POS tags are generic', async () => { @@ -85,7 +87,8 @@ test('resolveJapaneseNameSplits falls back to hint readings when POS tags are ge }), ); - assert.deepEqual(splits.get('鈴木みゆ'), { family: '鈴木', given: 'みゆ' }); + assert.equal(splits.kind, 'complete'); + assert.deepEqual(splits.splits.get('鈴木みゆ'), { family: '鈴木', given: 'みゆ' }); }); test('resolveJapaneseNameSplits skips names whose tokens do not reconstruct the name', async () => { @@ -96,7 +99,8 @@ test('resolveJapaneseNameSplits skips names whose tokens do not reconstruct the }), ); - assert.equal(splits.size, 0); + assert.equal(splits.kind, 'complete'); + assert.equal(splits.splits.size, 0); }); test('resolveJapaneseNameSplits skips ambiguous or untagged segmentations', async () => { @@ -117,7 +121,8 @@ test('resolveJapaneseNameSplits skips ambiguous or untagged segmentations', asyn }), ); - assert.equal(splits.size, 0); + assert.equal(splits.kind, 'complete'); + assert.equal(splits.splits.size, 0); }); test('resolveJapaneseNameSplits survives tokenizer failures', async () => { @@ -130,7 +135,8 @@ test('resolveJapaneseNameSplits survives tokenizer failures', async () => { (message) => warnings.push(message), ); - assert.equal(splits.size, 0); + assert.equal(splits.kind, 'incomplete'); + assert.equal(splits.splits.size, 0); assert.equal(warnings.length, 1); assert.match(warnings[0]!, /mecab unavailable/); }); diff --git a/src/main/character-dictionary-runtime/name-split-resolver.ts b/src/main/character-dictionary-runtime/name-split-resolver.ts index a1acedda..399a3a79 100644 --- a/src/main/character-dictionary-runtime/name-split-resolver.ts +++ b/src/main/character-dictionary-runtime/name-split-resolver.ts @@ -7,6 +7,10 @@ import type { ResolvedNameSplit, } from './types'; +export type JapaneseNameSplitResolution = + | { kind: 'complete'; splits: Map } + | { kind: 'incomplete'; splits: Map }; + const NAME_SEPARATOR_PATTERN = /[\s　・･·•]/; function joinSurfaces(tokens: NameSplitToken[]): string { @@ -87,8 +91,9 @@ export async function resolveJapaneseNameSplits( tokenize: NameSplitTokenizer, logWarn?: (message: string) => void, onCharacterResolved?: (completed: number, total: number) => void, -): Promise> { +): Promise { const splits = new Map(); + let tokenizerFailed = false; let resolvedCharacters = 0; for (const character of characters) { const familyHintReading = buildReadingFromHint(character.lastNameHint?.trim() || ''); @@ -99,12 +104,17 @@ export async function resolveJapaneseNameSplits( try { tokens = await tokenize(name); } catch (err) { + tokenizerFailed = true; logWarn?.( `[dictionary] name split tokenization failed for "${name}": ${(err as Error).message}`, ); continue; } - if (!tokens || tokens.length < 2 || joinSurfaces(tokens) !== name) continue; + if (!tokens) { + tokenizerFailed = true; + continue; + } + if (tokens.length < 2 || joinSurfaces(tokens) !== name) continue; const splitIndex = splitIndexFromPersonNamePos(tokens) ?? splitIndexFromHintReadings(tokens, familyHintReading, givenHintReading); @@ -118,5 +128,5 @@ export async function resolveJapaneseNameSplits( resolvedCharacters += 1; onCharacterResolved?.(resolvedCharacters, characters.length); } - return splits; + return tokenizerFailed ? { kind: 'incomplete', splits } : { kind: 'complete', splits }; } diff --git a/src/main/character-dictionary-runtime/snapshot-refresh.test.ts b/src/main/character-dictionary-runtime/snapshot-refresh.test.ts index b1d34fcf..25d0ecbd 100644 --- a/src/main/character-dictionary-runtime/snapshot-refresh.test.ts +++ b/src/main/character-dictionary-runtime/snapshot-refresh.test.ts @@ -7,7 +7,7 @@ import test from 'node:test'; import { createCharacterDictionaryRuntimeService } from '../character-dictionary-runtime'; import { getSnapshotPath, writeSnapshot } from './cache'; import { CHARACTER_DICTIONARY_FORMAT_VERSION } from './constants'; -import type { CharacterDictionarySnapshot } from './types'; +import type { CharacterDictionarySnapshot, NameSplitTokenizer } from './types'; const GRAPHQL_URL = 'https://graphql.anilist.co'; const PNG_1X1 = Buffer.from( @@ -121,7 +121,12 @@ test('generateForCurrentMedia refreshes same-version snapshots missing images wh } }); -test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable', async () => { +async function runNameSplitRefreshScenario(tokenizeJapaneseName: NameSplitTokenizer): Promise<{ + characterPageRequests: number; + firstResultFromCache: boolean; + refreshedNameSplitSource: CharacterDictionarySnapshot['nameSplitSource']; + secondResultFromCache: boolean; +}> { const userDataPath = makeTempDir(); const outputDir = path.join(userDataPath, 'character-dictionaries'); await writeSnapshot(getSnapshotPath(outputDir, 130298), { @@ -172,7 +177,6 @@ test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable' }) as typeof globalThis.fetch; try { - let tokenizerCalls = 0; const runtime = createCharacterDictionaryRuntimeService({ userDataPath, getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv', @@ -185,29 +189,54 @@ test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable' source: 'fallback', }), getNameMatchImagesEnabled: () => false, - tokenizeJapaneseName: async () => { - tokenizerCalls += 1; - return null; - }, + tokenizeJapaneseName, getJapaneseNameTokenizerAvailable: () => true, now: () => 1_700_000_000_500, }); - const result = await runtime.generateForCurrentMedia(); + const firstResult = await runtime.generateForCurrentMedia(); const refreshedSnapshot = JSON.parse( fs.readFileSync(getSnapshotPath(outputDir, 130298), 'utf8'), ) as CharacterDictionarySnapshot; + const secondResult = await runtime.generateForCurrentMedia(); - assert.equal(result.fromCache, false); - assert.equal(refreshedSnapshot.nameSplitSource, 'heuristic'); - - const retriedResult = await runtime.generateForCurrentMedia(); - assert.equal(retriedResult.fromCache, false); - assert.equal(characterPageRequests, 2); - assert.equal(tokenizerCalls, 2); + return { + characterPageRequests, + firstResultFromCache: firstResult.fromCache, + refreshedNameSplitSource: refreshedSnapshot.nameSplitSource, + secondResultFromCache: secondResult.fromCache, + }; } finally { globalThis.fetch = originalFetch; } +} + +test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable', async () => { + let tokenizerCalls = 0; + const result = await runNameSplitRefreshScenario(async () => { + tokenizerCalls += 1; + return null; + }); + + assert.equal(result.firstResultFromCache, false); + assert.equal(result.refreshedNameSplitSource, 'heuristic'); + assert.equal(result.secondResultFromCache, false); + assert.equal(result.characterPageRequests, 2); + assert.equal(tokenizerCalls, 2); +}); + +test('generateForCurrentMedia caches completed MeCab refreshes with no resolved splits', async () => { + let tokenizerCalls = 0; + const result = await runNameSplitRefreshScenario(async () => { + tokenizerCalls += 1; + return []; + }); + + assert.equal(result.firstResultFromCache, false); + assert.equal(result.refreshedNameSplitSource, 'mecab'); + assert.equal(result.secondResultFromCache, true); + assert.equal(result.characterPageRequests, 1); + assert.equal(tokenizerCalls, 1); }); test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is available', async () => { diff --git a/src/main/main-wiring.test.ts b/src/main/main-wiring.test.ts index 73c61ba0..e536442e 100644 --- a/src/main/main-wiring.test.ts +++ b/src/main/main-wiring.test.ts @@ -482,10 +482,10 @@ test('Linux visible overlay recreation avoids display fallback before tracked ge assert.doesNotMatch(actionBlock, /setOverlayWindowBounds\(getCurrentOverlayGeometry\(\)\)/); }); -test('known-word updates invalidate prefetched tokenizations before refreshing current subtitle', () => { +test('subtitle annotation updates invalidate prefetched tokenizations before refreshing current subtitle', () => { const source = readMainSource(); const actionBlock = source.match( - /const refreshCurrentSubtitleAfterKnownWordUpdate = \(\): void => \{(?[\s\S]*?)\n\};/, + /function refreshCurrentSubtitleAnnotations\(\): void \{(?[\s\S]*?)\n\}/, )?.groups?.body; assert.ok(actionBlock); @@ -503,6 +503,20 @@ test('known-word updates invalidate prefetched tokenizations before refreshing c ); }); +test('character portrait index readiness refreshes cached subtitle annotations', () => { + const source = readMainSource(); + const lookupDeps = source.match( + /const characterDictionaryImageLookup = createCharacterDictionaryImageLookup\(\{(?[\s\S]*?)\n\}\);/, + )?.groups?.body; + + assert.ok(lookupDeps); + assert.match(lookupDeps, /onIndexReady: \(\) => refreshCurrentSubtitleAnnotations\(\),/); + assert.match( + lookupDeps, + /onIndexReadyError: \(error\) =>[\s\S]*?logger\.warn\([\s\S]*?character portrait index became ready\.[\s\S]*?error,/, + ); +}); + test('subtitle processing controller resumes prefetch on settle, not on its emits', () => { const source = readMainSource(); const depsBlock = source.match(