diff --git a/changes/subtitle-tokenization-performance.md b/changes/subtitle-tokenization-performance.md index 5e3901bd..444b32a0 100644 --- a/changes/subtitle-tokenization-performance.md +++ b/changes/subtitle-tokenization-performance.md @@ -9,4 +9,5 @@ area: subtitles - Subtitle changes no longer restart the prefetch run per line (which discarded in-flight tokenization work); prefetch now only pauses for the live line and restarts on real seeks, cache invalidation, or option changes. Prefetch also stays paused across a provisional raw-subtitle emit and resumes only after the tokenized payload lands, so it never competes with the on-screen line for the parser window. - Added per-stage debug timings (`scanMs`, `mecabMs`, `frequencyMs`, `annotateMs`) to the subtitle tokenization pipeline log. - Fixed a reading that stopped covering its surface when an unmatched kana run extended the preceding token (for example a trailing る on 待ち合わせ), which silently disabled the known-word reading fallback for those tokens. +- Subtitle prefetching no longer stays paused for the rest of a cue when the same subtitle text is reported twice and there is nothing to tokenize. - Character name annotations no longer cost a dictionary lookup at every position in a line. The scanner now knows which name forms the current title's character dictionary actually contains and only checks where one can start, which removes the whole overhead of having the character dictionary enabled (measured: 21 lookups per line down to 10, the same as with it disabled). Titles with no cached character data keep the previous exhaustive scan, so a missing snapshot costs speed rather than a missing name. diff --git a/src/core/services/subtitle-processing-controller.test.ts b/src/core/services/subtitle-processing-controller.test.ts index ec35c170..fb54a734 100644 --- a/src/core/services/subtitle-processing-controller.test.ts +++ b/src/core/services/subtitle-processing-controller.test.ts @@ -539,3 +539,22 @@ test('default cache limit covers a full-length title without evicting', () => { assert.equal(controller.hasCachedSubtitle('line-0'), true); assert.equal(controller.hasCachedSubtitle('line-1999'), true); }); + +test('onSubtitleChange reports whether processing was scheduled', async () => { + const emitted: SubtitleData[] = []; + const controller = createSubtitleProcessingController({ + tokenizeSubtitle: async (text) => ({ text, tokens: [] }), + emitSubtitle: (payload) => emitted.push(payload), + }); + + // New text schedules work, so an emit (and anything gated on it) will follow. + assert.equal(controller.onSubtitleChange('字幕'), true); + await flushMicrotasks(); + + // A repeat emits nothing, so callers must not wait on an emit that is never + // coming (subtitle prefetching would stay paused for the rest of the cue). + const emittedCount = emitted.length; + assert.equal(controller.onSubtitleChange('字幕'), false); + await flushMicrotasks(); + assert.equal(emitted.length, emittedCount); +}); diff --git a/src/core/services/subtitle-processing-controller.ts b/src/core/services/subtitle-processing-controller.ts index cc1a6cf5..346bd331 100644 --- a/src/core/services/subtitle-processing-controller.ts +++ b/src/core/services/subtitle-processing-controller.ts @@ -17,7 +17,12 @@ export interface SubtitleProcessingControllerDeps { export const DEFAULT_SUBTITLE_TOKENIZATION_CACHE_LIMIT = 2500; export interface SubtitleProcessingController { - onSubtitleChange: (text: string) => void; + /** + * Returns whether the text was new and processing was scheduled. A false + * return means nothing will be emitted for this event, which callers that + * gate work on the emit (such as pausing subtitle prefetching) need to know. + */ + onSubtitleChange: (text: string) => boolean; refreshCurrentSubtitle: (textOverride?: string) => void; invalidateTokenizationCache: () => void; preCacheTokenization: (text: string, data: SubtitleData) => void; @@ -171,7 +176,7 @@ export function createSubtitleProcessingController( return { onSubtitleChange: (text: string) => { if (text === latestText) { - return; + return false; } latestText = text; if ( @@ -183,6 +188,7 @@ export function createSubtitleProcessingController( lastPlainEmittedText = text; } processLatest(); + return true; }, refreshCurrentSubtitle: (textOverride?: string) => { if (typeof textOverride === 'string') { diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts index 4061f8d6..c055c068 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts @@ -2788,3 +2788,100 @@ test('addYomitanNoteViaSearch sanitizes invalid payload note ids while keeping v duplicateNoteIds: [18, 7], }); }); + +test('requestYomitanScanTokens still finds an emphatically elongated name a longer generic match would swallow', async () => { + // Yomitan collapses emphatic sequences, so ミナァァト resolves to the ミナト + // entry. The generic word とミナ starts earlier and would swallow the name + // unless the pre-pass reserves it, so this only passes when the candidate + // prefilter still treats the elongated spelling as a possible name start. + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { + return { + profileCurrent: 0, + profiles: [ + { + options: { + scanning: { length: 40 }, + dictionaries: [ + { name: 'JMdict', enabled: true, id: 0 }, + { name: 'SubMiner Character Dictionary (AniList 1)', enabled: true, id: 1 }, + ], + }, + }, + ], + }; + } + if (action === 'getDictionaryInfo') { + return []; + } + const text = (params as { text?: string } | undefined)?.text ?? ''; + if (text.startsWith('とミナ')) { + return { + originalTextLength: 3, + dictionaryEntries: [ + { + headwords: [ + { + term: 'トミナ', + reading: 'とみな', + sources: [{ originalText: 'とミナ', isPrimary: true, matchType: 'exact' }], + }, + ], + definitions: [{ dictionary: 'JMdict' }], + }, + ], + }; + } + if (text.startsWith('ミナァァト')) { + return { + originalTextLength: 5, + dictionaryEntries: [ + { + headwords: [ + { + term: 'ミナト', + reading: 'みなと', + sources: [{ originalText: 'ミナァァト', isPrimary: true, matchType: 'exact' }], + }, + ], + definitions: [{ dictionary: 'SubMiner Character Dictionary (AniList 1)' }], + }, + ], + }; + } + if (text.startsWith('と')) { + return { + originalTextLength: 1, + dictionaryEntries: [ + { + headwords: [ + { + term: 'と', + reading: 'と', + sources: [{ originalText: 'と', isPrimary: true, matchType: 'exact' }], + }, + ], + definitions: [{ dictionary: 'JMdict' }], + }, + ], + }; + } + return { originalTextLength: 0, dictionaryEntries: [] }; + }); + + const result = await requestYomitanScanTokens( + 'とミナァァト', + deps, + { error: () => undefined }, + { + includeNameMatchMetadata: true, + currentCharacterDictionaryMediaId: 1, + nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] }, + }, + ); + + const nameToken = result?.find((token) => token.isNameMatch === true); + assert.ok(nameToken, 'expected the elongated name to be reserved by the pre-pass'); + assert.equal(nameToken?.headword, 'ミナト'); + assert.equal(nameToken?.startPos, 1); +}); diff --git a/src/core/services/tokenizer/yomitan-scan-runtime-script.ts b/src/core/services/tokenizer/yomitan-scan-runtime-script.ts index 7901b21f..3a6ca44a 100644 --- a/src/core/services/tokenizer/yomitan-scan-runtime-script.ts +++ b/src/core/services/tokenizer/yomitan-scan-runtime-script.ts @@ -495,7 +495,7 @@ const YOMITAN_SCANNING_HELPERS = String.raw` // Bump whenever the install script below changes so already-loaded parser // windows re-install the new scan runtime instead of running the stale one. -export const YOMITAN_SCAN_RUNTIME_VERSION = 3; +export const YOMITAN_SCAN_RUNTIME_VERSION = 4; export const YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL = '__subminer-yomitan-scan-runtime-missing__'; export interface YomitanScanRequestParams { @@ -731,13 +731,33 @@ ${YOMITAN_SCANNING_HELPERS} ? nameCandidateIndex : null; const normalizedText = activeNameCandidateIndex ? convertKatakanaToHiragana(text) : ""; + // Yomitan collapses emphatic sequences before matching (すっっごーーい → + // すごい), so a stretched name still resolves to its entry. Skipping these + // characters keeps such spellings candidates; the filter only ever grows + // the probe set, so a false positive costs one lookup, never a name. + const EMPHATIC_SKIP_CHARS = new Set(["ぁ", "ぃ", "ぅ", "ぇ", "ぉ", "っ", "ゃ", "ゅ", "ょ", "ー"]); + function matchesCandidateFormAt(form, position) { + let textIndex = position; + for (let formIndex = 0; formIndex < form.length; formIndex += 1) { + while ( + textIndex < normalizedText.length && + normalizedText[textIndex] !== form[formIndex] && + EMPHATIC_SKIP_CHARS.has(normalizedText[textIndex]) + ) { + textIndex += 1; + } + if (normalizedText[textIndex] !== form[formIndex]) { return false; } + textIndex += 1; + } + return true; + } function couldNameStartAt(position, codePoint) { if (!activeNameCandidateIndex) { return true; } if (isHalfwidthKatakanaCodePoint(codePoint)) { return true; } const bucket = activeNameCandidateIndex.byFirstChar.get(normalizedText[position]); if (!bucket) { return false; } for (const form of bucket) { - if (normalizedText.startsWith(form, position)) { return true; } + if (matchesCandidateFormAt(form, position)) { return true; } } return false; } diff --git a/src/main.ts b/src/main.ts index 267114a8..7a543f93 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4388,7 +4388,12 @@ const { // Pause only; restarting the prefetch run here would discard in-flight // tokenization work on every line. Real seeks restart via onTimePosUpdate. subtitlePrefetchService?.pause(); - subtitleProcessingController.onSubtitleChange(text); + if (!subtitleProcessingController.onSubtitleChange(text)) { + // Repeat of the current text: nothing will be tokenized, so no emit is + // coming to release the pause. Resume now instead of idling prefetch + // for the rest of the cue. + subtitlePrefetchService?.resume(); + } }, refreshDiscordPresence: () => { discordPresenceRuntime.publishDiscordPresence(); diff --git a/src/main/main-wiring.test.ts b/src/main/main-wiring.test.ts index 1e25ed39..d7599cc1 100644 --- a/src/main/main-wiring.test.ts +++ b/src/main/main-wiring.test.ts @@ -234,10 +234,16 @@ test('subtitle change pauses prefetch without restarting its run before tokenizi // Restarting the run per line (onSeek) discards in-flight prefetch work; // only real seeks restart via onTimePosUpdate. assert.doesNotMatch(actionBlock, /subtitlePrefetchService\?\.onSeek\(/); - assert.match(actionBlock, /subtitleProcessingController\.onSubtitleChange\(text\);/); + assert.match(actionBlock, /subtitleProcessingController\.onSubtitleChange\(text\)/); assert.ok( actionBlock.indexOf('subtitlePrefetchService?.pause();') < - actionBlock.indexOf('subtitleProcessingController.onSubtitleChange(text);'), + actionBlock.indexOf('subtitleProcessingController.onSubtitleChange(text)'), + ); + // A repeated subtitle emits nothing, so the pause has to be released here or + // prefetching idles until the next distinct line. + assert.match( + actionBlock, + /if \(!subtitleProcessingController\.onSubtitleChange\(text\)\) \{[\s\S]*?subtitlePrefetchService\?\.resume\(\);/, ); });