From e7cef039f35fe94362e75a8f5951db735436840d Mon Sep 17 00:00:00 2001 From: sudacode Date: Mon, 3 Aug 2026 22:33:01 -0700 Subject: [PATCH] perf(tokenizer): single-pass Yomitan scan with install-once runtime and cross-line cache - drop the duplicate parseText full parse per line; the termsFind scanner walk is now authoritative and emits its own unparsed filler runs (parseText kept only as error fallback) - install scan helpers once per parser window (__subminerYomitanScan) instead of re-shipping ~500 lines of script per subtitle line - persist termsFind results across lines in a window-scoped LRU keyed by substring, invalidated via a cache epoch on dictionary/settings changes - skip lookups at punctuation/whitespace positions and cap the shrinking-window retry ladder at 4 lookups per position - build tokenizer runtime deps once (JLPT lookup cache never hit before; mecab availability check ran per line) - stop restarting the prefetch run on every subtitle change; resume prefetch only after the tokenized payload lands, not on provisional raw emits - add per-stage debug timings (scanMs/mecabMs/frequencyMs/annotateMs) --- changes/subtitle-tokenization-performance.md | 10 + src/core/services/tokenizer.test.ts | 38 +- src/core/services/tokenizer.ts | 48 +- .../tokenizer/golden-corpus-harness.ts | 11 +- .../tokenizer/yomitan-parser-runtime.test.ts | 1411 +++++++---------- .../tokenizer/yomitan-parser-runtime.ts | 386 +++-- src/main.ts | 14 +- src/main/main-wiring.test.ts | 12 +- .../autoplay-subtitle-priming-runtime.test.ts | 12 +- .../autoplay-subtitle-priming-runtime.ts | 10 +- .../runtime/composers/mpv-runtime-composer.ts | 41 +- 11 files changed, 959 insertions(+), 1034 deletions(-) create mode 100644 changes/subtitle-tokenization-performance.md diff --git a/changes/subtitle-tokenization-performance.md b/changes/subtitle-tokenization-performance.md new file mode 100644 index 00000000..a6627555 --- /dev/null +++ b/changes/subtitle-tokenization-performance.md @@ -0,0 +1,10 @@ +type: changed +area: subtitles + +- Subtitle tokenization no longer runs a duplicate full `parseText` pass per line: the termsFind scanner walk is now the only tokenizer and emits its own hoverable filler runs for unmatched text (parseText is kept only as an error fallback). This roughly halves the dictionary work per line. +- The Yomitan scanning helpers are now installed once per parser window (`__subminerYomitanScan`) instead of re-shipping and re-parsing a ~500-line script for every subtitle line; each line only evaluates a tiny call. +- termsFind lookups are cached across subtitle lines in a window-persistent LRU keyed by substring, so repeated particles and verb forms stop costing backend round trips. The cache invalidates on dictionary/settings changes and window reloads. +- The scanner walk now skips lookups at punctuation and whitespace positions (latin letters and digits still look up, e.g. Tシャツ) and caps the shrinking-window retry ladder at four extra lookups per position. +- Tokenizer runtime dependencies are built once instead of per line, fixing a JLPT lookup cache that never hit (it was keyed on a per-call closure identity and leaked a Map per line) and a `which mecab` availability check that re-ran synchronously on every line when MeCab is absent. +- Subtitle changes no longer restart the prefetch run per line (which discarded in-flight tokenization work); prefetch now only pauses for the live line and restarts on real seeks, cache invalidation, or option changes. Prefetch also stays paused across a provisional raw-subtitle emit and resumes only after the tokenized payload lands, so it never competes with the on-screen line for the parser window. +- Added per-stage debug timings (`scanMs`, `mecabMs`, `frequencyMs`, `annotateMs`) to the subtitle tokenization pipeline log. diff --git a/src/core/services/tokenizer.test.ts b/src/core/services/tokenizer.test.ts index 23deee51..390d4c85 100644 --- a/src/core/services/tokenizer.test.ts +++ b/src/core/services/tokenizer.test.ts @@ -2934,44 +2934,12 @@ test('tokenizeSubtitle preserves Yomitan compound token when MeCab components ar return []; } - if (script.includes('parseText')) { - return [ - { - source: 'scanning-parser', - index: 0, - content: [ - [ - { - text: '取り組んで', - reading: 'とりくんで', - headwords: [[{ term: '取り組む' }]], - }, - ], - [ - { - text: 'もらいます', - reading: 'もらいます', - headwords: [[{ term: 'もらう' }]], - }, - ], - ], - }, - ]; - } - return [ { - surface: '取り', - reading: 'とり', - headword: '取る', + surface: '取り組んで', + reading: 'とりくんで', + headword: '取り組む', startPos: 0, - endPos: 2, - }, - { - surface: '組んで', - reading: 'くんで', - headword: '組む', - startPos: 2, endPos: 5, }, { diff --git a/src/core/services/tokenizer.ts b/src/core/services/tokenizer.ts index 676dd213..a1d8c76f 100644 --- a/src/core/services/tokenizer.ts +++ b/src/core/services/tokenizer.ts @@ -716,15 +716,29 @@ function getAnnotationOptions(deps: TokenizerServiceDeps): TokenizerAnnotationOp }; } +// Per-line stage durations for the pipeline debug log; every field is filled in +// by the stage that awaits the corresponding work. +interface TokenizationStageTimings { + scanMs?: number; + mecabMs?: number; + frequencyMs?: number; + annotateMs?: number; +} + async function parseWithYomitanInternalParser( text: string, deps: TokenizerServiceDeps, options: TokenizerAnnotationOptions, + stageTimings?: TokenizationStageTimings, ): Promise { + const scanStartedAtMs = Date.now(); const selectedTokens = await requestYomitanScanTokens(text, deps, logger, { includeNameMatchMetadata: options.nameMatchEnabled, currentCharacterDictionaryMediaId: deps.getCurrentCharacterDictionaryMediaId?.() ?? null, }); + if (stageTimings) { + stageTimings.scanMs = Date.now() - scanStartedAtMs; + } if (!selectedTokens || selectedTokens.length === 0) { return null; } @@ -757,6 +771,7 @@ async function parseWithYomitanInternalParser( const frequencyRankPromise: Promise = options.frequencyEnabled ? (async () => { + const frequencyStartedAtMs = Date.now(); const frequencyMatchMode = options.frequencyMatchMode; const termReadingList = buildYomitanFrequencyTermReadingList( normalizedSelectedTokens, @@ -767,12 +782,17 @@ async function parseWithYomitanInternalParser( deps, logger, ); - return buildYomitanFrequencyIndex(yomitanFrequencies); + const frequencyIndex = buildYomitanFrequencyIndex(yomitanFrequencies); + if (stageTimings) { + stageTimings.frequencyMs = Date.now() - frequencyStartedAtMs; + } + return frequencyIndex; })() : Promise.resolve({ byPair: new Map(), byTerm: new Map() }); const mecabEnrichmentPromise: Promise = needsMecabPosEnrichment(options) ? (async () => { + const mecabStartedAtMs = Date.now(); try { const mecabTokens = await deps.tokenizeWithMecab(text); const enrichTokensWithMecab = deps.enrichTokensWithMecab ?? enrichTokensWithMecabAsync; @@ -786,6 +806,10 @@ async function parseWithYomitanInternalParser( `textLength=${text.length}`, ); return normalizedSelectedTokens; + } finally { + if (stageTimings) { + stageTimings.mecabMs = Date.now() - mecabStartedAtMs; + } } })() : Promise.resolve(normalizedSelectedTokens); @@ -876,15 +900,35 @@ export async function tokenizeSubtitle( const annotationOptions = getAnnotationOptions(deps); annotationOptions.sourceText = tokenizeText; - const yomitanTokens = await parseWithYomitanInternalParser(tokenizeText, deps, annotationOptions); + const stageTimings: TokenizationStageTimings = {}; + const startedAtMs = Date.now(); + const logStageTimings = (tokenCount: number): void => { + logger.debug( + `Subtitle tokenization stages; textLength=${tokenizeText.length}, tokenCount=${tokenCount}, ` + + `scanMs=${stageTimings.scanMs ?? '-'}, mecabMs=${stageTimings.mecabMs ?? '-'}, ` + + `frequencyMs=${stageTimings.frequencyMs ?? '-'}, annotateMs=${stageTimings.annotateMs ?? '-'}, ` + + `totalMs=${Date.now() - startedAtMs}`, + ); + }; + + const yomitanTokens = await parseWithYomitanInternalParser( + tokenizeText, + deps, + annotationOptions, + stageTimings, + ); if (yomitanTokens && yomitanTokens.length > 0) { + const annotateStartedAtMs = Date.now(); const annotatedTokens = await applyAnnotationStage(yomitanTokens, deps, annotationOptions); const renderedTokens = applyCharacterNameImages(annotatedTokens, deps, annotationOptions); + stageTimings.annotateMs = Date.now() - annotateStartedAtMs; + logStageTimings(renderedTokens.length); return { text: displayText, tokens: renderedTokens.length > 0 ? renderedTokens : null, }; } + logStageTimings(0); return { text: displayText, tokens: null }; } diff --git a/src/core/services/tokenizer/golden-corpus-harness.ts b/src/core/services/tokenizer/golden-corpus-harness.ts index 83eeeb57..ce17fa35 100644 --- a/src/core/services/tokenizer/golden-corpus-harness.ts +++ b/src/core/services/tokenizer/golden-corpus-harness.ts @@ -366,8 +366,11 @@ export function createReplayMessageStore(messages: GoldenRecordedMessage[]): Rep }; } -async function runInjectedScriptInVm(script: string, store: ReplayMessageStore): Promise { - return await vm.runInNewContext(script, { +// One persistent context per fixture, matching the real parser window: the +// scan runtime installs itself once into globalThis and later per-line call +// scripts reuse it. +function createInjectedScriptVm(store: ReplayMessageStore): (script: string) => Promise { + const context = vm.createContext({ chrome: { runtime: { lastError: null, @@ -393,6 +396,7 @@ async function runInjectedScriptInVm(script: string, store: ReplayMessageStore): Set, String, }); + return async (script: string) => await vm.runInContext(script, context); } export function createReplayTokenizerDeps(fixture: GoldenFixture): TokenizerServiceDeps { @@ -400,13 +404,14 @@ export function createReplayTokenizerDeps(fixture: GoldenFixture): TokenizerServ const scriptResults = new Map( fixture.recording.scripts.map((entry) => [entry.sha256, entry] as const), ); + const runInjectedScriptInVm = createInjectedScriptVm(store); const parserWindow = { isDestroyed: () => false, webContents: { executeJavaScript: async (script: string) => { try { - return await runInjectedScriptInVm(script, store); + return await runInjectedScriptInVm(script); } catch (vmError) { const recorded = scriptResults.get(hashInjectedScript(script)); if (recorded) { diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts index 00c808a5..07629154 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts @@ -6,6 +6,7 @@ import test from 'node:test'; import * as vm from 'node:vm'; import { addYomitanNoteViaSearch, + clearYomitanParserCachesForWindow, extractYomitanCurrentAnkiDeckName, getYomitanDictionaryInfo, importYomitanDictionaryFromZip, @@ -42,11 +43,8 @@ function createDeps( }; } -async function runInjectedYomitanScript( - script: string, - handler: (action: string, params: unknown) => unknown, -): Promise { - return await vm.runInNewContext(script, { +function createYomitanScriptSandbox(handler: (action: string, params: unknown) => unknown) { + return { chrome: { runtime: { lastError: null, @@ -73,9 +71,45 @@ async function runInjectedYomitanScript( RegExp, Set, String, + }; +} + +async function runInjectedYomitanScript( + script: string, + handler: (action: string, params: unknown) => unknown, +): Promise { + return await vm.runInNewContext(script, createYomitanScriptSandbox(handler)); +} + +// Persistent page context shared across executeJavaScript calls, matching the +// real parser window: the scan runtime is installed once via +// globalThis.__subminerYomitanScan and per-line calls reuse it (and its +// cross-line termsFind cache). +function createPersistentYomitanScriptRunner( + handler: (action: string, params: unknown) => unknown, +): (script: string) => Promise { + const context = vm.createContext(createYomitanScriptSandbox(handler)); + return async (script: string) => await vm.runInContext(script, context); +} + +// Deps whose parser window executes every injected script (profile metadata, +// scan runtime install, per-line scan calls, parseText fallback) inside one +// persistent vm context, dispatching backend actions to `handler`. +function createScanDeps( + handler: (action: string, params: unknown) => unknown, + options?: { onScript?: (script: string) => void }, +) { + const runScript = createPersistentYomitanScriptRunner(handler); + return createDeps(async (script) => { + options?.onScript?.(script); + return await runScript(script); }); } +function countTermsFindLookups(lookups: string[], prefix: string): number { + return lookups.filter((lookupText) => lookupText.startsWith(prefix)).length; +} + test('syncYomitanDefaultAnkiServer updates default profile server when script reports update', async () => { let scriptValue = ''; const deps = createDeps(async (script) => { @@ -650,56 +684,45 @@ test('requestYomitanTermFrequencies caches repeated term+reading lookups', async assert.equal(frequencyCalls, 1); }); -test('requestYomitanScanTokens prefers parseText tokenization over termsFind fragments', async () => { +test('requestYomitanScanTokens tokenizes with the in-window scanner and no parseText request', async () => { const scripts: string[] = []; - const deps = createDeps(async (script) => { - scripts.push(script); - if (script.includes('optionsGetFull')) { - return { - profileCurrent: 0, - profiles: [ - { - options: { - scanning: { length: 40 }, + const actions: string[] = []; + const deps = createScanDeps( + (action, params) => { + actions.push(action); + if (action === 'optionsGetFull') { + return { + profileCurrent: 0, + profiles: [{ options: { scanning: { length: 40 } } }], + }; + } + if (action === 'getDictionaryInfo') { + return []; + } + if (action === 'termsFind') { + const text = (params as { text?: string } | undefined)?.text ?? ''; + if (!text.startsWith('取り組んで')) { + return { originalTextLength: 0, dictionaryEntries: [] }; + } + return { + originalTextLength: 5, + dictionaryEntries: [ + { + headwords: [ + { + term: '取り組む', + reading: 'とりくむ', + sources: [{ originalText: '取り組んで', isPrimary: true, matchType: 'exact' }], + }, + ], }, - }, - ], - }; - } - if (script.includes('parseText')) { - return [ - { - source: 'scanning-parser', - index: 0, - content: [ - [ - { - text: '取り組んで', - reading: 'とりくんで', - headwords: [[{ term: '取り組む' }]], - }, - ], ], - }, - ]; - } - return [ - { - surface: '取り', - reading: 'とり', - headword: '取る', - startPos: 0, - endPos: 2, - }, - { - surface: '組んで', - reading: 'くんで', - headword: '組む', - startPos: 2, - endPos: 5, - }, - ]; - }); + }; + } + throw new Error(`unexpected action: ${action}`); + }, + { onScript: (script) => scripts.push(script) }, + ); const result = await requestYomitanScanTokens('取り組んで', deps, { error: () => undefined, @@ -710,18 +733,26 @@ test('requestYomitanScanTokens prefers parseText tokenization over termsFind fra surface: '取り組んで', reading: 'とりくんで', headword: '取り組む', + headwordReading: 'とりくむ', startPos: 0, endPos: 5, + isNameMatch: false, + frequencyRank: undefined, }, ]); - assert.ok(scripts.some((script) => script.includes('parseText'))); - assert.ok(scripts.some((script) => script.includes('termsFind'))); + // The duplicate full parse per line is gone: the scanner walk is the only + // tokenization request. + assert.ok(!actions.includes('parseText')); + const installScript = scripts.find((script) => script.includes('termsFind')); + assert.ok(installScript, 'expected the scan runtime install script'); + assert.match(installScript ?? '', /matchType:\s*"exact"/); + assert.match(installScript ?? '', /deinflect:\s*true/); }); test('requestYomitanScanTokens warns when active Yomitan profile has no dictionaries', async () => { const warnings: Array<{ message: string; details: unknown }> = []; - const deps = createDeps(async (script) => { - if (script.includes('optionsGetFull')) { + const deps = createScanDeps((action) => { + if (action === 'optionsGetFull') { return { profileCurrent: 0, profiles: [ @@ -734,13 +765,10 @@ test('requestYomitanScanTokens warns when active Yomitan profile has no dictiona ], }; } - if (script.includes('parseText')) { + if (action === 'getDictionaryInfo') { return []; } - if (script.includes('termsFind')) { - return []; - } - return null; + return { originalTextLength: 0, dictionaryEntries: [] }; }); await requestYomitanScanTokens('字幕', deps, { @@ -759,270 +787,86 @@ test('requestYomitanScanTokens warns when active Yomitan profile has no dictiona }); }); -test('requestYomitanScanTokens keeps scanner metadata when parse spans agree', async () => { - const deps = createDeps(async (script) => { - if (script.includes('optionsGetFull')) { +test('requestYomitanScanTokens emits unparsed filler runs for text the scanner skips', async () => { + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { return { profileCurrent: 0, - profiles: [ - { - options: { - scanning: { length: 40 }, + profiles: [{ options: { scanning: { length: 40 } } }], + }; + } + if (action === 'getDictionaryInfo') { + return []; + } + const text = (params as { text?: string } | undefined)?.text ?? ''; + const singleCharEntry = (term: string, reading: string) => ({ + originalTextLength: 1, + dictionaryEntries: [ + { + headwords: [ + { + term, + reading, + sources: [{ originalText: text[0], isPrimary: true, matchType: 'exact' }], }, + ], + }, + ], + }); + if (text.startsWith('や')) { + return singleCharEntry('や', 'や'); + } + if (text.startsWith('ほ')) { + return singleCharEntry('帆', 'ほ'); + } + if (text.startsWith('ミナト')) { + return { + originalTextLength: 3, + dictionaryEntries: [ + { + headwords: [ + { + term: 'ミナト', + reading: 'みなと', + sources: [{ originalText: 'ミナト', isPrimary: true, matchType: 'exact' }], + }, + ], }, ], }; } - if (script.includes('parseText')) { - return [ - { - source: 'scanning-parser', - index: 0, - content: [ - [ - { - text: 'アクア', - reading: 'あくあ', - headwords: [[{ term: 'アクア' }]], - }, - ], - ], - }, - ]; - } - return [ - { - surface: 'アクア', - reading: 'あくあ', - headword: 'アクア', - startPos: 0, - endPos: 3, - isNameMatch: true, - wordClasses: ['n'], - }, - ]; - }); - - const result = await requestYomitanScanTokens('アクア', deps, { - error: () => undefined, - }); - - assert.deepEqual(result, [ - { - surface: 'アクア', - reading: 'あくあ', - headword: 'アクア', - startPos: 0, - endPos: 3, - isNameMatch: true, - wordClasses: ['n'], - }, - ]); -}); - -test('requestYomitanScanTokens keeps scanner metadata for matching spans when parse segmentation has filler chunks', async () => { - const deps = createDeps(async (script) => { - if (script.includes('optionsGetFull')) { - return { - profileCurrent: 0, - profiles: [ - { - options: { - scanning: { length: 40 }, - }, - }, - ], - }; - } - if (script.includes('parseText')) { - return [ - { - source: 'scanning-parser', - index: 0, - content: [ - [ - { - text: 'や', - reading: '', - headwords: [[{ term: 'や' }]], - }, - ], - [ - { - text: 'ほ', - reading: '', - headwords: [[{ term: '帆' }]], - }, - ], - [ - { - text: 'っ ', - reading: '', - }, - ], - [ - { - text: 'ミナト', - reading: '', - headwords: [[{ term: 'ミナト' }]], - }, - ], - ], - }, - ]; - } - // The termsFind scanner skips the unmatched っ+space chunk, so its spans - // do not line up 1:1 with the parseText segmentation above. - return [ - { - surface: 'や', - reading: 'や', - headword: 'や', - headwordReading: 'や', - startPos: 0, - endPos: 1, - frequencyRank: 57, - }, - { - surface: 'ほ', - reading: 'ほ', - headword: '帆', - headwordReading: 'ほ', - startPos: 1, - endPos: 2, - frequencyRank: 15414, - }, - { - surface: 'ミナト', - reading: 'ミナト', - headword: 'ミナト', - headwordReading: 'みなと', - startPos: 4, - endPos: 7, - isNameMatch: true, - frequencyRank: 75133, - }, - ]; + return { originalTextLength: 0, dictionaryEntries: [] }; }); const result = await requestYomitanScanTokens('やほっ ミナト', deps, { error: () => undefined, }); - assert.deepEqual(result, [ - { - surface: 'や', - reading: 'や', - headword: 'や', - headwordReading: 'や', - startPos: 0, - endPos: 1, - frequencyRank: 57, - }, - { - surface: 'ほ', - reading: 'ほ', - headword: '帆', - headwordReading: 'ほ', - startPos: 1, - endPos: 2, - frequencyRank: 15414, - }, - { - surface: 'っ ', - reading: '', - headword: 'っ ', - startPos: 2, - endPos: 4, - isUnparsedRun: true, - }, - { - surface: 'ミナト', - reading: 'ミナト', - headword: 'ミナト', - headwordReading: 'みなと', - startPos: 4, - endPos: 7, - isNameMatch: true, - frequencyRank: 75133, - }, - ]); -}); - -test('requestYomitanScanTokens falls back to left-to-right termsFind scanning', async () => { - const scripts: string[] = []; - const deps = createDeps(async (script) => { - scripts.push(script); - if (script.includes('optionsGetFull')) { - return { - profileCurrent: 0, - profiles: [ - { - options: { - scanning: { length: 40 }, - }, - }, - ], - }; - } - if (script.includes('parseText')) { - return []; - } - return [ - { - surface: 'カズマ', - reading: 'かずま', - headword: 'カズマ', - startPos: 0, - endPos: 3, - }, - ]; - }); - - const result = await requestYomitanScanTokens('カズマ', deps, { - error: () => undefined, - }); - - assert.deepEqual(result, [ - { - surface: 'カズマ', - reading: 'かずま', - headword: 'カズマ', - startPos: 0, - endPos: 3, - }, - ]); - assert.ok(scripts.some((script) => script.includes('parseText'))); - const scannerScript = scripts.find((script) => script.includes('termsFind')); - assert.ok(scannerScript, 'expected termsFind scanning request script'); - assert.doesNotMatch(scannerScript ?? '', /parseText/); - assert.match(scannerScript ?? '', /matchType:\s*"exact"/); - assert.match(scannerScript ?? '', /deinflect:\s*true/); + assert.deepEqual( + result?.map(({ surface, headword, startPos, endPos, isUnparsedRun }) => ({ + surface, + headword, + startPos, + endPos, + isUnparsedRun, + })), + [ + { surface: 'や', headword: 'や', startPos: 0, endPos: 1, isUnparsedRun: undefined }, + { surface: 'ほ', headword: '帆', startPos: 1, endPos: 2, isUnparsedRun: undefined }, + // The unmatched っ + space becomes a hoverable filler run, replacing the + // parseText filler chunks the pipeline used to rely on. + { surface: 'っ ', headword: 'っ ', startPos: 2, endPos: 4, isUnparsedRun: true }, + { surface: 'ミナト', headword: 'ミナト', startPos: 4, endPos: 7, isUnparsedRun: undefined }, + ], + ); + assert.equal(result?.[2]?.reading, ''); }); test('requestYomitanScanTokens extracts best frequency rank from selected termsFind entry', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { return { profileCurrent: 0, - profileIndex: 0, - scanLength: 40, - dictionaries: ['JPDBv2㋕', 'Jiten', 'CC100'], - dictionaryPriorityByName: { - 'JPDBv2㋕': 0, - Jiten: 1, - CC100: 2, - }, - dictionaryFrequencyModeByName: { - 'JPDBv2㋕': 'rank-based', - Jiten: 'rank-based', - CC100: 'rank-based', - }, profiles: [ { options: { @@ -1037,14 +881,9 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF ], }; } - return null; - }); - - await requestYomitanScanTokens('潜み', deps, { - error: () => undefined, - }); - - const result = await runInjectedYomitanScript(scannerScript, (action, params) => { + if (action === 'getDictionaryInfo') { + return []; + } if (action !== 'termsFind') { throw new Error(`unexpected action: ${action}`); } @@ -1090,6 +929,10 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF }; }); + const result = await requestYomitanScanTokens('潜み', deps, { + error: () => undefined, + }); + assert.deepEqual(result, [ { surface: '潜み', @@ -1105,20 +948,10 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF }); test('requestYomitanScanTokens retries shorter windows when a greedy match has no exact-source headword', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { return { profileCurrent: 0, - profileIndex: 0, - scanLength: 40, - dictionaries: ['JMdict'], - dictionaryPriorityByName: { JMdict: 0 }, - dictionaryFrequencyModeByName: {}, profiles: [ { options: { @@ -1129,14 +962,9 @@ test('requestYomitanScanTokens retries shorter windows when a greedy match has n ], }; } - return null; - }); - - await requestYomitanScanTokens('平 （平）', deps, { - error: () => undefined, - }); - - const result = await runInjectedYomitanScript(scannerScript, (action, params) => { + if (action === 'getDictionaryInfo') { + return []; + } if (action !== 'termsFind') { throw new Error(`unexpected action: ${action}`); } @@ -1179,6 +1007,10 @@ test('requestYomitanScanTokens retries shorter windows when a greedy match has n }; }); + const result = await requestYomitanScanTokens('平 （平）', deps, { + error: () => undefined, + }); + assert.deepEqual(result, [ { surface: '平', @@ -1204,13 +1036,8 @@ test('requestYomitanScanTokens retries shorter windows when a greedy match has n }); test('requestYomitanScanTokens emits complete readings for kanji-kana compounds', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { return { profileCurrent: 0, profiles: [ @@ -1223,14 +1050,9 @@ test('requestYomitanScanTokens emits complete readings for kanji-kana compounds' ], }; } - return null; - }); - - await requestYomitanScanTokens('待ち合わせてる', deps, { - error: () => undefined, - }); - - const result = await runInjectedYomitanScript(scannerScript, (action, params) => { + if (action === 'getDictionaryInfo') { + return []; + } if (action !== 'termsFind') { throw new Error(`unexpected action: ${action}`); } @@ -1256,6 +1078,10 @@ test('requestYomitanScanTokens emits complete readings for kanji-kana compounds' }; }); + const result = await requestYomitanScanTokens('待ち合わせてる', deps, { + error: () => undefined, + }); + assert.deepEqual(result, [ { surface: '待ち合わせてる', @@ -1271,28 +1097,10 @@ test('requestYomitanScanTokens emits complete readings for kanji-kana compounds' }); test('requestYomitanScanTokens uses frequency from later exact-match entry when first exact entry has none', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { return { profileCurrent: 0, - profileIndex: 0, - scanLength: 40, - dictionaries: ['JPDBv2㋕', 'Jiten', 'CC100'], - dictionaryPriorityByName: { - 'JPDBv2㋕': 0, - Jiten: 1, - CC100: 2, - }, - dictionaryFrequencyModeByName: { - 'JPDBv2㋕': 'rank-based', - Jiten: 'rank-based', - CC100: 'rank-based', - }, profiles: [ { options: { @@ -1307,14 +1115,9 @@ test('requestYomitanScanTokens uses frequency from later exact-match entry when ], }; } - return null; - }); - - await requestYomitanScanTokens('者', deps, { - error: () => undefined, - }); - - const result = await runInjectedYomitanScript(scannerScript, (action, params) => { + if (action === 'getDictionaryInfo') { + return []; + } if (action !== 'termsFind') { throw new Error(`unexpected action: ${action}`); } @@ -1364,6 +1167,10 @@ test('requestYomitanScanTokens uses frequency from later exact-match entry when }; }); + const result = await requestYomitanScanTokens('者', deps, { + error: () => undefined, + }); + assert.deepEqual(result, [ { surface: '者', @@ -1379,28 +1186,10 @@ test('requestYomitanScanTokens uses frequency from later exact-match entry when }); test('requestYomitanScanTokens can use frequency from later exact secondary-match entry', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { return { profileCurrent: 0, - profileIndex: 0, - scanLength: 40, - dictionaries: ['JPDBv2㋕', 'Jiten', 'CC100'], - dictionaryPriorityByName: { - 'JPDBv2㋕': 0, - Jiten: 1, - CC100: 2, - }, - dictionaryFrequencyModeByName: { - 'JPDBv2㋕': 'rank-based', - Jiten: 'rank-based', - CC100: 'rank-based', - }, profiles: [ { options: { @@ -1415,14 +1204,9 @@ test('requestYomitanScanTokens can use frequency from later exact secondary-matc ], }; } - return null; - }); - - await requestYomitanScanTokens('者', deps, { - error: () => undefined, - }); - - const result = await runInjectedYomitanScript(scannerScript, (action, params) => { + if (action === 'getDictionaryInfo') { + return []; + } if (action !== 'termsFind') { throw new Error(`unexpected action: ${action}`); } @@ -1466,6 +1250,10 @@ test('requestYomitanScanTokens can use frequency from later exact secondary-matc }; }); + const result = await requestYomitanScanTokens('者', deps, { + error: () => undefined, + }); + assert.deepEqual(result, [ { surface: '者', @@ -1481,28 +1269,10 @@ test('requestYomitanScanTokens can use frequency from later exact secondary-matc }); test('requestYomitanScanTokens uses exact frequency entry when selected reading differs', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { return { profileCurrent: 0, - profileIndex: 0, - scanLength: 40, - dictionaries: ['JPDBv2㋕', 'Jiten', 'CC100'], - dictionaryPriorityByName: { - 'JPDBv2㋕': 0, - Jiten: 1, - CC100: 2, - }, - dictionaryFrequencyModeByName: { - 'JPDBv2㋕': 'rank-based', - Jiten: 'rank-based', - CC100: 'rank-based', - }, profiles: [ { options: { @@ -1517,14 +1287,9 @@ test('requestYomitanScanTokens uses exact frequency entry when selected reading ], }; } - return null; - }); - - await requestYomitanScanTokens('第二走者', deps, { - error: () => undefined, - }); - - const result = (await runInjectedYomitanScript(scannerScript, (action, params) => { + if (action === 'getDictionaryInfo') { + return []; + } if (action !== 'termsFind') { throw new Error(`unexpected action: ${action}`); } @@ -1566,7 +1331,11 @@ test('requestYomitanScanTokens uses exact frequency entry when selected reading }, ], }; - })) as Array>; + }); + + const result = await requestYomitanScanTokens('第二走者', deps, { + error: () => undefined, + }); assert.deepEqual(result?.[0], { surface: '第二', @@ -1625,10 +1394,10 @@ test('requestYomitanScanTokens marks tokens backed by SubMiner character diction }); test('requestYomitanScanTokens skips name-match work when disabled', async () => { - let scannerScript = ''; + let scanCallScript = ''; const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; + if (script.includes('__subminerYomitanScan(')) { + scanCallScript = script; } if (script.includes('optionsGetFull')) { return { @@ -1663,72 +1432,69 @@ test('requestYomitanScanTokens skips name-match work when disabled', async () => assert.equal(result?.length, 1); assert.equal((result?.[0] as { isNameMatch?: boolean } | undefined)?.isNameMatch, undefined); - assert.match(scannerScript, /const includeNameMatchMetadata = false;/); + assert.match(scanCallScript, /"includeNameMatchMetadata":false/); }); test('requestYomitanScanTokens marks grouped entries when SubMiner dictionary alias only exists on definitions', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { - return { - profileCurrent: 0, - profiles: [ - { - options: { - scanning: { length: 40 }, + const scripts: string[] = []; + const deps = createScanDeps( + (action, params) => { + if (action === 'optionsGetFull') { + return { + profileCurrent: 0, + profiles: [ + { + options: { + scanning: { length: 40 }, + }, }, - }, - ], - }; - } - return null; - }); + ], + }; + } + if (action === 'getDictionaryInfo') { + return []; + } + if (action === 'termsFind') { + const text = (params as { text?: string } | undefined)?.text; + if (text === 'カズマ') { + return { + originalTextLength: 3, + dictionaryEntries: [ + { + dictionaryAlias: '', + headwords: [ + { + term: 'カズマ', + reading: 'かずま', + sources: [{ originalText: 'カズマ', isPrimary: true, matchType: 'exact' }], + }, + ], + definitions: [ + { dictionary: 'JMdict', dictionaryAlias: 'JMdict' }, + { + dictionary: 'SubMiner Character Dictionary (AniList 130298)', + dictionaryAlias: 'SubMiner Character Dictionary (AniList 130298)', + }, + ], + }, + ], + }; + } + return { originalTextLength: 0, dictionaryEntries: [] }; + } + throw new Error(`unexpected action: ${action}`); + }, + { onScript: (script) => scripts.push(script) }, + ); - await requestYomitanScanTokens( + const result = await requestYomitanScanTokens( 'カズマ', deps, { error: () => undefined }, { includeNameMatchMetadata: true }, ); - assert.match(scannerScript, /getPreferredHeadword/); - - const result = await runInjectedYomitanScript(scannerScript, (action, params) => { - if (action === 'termsFind') { - const text = (params as { text?: string } | undefined)?.text; - if (text === 'カズマ') { - return { - originalTextLength: 3, - dictionaryEntries: [ - { - dictionaryAlias: '', - headwords: [ - { - term: 'カズマ', - reading: 'かずま', - sources: [{ originalText: 'カズマ', isPrimary: true, matchType: 'exact' }], - }, - ], - definitions: [ - { dictionary: 'JMdict', dictionaryAlias: 'JMdict' }, - { - dictionary: 'SubMiner Character Dictionary (AniList 130298)', - dictionaryAlias: 'SubMiner Character Dictionary (AniList 130298)', - }, - ], - }, - ], - }; - } - return { originalTextLength: 0, dictionaryEntries: [] }; - } - throw new Error(`unexpected action: ${action}`); - }); - + assert.ok(scripts.some((script) => script.includes('getPreferredHeadword'))); assert.equal(Array.isArray(result), true); assert.equal((result as { length?: number } | null)?.length, 1); assert.equal((result as Array<{ surface?: string }>)[0]?.surface, 'カズマ'); @@ -1739,13 +1505,8 @@ test('requestYomitanScanTokens marks grouped entries when SubMiner dictionary al }); test('requestYomitanScanTokens ignores SubMiner character entries from other media', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { return { profileCurrent: 0, profiles: [ @@ -1757,17 +1518,9 @@ test('requestYomitanScanTokens ignores SubMiner character entries from other med ], }; } - return null; - }); - - await requestYomitanScanTokens( - 'カズ', - deps, - { error: () => undefined }, - { includeNameMatchMetadata: true, currentCharacterDictionaryMediaId: 21202 }, - ); - - const result = await runInjectedYomitanScript(scannerScript, (action, params) => { + if (action === 'getDictionaryInfo') { + return []; + } if (action !== 'termsFind') { throw new Error(`unexpected action: ${action}`); } @@ -1807,17 +1560,21 @@ test('requestYomitanScanTokens ignores SubMiner character entries from other med }; }); - assert.deepEqual(result, []); + const result = await requestYomitanScanTokens( + 'カズ', + deps, + { error: () => undefined }, + { includeNameMatchMetadata: true, currentCharacterDictionaryMediaId: 21202 }, + ); + + // No dictionary-backed token survives (the only match belongs to another + // media's character dictionary), so the line reports no tokenization. + assert.equal(result, null); }); test('requestYomitanScanTokens accepts SubMiner character entries with structured-content media data', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { return { profileCurrent: 0, profiles: [ @@ -1829,17 +1586,9 @@ test('requestYomitanScanTokens accepts SubMiner character entries with structure ], }; } - return null; - }); - - await requestYomitanScanTokens( - 'アクア', - deps, - { error: () => undefined }, - { includeNameMatchMetadata: true, currentCharacterDictionaryMediaId: 21699 }, - ); - - const result = await runInjectedYomitanScript(scannerScript, (action, params) => { + if (action === 'getDictionaryInfo') { + return []; + } if (action !== 'termsFind') { throw new Error(`unexpected action: ${action}`); } @@ -1885,46 +1634,20 @@ test('requestYomitanScanTokens accepts SubMiner character entries with structure }; }); + const result = await requestYomitanScanTokens( + 'アクア', + deps, + { error: () => undefined }, + { includeNameMatchMetadata: true, currentCharacterDictionaryMediaId: 21699 }, + ); + assert.equal(Array.isArray(result), true); assert.equal((result as Array<{ surface?: string }>)[0]?.surface, 'アクア'); assert.equal((result as Array<{ isNameMatch?: boolean }>)[0]?.isNameMatch, true); }); test('requestYomitanScanTokens greedily tokenizes character names before longer generic matches', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { - return { - profileCurrent: 0, - profiles: [ - { - options: { - scanning: { length: 40 }, - dictionaries: [ - { name: 'JMdict', enabled: true }, - { name: 'SubMiner Character Dictionary (AniList 130298)', enabled: true }, - ], - }, - }, - ], - }; - } - return null; - }); - - await requestYomitanScanTokens( - '美姫とヨータ', - deps, - { error: () => undefined }, - { includeNameMatchMetadata: true }, - ); - - assert.match(scannerScript, /const greedyNameScanEnabled = true;/); - + let scanCallScript = ''; const nameEntry = (term: string, reading: string) => ({ headwords: [ { @@ -1951,42 +1674,79 @@ test('requestYomitanScanTokens greedily tokenizes character names before longer definitions: [{ dictionary: 'JMdict', dictionaryAlias: 'JMdict' }], }); - const result = await runInjectedYomitanScript(scannerScript, (action, params) => { - if (action !== 'termsFind') { - throw new Error(`unexpected action: ${action}`); - } - const text = (params as { text?: string } | undefined)?.text ?? ''; - if (text.startsWith('美姫')) { - return { originalTextLength: 2, dictionaryEntries: [nameEntry('美姫', 'みき')] }; - } - if (text.startsWith('とヨータ')) { - // Greedy generic match: とヨー normalizes to とよう (渡洋). Without the - // name pre-pass this consumes the ヨ of ヨータ. - return { - originalTextLength: 3, - dictionaryEntries: [jmdictEntry('渡洋', 'とよう', 'とヨー'), jmdictEntry('と', 'と', 'と')], - }; - } - if (text.startsWith('ヨータ')) { - return { originalTextLength: 3, dictionaryEntries: [nameEntry('ヨータ', 'よーた')] }; - } - if (text === 'と') { - return { originalTextLength: 1, dictionaryEntries: [jmdictEntry('と', 'と', 'と')] }; - } - return { originalTextLength: 0, dictionaryEntries: [] }; - }); + const deps = createScanDeps( + (action, params) => { + if (action === 'optionsGetFull') { + return { + profileCurrent: 0, + profiles: [ + { + options: { + scanning: { length: 40 }, + dictionaries: [ + { name: 'JMdict', enabled: true }, + { name: 'SubMiner Character Dictionary (AniList 130298)', enabled: true }, + ], + }, + }, + ], + }; + } + if (action === 'getDictionaryInfo') { + return []; + } + if (action !== 'termsFind') { + throw new Error(`unexpected action: ${action}`); + } + const text = (params as { text?: string } | undefined)?.text ?? ''; + if (text.startsWith('美姫')) { + return { originalTextLength: 2, dictionaryEntries: [nameEntry('美姫', 'みき')] }; + } + if (text.startsWith('とヨータ')) { + // Greedy generic match: とヨー normalizes to とよう (渡洋). Without the + // name pre-pass this consumes the ヨ of ヨータ. + return { + originalTextLength: 3, + dictionaryEntries: [ + jmdictEntry('渡洋', 'とよう', 'とヨー'), + jmdictEntry('と', 'と', 'と'), + ], + }; + } + if (text.startsWith('ヨータ')) { + return { originalTextLength: 3, dictionaryEntries: [nameEntry('ヨータ', 'よーた')] }; + } + if (text === 'と') { + return { originalTextLength: 1, dictionaryEntries: [jmdictEntry('と', 'と', 'と')] }; + } + return { originalTextLength: 0, dictionaryEntries: [] }; + }, + { + onScript: (script) => { + if (script.includes('__subminerYomitanScan(')) { + scanCallScript = script; + } + }, + }, + ); + const result = await requestYomitanScanTokens( + '美姫とヨータ', + deps, + { error: () => undefined }, + { includeNameMatchMetadata: true }, + ); + + assert.match(scanCallScript, /"greedyNameScanEnabled":true/); assert.equal(Array.isArray(result), true); assert.deepEqual( - (result as Array>).map( - ({ surface, headword, startPos, endPos, isNameMatch }) => ({ - surface, - headword, - startPos, - endPos, - isNameMatch, - }), - ), + result?.map(({ surface, headword, startPos, endPos, isNameMatch }) => ({ + surface, + headword, + startPos, + endPos, + isNameMatch, + })), [ { surface: '美姫', headword: '美姫', startPos: 0, endPos: 2, isNameMatch: true }, { surface: 'と', headword: 'と', startPos: 2, endPos: 3, isNameMatch: false }, @@ -1996,40 +1756,6 @@ test('requestYomitanScanTokens greedily tokenizes character names before longer }); test('requestYomitanScanTokens lets a longer generic word beat a shorter name at the same position', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { - return { - profileCurrent: 0, - profiles: [ - { - options: { - scanning: { length: 40 }, - dictionaries: [ - { name: 'JMdict', enabled: true }, - { name: 'SubMiner Character Dictionary (AniList 130298)', enabled: true }, - ], - }, - }, - ], - }; - } - return null; - }); - - await requestYomitanScanTokens( - '空気変わって', - deps, - { error: () => undefined }, - { includeNameMatchMetadata: true }, - ); - - assert.match(scannerScript, /const greedyNameScanEnabled = true;/); - const nameEntry = (term: string, reading: string) => ({ headwords: [ { @@ -2056,7 +1782,26 @@ test('requestYomitanScanTokens lets a longer generic word beat a shorter name at definitions: [{ dictionary: 'JMdict', dictionaryAlias: 'JMdict' }], }); - const result = await runInjectedYomitanScript(scannerScript, (action, params) => { + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { + return { + profileCurrent: 0, + profiles: [ + { + options: { + scanning: { length: 40 }, + dictionaries: [ + { name: 'JMdict', enabled: true }, + { name: 'SubMiner Character Dictionary (AniList 130298)', enabled: true }, + ], + }, + }, + ], + }; + } + if (action === 'getDictionaryInfo') { + return []; + } if (action !== 'termsFind') { throw new Error(`unexpected action: ${action}`); } @@ -2078,17 +1823,22 @@ test('requestYomitanScanTokens lets a longer generic word beat a shorter name at return { originalTextLength: 0, dictionaryEntries: [] }; }); + const result = await requestYomitanScanTokens( + '空気変わって', + deps, + { error: () => undefined }, + { includeNameMatchMetadata: true }, + ); + assert.equal(Array.isArray(result), true); assert.deepEqual( - (result as Array>).map( - ({ surface, headword, startPos, endPos, isNameMatch }) => ({ - surface, - headword, - startPos, - endPos, - isNameMatch, - }), - ), + result?.map(({ surface, headword, startPos, endPos, isNameMatch }) => ({ + surface, + headword, + startPos, + endPos, + isNameMatch, + })), [ { surface: '空気', headword: '空気', startPos: 0, endPos: 2, isNameMatch: false }, { surface: '変わって', headword: '変わる', startPos: 2, endPos: 6, isNameMatch: false }, @@ -2097,27 +1847,35 @@ test('requestYomitanScanTokens lets a longer generic word beat a shorter name at }); test('requestYomitanScanTokens skips greedy name scan without an enabled character dictionary', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { - return { - profileCurrent: 0, - profiles: [ - { - options: { - scanning: { length: 40 }, - dictionaries: [{ name: 'JMdict', enabled: true }], + let scanCallScript = ''; + const deps = createScanDeps( + (action) => { + if (action === 'optionsGetFull') { + return { + profileCurrent: 0, + profiles: [ + { + options: { + scanning: { length: 40 }, + dictionaries: [{ name: 'JMdict', enabled: true }], + }, }, - }, - ], - }; - } - return null; - }); + ], + }; + } + if (action === 'getDictionaryInfo') { + return []; + } + return { originalTextLength: 0, dictionaryEntries: [] }; + }, + { + onScript: (script) => { + if (script.includes('__subminerYomitanScan(')) { + scanCallScript = script; + } + }, + }, + ); await requestYomitanScanTokens( 'アクア', @@ -2126,136 +1884,12 @@ test('requestYomitanScanTokens skips greedy name scan without an enabled charact { includeNameMatchMetadata: true }, ); - assert.match(scannerScript, /const greedyNameScanEnabled = false;/); -}); - -test('requestYomitanScanTokens replaces parseText segmentation where greedy name tokens re-segment', async () => { - const deps = createDeps(async (script) => { - if (script.includes('optionsGetFull')) { - return { - profileCurrent: 0, - profiles: [ - { - options: { - scanning: { length: 40 }, - dictionaries: [ - { name: 'JMdict', enabled: true }, - { name: 'SubMiner Character Dictionary (AniList 130298)', enabled: true }, - ], - }, - }, - ], - }; - } - if (script.includes('parseText')) { - // parseText walks greedily too, so it merges と with ヨー into 渡洋 and - // strands the タ. - return [ - { - source: 'scanning-parser', - index: 0, - content: [ - [ - { - text: '美姫', - reading: 'みき', - headwords: [[{ term: '美姫' }]], - }, - ], - [ - { - text: 'とヨー', - reading: 'とよう', - headwords: [[{ term: '渡洋' }]], - }, - ], - [ - { - text: 'タ', - reading: '', - }, - ], - ], - }, - ]; - } - return [ - { - surface: '美姫', - reading: 'みき', - headword: '美姫', - headwordReading: 'みき', - startPos: 0, - endPos: 2, - isNameMatch: true, - }, - { - surface: 'と', - reading: 'と', - headword: 'と', - headwordReading: 'と', - startPos: 2, - endPos: 3, - isNameMatch: false, - }, - { - surface: 'ヨータ', - reading: 'ヨータ', - headword: 'ヨータ', - headwordReading: 'よーた', - startPos: 3, - endPos: 6, - isNameMatch: true, - }, - ]; - }); - - const result = await requestYomitanScanTokens( - '美姫とヨータ', - deps, - { error: () => undefined }, - { includeNameMatchMetadata: true }, - ); - - assert.deepEqual(result, [ - { - surface: '美姫', - reading: 'みき', - headword: '美姫', - headwordReading: 'みき', - startPos: 0, - endPos: 2, - isNameMatch: true, - }, - { - surface: 'と', - reading: 'と', - headword: 'と', - headwordReading: 'と', - startPos: 2, - endPos: 3, - isNameMatch: false, - }, - { - surface: 'ヨータ', - reading: 'ヨータ', - headword: 'ヨータ', - headwordReading: 'よーた', - startPos: 3, - endPos: 6, - isNameMatch: true, - }, - ]); + assert.match(scanCallScript, /"greedyNameScanEnabled":false/); }); test('requestYomitanScanTokens preserves matched headword word classes', async () => { - let scannerScript = ''; - const deps = createDeps(async (script) => { - if (script.includes('termsFind')) { - scannerScript = script; - return []; - } - if (script.includes('optionsGetFull')) { + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { return { profileCurrent: 0, profiles: [ @@ -2267,12 +1901,9 @@ test('requestYomitanScanTokens preserves matched headword word classes', async ( ], }; } - return null; - }); - - await requestYomitanScanTokens('は', deps, { error: () => undefined }); - - const result = await runInjectedYomitanScript(scannerScript, (action, params) => { + if (action === 'getDictionaryInfo') { + return []; + } if (action !== 'termsFind') { throw new Error(`unexpected action: ${action}`); } @@ -2299,12 +1930,14 @@ test('requestYomitanScanTokens preserves matched headword word classes', async ( }; }); + const result = await requestYomitanScanTokens('は', deps, { error: () => undefined }); + assert.deepEqual((result as Array<{ wordClasses?: string[] }>)[0]?.wordClasses, ['prt']); }); test('requestYomitanScanTokens skips fallback fragments without exact primary source matches', async () => { - const deps = createDeps(async (script) => { - if (script.includes('optionsGetFull')) { + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { return { profileCurrent: 0, profiles: [ @@ -2316,12 +1949,14 @@ test('requestYomitanScanTokens skips fallback fragments without exact primary so ], }; } + if (action === 'getDictionaryInfo') { + return []; + } + if (action !== 'termsFind') { + throw new Error(`unexpected action: ${action}`); + } - return await runInjectedYomitanScript(script, (action, params) => { - if (action !== 'termsFind') { - throw new Error(`unexpected action: ${action}`); - } - + { const text = (params as { text?: string } | undefined)?.text ?? ''; if (text.startsWith('だが ')) { return { @@ -2420,7 +2055,7 @@ test('requestYomitanScanTokens skips fallback fragments without exact primary so }; } return { originalTextLength: 0, dictionaryEntries: [] }; - }); + } }); const result = await requestYomitanScanTokens('だが それでも届かぬ高みがあった', deps, { @@ -2459,6 +2094,14 @@ test('requestYomitanScanTokens skips fallback fragments without exact primary so startPos: 10, endPos: 12, }, + // が has no exact primary source match, so it survives only as an + // unparsed filler run (the parseText segmentation used to supply this). + { + surface: 'が', + headword: 'が', + startPos: 12, + endPos: 13, + }, { surface: 'あった', headword: 'ある', @@ -2467,6 +2110,170 @@ test('requestYomitanScanTokens skips fallback fragments without exact primary so }, ], ); + assert.equal(result?.[4]?.isUnparsedRun, true); +}); + +function createSingleTermScanHandler(lookups: string[]) { + return (action: string, params: unknown): unknown => { + if (action === 'optionsGetFull') { + return { + profileCurrent: 0, + profiles: [{ options: { scanning: { length: 40 } } }], + }; + } + if (action === 'getDictionaryInfo') { + return []; + } + if (action !== 'termsFind') { + throw new Error(`unexpected action: ${action}`); + } + const text = (params as { text?: string } | undefined)?.text ?? ''; + lookups.push(text); + if (text.startsWith('猫')) { + return { + originalTextLength: 1, + dictionaryEntries: [ + { + headwords: [ + { + term: '猫', + reading: 'ねこ', + sources: [{ originalText: '猫', isPrimary: true, matchType: 'exact' }], + }, + ], + }, + ], + }; + } + return { originalTextLength: 0, dictionaryEntries: [] }; + }; +} + +test('requestYomitanScanTokens reuses the cross-line termsFind cache for repeated lookups', async () => { + const lookups: string[] = []; + const deps = createScanDeps(createSingleTermScanHandler(lookups)); + + const first = await requestYomitanScanTokens('猫', deps, { error: () => undefined }); + const second = await requestYomitanScanTokens('猫', deps, { error: () => undefined }); + + assert.equal(first?.length, 1); + assert.equal(second?.length, 1); + // The second line hits the window-persistent cache: no new backend lookup. + assert.equal(countTermsFindLookups(lookups, '猫'), 1); +}); + +test('clearYomitanParserCachesForWindow invalidates the cross-line termsFind cache', async () => { + const lookups: string[] = []; + const deps = createScanDeps(createSingleTermScanHandler(lookups)); + + await requestYomitanScanTokens('猫', deps, { error: () => undefined }); + clearYomitanParserCachesForWindow(deps.getYomitanParserWindow() as never); + await requestYomitanScanTokens('猫', deps, { error: () => undefined }); + + assert.equal(countTermsFindLookups(lookups, '猫'), 2); +}); + +test('requestYomitanScanTokens skips termsFind lookups at punctuation and whitespace positions', async () => { + const lookups: string[] = []; + const deps = createScanDeps(createSingleTermScanHandler(lookups)); + + const result = await requestYomitanScanTokens('「猫」…♪', deps, { error: () => undefined }); + + assert.equal(result?.length, 1); + assert.equal(result?.[0]?.surface, '猫'); + assert.equal(countTermsFindLookups(lookups, '猫'), 1); + for (const skipped of ['「', '」', '…', '♪']) { + assert.equal(countTermsFindLookups(lookups, skipped), 0, `expected no lookup at ${skipped}`); + } +}); + +test('requestYomitanScanTokens caps the shrinking-window retry ladder per position', async () => { + const lookups: string[] = []; + const deps = createScanDeps((action, params) => { + if (action === 'optionsGetFull') { + return { + profileCurrent: 0, + profiles: [{ options: { scanning: { length: 40 } } }], + }; + } + if (action === 'getDictionaryInfo') { + return []; + } + const text = (params as { text?: string } | undefined)?.text ?? ''; + lookups.push(text); + // Every window "matches" its whole length but never yields an + // exact-source headword, the worst case for the retry ladder. + return { + originalTextLength: text.length, + dictionaryEntries: [ + { + headwords: [ + { + term: 'ミスマッチ', + reading: 'みすまっち', + sources: [{ originalText: 'ZZZ', isPrimary: true, matchType: 'exact' }], + }, + ], + }, + ], + }; + }); + + const result = await requestYomitanScanTokens('あいうえおかきくけこ', deps, { + error: () => undefined, + }); + + assert.equal(result, null); + // Position 0: one initial window lookup plus at most four shrinking retries. + assert.equal(countTermsFindLookups(lookups, 'あいうえお'), 5); +}); + +test('requestYomitanScanTokens falls back to parseText when the scanner eval fails', async () => { + const deps = createDeps(async (script) => { + if (script.includes('optionsGetFull')) { + return { + profileCurrent: 0, + profiles: [{ options: { scanning: { length: 40 } } }], + }; + } + if (script.includes('__subminerYomitanScan(')) { + throw new Error('eval failed'); + } + if (script.includes('parseText')) { + return [ + { + source: 'scanning-parser', + index: 0, + content: [ + [ + { + text: '取り組んで', + reading: 'とりくんで', + headwords: [[{ term: '取り組む' }]], + }, + ], + ], + }, + ]; + } + return null; + }); + + const errors: string[] = []; + const result = await requestYomitanScanTokens('取り組んで', deps, { + error: (message) => errors.push(message), + }); + + assert.deepEqual(result, [ + { + surface: '取り組んで', + reading: 'とりくんで', + headword: '取り組む', + startPos: 0, + endPos: 5, + }, + ]); + assert.equal(errors.length, 1); }); test('getYomitanDictionaryInfo requests dictionary info via backend action', async () => { diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.ts b/src/core/services/tokenizer/yomitan-parser-runtime.ts index 1ee0b8ed..ce6fcddc 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.ts @@ -81,6 +81,13 @@ const yomitanFrequencyCacheByWindow = new WeakMap< BrowserWindow, Map >(); +// Epoch passed with every scan request; the in-window termsFind cache clears +// itself when the epoch changes (dictionary imports, settings changes). +const yomitanScanCacheEpochByWindow = new WeakMap(); + +function getYomitanScanCacheEpoch(window: BrowserWindow): number { + return yomitanScanCacheEpochByWindow.get(window) ?? 0; +} function isObject(value: unknown): value is Record { return Boolean(value && typeof value === 'object'); @@ -99,6 +106,7 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] { typeof entry.startPos === 'number' && typeof entry.endPos === 'number' && (entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean') && + (entry.isUnparsedRun === undefined || typeof entry.isUnparsedRun === 'boolean') && (entry.frequencyRank === undefined || typeof entry.frequencyRank === 'number') && (entry.wordClasses === undefined || (Array.isArray(entry.wordClasses) && @@ -107,13 +115,9 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] { ); } -function scanTokenSpanKey(token: YomitanScanToken): string { - return `${token.startPos}:${token.endPos}:${token.surface}`; -} - // Maps a parse-selected token to the scanner-token shape carried out of the -// parser runtime. Shared by both selectYomitanParseTokens fallback paths so the -// projected fields stay in sync as the shape changes. +// parser runtime, used by the parseText fallback path when the in-window +// scanner is unavailable. function toYomitanScanToken(token: { surface: string; reading: string; @@ -132,66 +136,6 @@ function toYomitanScanToken(token: { }; } -// parseText segmentation is authoritative (it emits filler chunks for text the -// termsFind scanner skips), but only the termsFind scanner carries annotation -// metadata (isNameMatch, frequencyRank, headwordReading, wordClasses). Graft -// scanner tokens onto the parseText segmentation per matching span so one -// unmatched chunk degrades only itself instead of dropping the whole line's -// metadata. -// -// Exception: character-name tokens. The greedy name scan can re-segment text -// around a name (e.g. とヨータ → と + ヨータ instead of とヨー + タ), so -// parseText segmentation cannot be authoritative there. Each name span is -// expanded until it aligns with token boundaries in both segmentations, then -// the parse tokens inside are replaced with the scanner tokens. -function mergeScannerTokensIntoParseTokens( - parseScanTokens: YomitanScanToken[], - scannerTokens: YomitanScanToken[], -): YomitanScanToken[] { - const scannerTokensBySpan = new Map(); - for (const token of scannerTokens) { - scannerTokensBySpan.set(scanTokenSpanKey(token), token); - } - const graftedTokens = parseScanTokens.map( - (token) => scannerTokensBySpan.get(scanTokenSpanKey(token)) ?? token, - ); - - const nameTokens = scannerTokens.filter((token) => token.isNameMatch === true); - if (nameTokens.length === 0) { - return graftedTokens; - } - - const regions = nameTokens.map((token) => ({ start: token.startPos, end: token.endPos })); - const allTokens = [...parseScanTokens, ...scannerTokens]; - let expanded = true; - while (expanded) { - expanded = false; - for (const region of regions) { - for (const token of allTokens) { - const overlaps = token.startPos < region.end && token.endPos > region.start; - const extendsBeyond = token.startPos < region.start || token.endPos > region.end; - if (overlaps && extendsBeyond) { - region.start = Math.min(region.start, token.startPos); - region.end = Math.max(region.end, token.endPos); - expanded = true; - } - } - } - } - - const isInsideNameRegion = (token: YomitanScanToken): boolean => - regions.some((region) => token.startPos >= region.start && token.endPos <= region.end); - - const merged = graftedTokens.filter((token) => !isInsideNameRegion(token)); - for (const token of scannerTokens) { - if (isInsideNameRegion(token)) { - merged.push(token); - } - } - merged.sort((a, b) => a.startPos - b.startPos || a.endPos - b.endPos); - return merged; -} - function makeTermReadingCacheKey(term: string, reading: string | null): string { return `${term}\u0000${reading ?? ''}`; } @@ -208,6 +152,7 @@ function getWindowFrequencyCache(window: BrowserWindow): Map {}); return true; } catch (err) { @@ -1362,58 +1311,144 @@ const YOMITAN_SCANNING_HELPERS = String.raw` } `; -function buildYomitanScanningScript( - text: string, - profileIndex: number, - scanLength: number, - includeNameMatchMetadata: boolean, - greedyNameScanEnabled: boolean, - currentCharacterDictionaryMediaId: number | null, - dictionaryPriorityByName: Record, - dictionaryFrequencyModeByName: Partial>, -): string { - return ` - (async () => { - const invoke = (action, params) => - new Promise((resolve, reject) => { - chrome.runtime.sendMessage({ action, params }, (response) => { - if (chrome.runtime.lastError) { - reject(new Error(chrome.runtime.lastError.message)); - return; - } - if (!response || typeof response !== "object") { - reject(new Error("Invalid response from Yomitan backend")); - return; - } - if (response.error) { - reject(new Error(response.error.message || "Yomitan backend error")); - return; - } - resolve(response.result); - }); +// Bump whenever the install script below changes so already-loaded parser +// windows re-install the new scan runtime instead of running the stale one. +const YOMITAN_SCAN_RUNTIME_VERSION = 1; +const YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL = '__subminer-yomitan-scan-runtime-missing__'; + +interface YomitanScanRequestParams { + text: string; + profileIndex: number; + scanLength: number; + includeNameMatchMetadata: boolean; + greedyNameScanEnabled: boolean; + currentCharacterDictionaryMediaId: number | null; + dictionaryPriorityByName: Record; + dictionaryFrequencyModeByName: Partial>; + cacheEpoch: number; +} + +// Installed once per parser window (and re-installed after in-page reloads): +// keeps V8 from re-parsing the helper bundle on every subtitle line, and hosts +// the cross-line termsFind cache. Each subtitle line then only evaluates a tiny +// call into globalThis.__subminerYomitanScan. +const YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT = String.raw` + (() => { + if (globalThis.__subminerYomitanScanVersion === ${YOMITAN_SCAN_RUNTIME_VERSION}) { + return true; + } + const invoke = (action, params) => + new Promise((resolve, reject) => { + chrome.runtime.sendMessage({ action, params }, (response) => { + if (chrome.runtime.lastError) { + reject(new Error(chrome.runtime.lastError.message)); + return; + } + if (!response || typeof response !== "object") { + reject(new Error("Invalid response from Yomitan backend")); + return; + } + if (response.error) { + reject(new Error(response.error.message || "Yomitan backend error")); + return; + } + resolve(response.result); }); + }); + // Cross-line termsFind LRU keyed by profile + substring: subtitle lines + // repeat particles and inflections constantly, so most lookups hit here. + // Entries hold in-flight promises so concurrent identical lookups dedupe. + const termsFindCache = new Map(); + const TERMS_FIND_CACHE_LIMIT = 2000; + let termsFindCacheEpoch = -1; + const MAX_SHRINKING_WINDOW_RETRY_LOOKUPS = 4; + globalThis.__subminerYomitanScanVersion = ${YOMITAN_SCAN_RUNTIME_VERSION}; + globalThis.__subminerYomitanScan = async (scanParams) => { + const { + text, + profileIndex, + scanLength, + includeNameMatchMetadata, + greedyNameScanEnabled, + currentCharacterDictionaryMediaId, + dictionaryPriorityByName, + dictionaryFrequencyModeByName, + cacheEpoch + } = scanParams; + if (cacheEpoch !== termsFindCacheEpoch) { + termsFindCache.clear(); + termsFindCacheEpoch = cacheEpoch; + } ${YOMITAN_SCANNING_HELPERS} - const includeNameMatchMetadata = ${includeNameMatchMetadata ? 'true' : 'false'}; - const greedyNameScanEnabled = ${greedyNameScanEnabled ? 'true' : 'false'}; - const currentCharacterDictionaryMediaId = ${ - currentCharacterDictionaryMediaId !== null - ? String(currentCharacterDictionaryMediaId) - : 'null' - }; - const dictionaryPriorityByName = ${JSON.stringify(dictionaryPriorityByName)}; - const dictionaryFrequencyModeByName = ${JSON.stringify(dictionaryFrequencyModeByName)}; - const text = ${JSON.stringify(text)}; + const CAPTION_OPENING_BRACKETS = new Set(["(", "（", "[", "［", "{", "｛", "「", "『", "【", "〈", "《", "≪", "＜", "<"]); + function shouldEmitUnparsedRunAsToken(runText) { + if (!/[\p{L}\p{N}]/u.test(runText)) { return false; } + const firstChar = Array.from(runText.trim())[0]; + return firstChar !== undefined && !CAPTION_OPENING_BRACKETS.has(firstChar); + } + function isLookupWorthyCodePoint(codePoint) { + if (isCodePointJapanese(codePoint)) { return true; } + return /[\p{L}\p{N}]/u.test(String.fromCodePoint(codePoint)); + } + function isKanaOnlyRunText(runText) { + const chars = Array.from(runText); + return chars.length > 0 && chars.every((char) => isCodePointKana(char.codePointAt(0))); + } const details = {matchType: "exact", deinflect: true}; const tokens = []; - const termsFindCache = new Map(); async function termsFindAt(position, windowLength) { - const cacheKey = position + ":" + windowLength; - const cached = termsFindCache.get(cacheKey); - if (cached) { return cached; } const substring = text.substring(position, position + windowLength); - const result = await invoke("termsFind", { text: substring, details, optionsContext: { index: ${profileIndex} } }); - termsFindCache.set(cacheKey, result); - return result; + const cacheKey = profileIndex + "" + substring; + const cached = termsFindCache.get(cacheKey); + if (cached !== undefined) { + termsFindCache.delete(cacheKey); + termsFindCache.set(cacheKey, cached); + return await cached; + } + const pending = invoke("termsFind", { text: substring, details, optionsContext: { index: profileIndex } }); + termsFindCache.set(cacheKey, pending); + while (termsFindCache.size > TERMS_FIND_CACHE_LIMIT) { + const oldestKey = termsFindCache.keys().next().value; + if (oldestKey === undefined) { break; } + termsFindCache.delete(oldestKey); + } + try { + return await pending; + } catch (error) { + termsFindCache.delete(cacheKey); + throw error; + } + } + // Text the walk skips accumulates into unparsed runs, mirroring the + // filler chunks the parseText segmentation used to provide: runs stay + // hoverable (flagged isUnparsedRun) unless they are punctuation-only or + // caption-style asides, and kana continuations of a longer headword + // extend the previous token instead. + function flushUnparsedRun(runStart, runEnd) { + if (runStart === null || runEnd <= runStart) { return; } + const runText = text.substring(runStart, runEnd); + const previousToken = tokens[tokens.length - 1]; + if ( + previousToken && + previousToken.endPos === runStart && + isKanaOnlyRunText(runText) && + typeof previousToken.headword === "string" && + previousToken.headword.length > previousToken.surface.length && + previousToken.headword.startsWith(previousToken.surface + runText) + ) { + previousToken.surface += runText; + previousToken.endPos = runEnd; + return; + } + if (!shouldEmitUnparsedRunAsToken(runText)) { return; } + tokens.push({ + surface: runText, + reading: "", + headword: runText, + startPos: runStart, + endPos: runEnd, + isUnparsedRun: true + }); } function buildScanToken(position, source, preferredHeadword) { const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : ""; @@ -1469,9 +1504,9 @@ ${YOMITAN_SCANNING_HELPERS} namePos += String.fromCodePoint(codePoint).length; continue; } - const result = await termsFindAt(namePos, ${scanLength}); + const result = await termsFindAt(namePos, scanLength); const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : []; - const textWindow = text.substring(namePos, namePos + ${scanLength}); + const textWindow = text.substring(namePos, namePos + scanLength); const nameMatch = findLongestNameMatch(dictionaryEntries, textWindow); // A name only claims its span when no strictly longer generic word // starts at the same position (a character named 空 must not split @@ -1502,26 +1537,42 @@ ${YOMITAN_SCANNING_HELPERS} } let i = 0; let nameIndex = 0; + let unparsedRunStart = null; while (i < text.length) { while (nameIndex < nameTokens.length && nameTokens[nameIndex].startPos < i) { nameIndex += 1; } const nextNameToken = nameIndex < nameTokens.length ? nameTokens[nameIndex] : null; if (nextNameToken && nextNameToken.startPos === i) { + flushUnparsedRun(unparsedRunStart, i); + unparsedRunStart = null; tokens.push(nextNameToken); i = nextNameToken.endPos; nameIndex += 1; continue; } + const codePoint = text.codePointAt(i); + // Punctuation and whitespace can never start a token: skip the backend + // round trip entirely. Latin letters and digits stay lookup-worthy + // (terms like Tシャツ start on an ASCII letter). + if (!isLookupWorthyCodePoint(codePoint)) { + if (unparsedRunStart === null) { unparsedRunStart = i; } + i += String.fromCodePoint(codePoint).length; + continue; + } // Cap the window at the next reserved name span so a generic match // cannot consume into it. - const windowLength = nextNameToken ? Math.min(${scanLength}, nextNameToken.startPos - i) : ${scanLength}; + const windowLength = nextNameToken ? Math.min(scanLength, nextNameToken.startPos - i) : scanLength; let attempt = await findTokenAt(i, windowLength); // Yomitan text normalization can consume characters (whitespace, // punctuation) beyond the matched term, leaving no headword whose // source equals the consumed text. Retry with shorter windows so a // valid prefix term (e.g. a character name before a paren) still - // tokenizes instead of the position being skipped. + // tokenizes instead of the position being skipped. The ladder is + // capped: without a cap it degrades to O(scanLength) lookups at a + // single position. let retryLength = Math.min(attempt.matchedLength, windowLength) - 1; - while (!attempt.token && retryLength >= 1) { + let retryLookupsRemaining = MAX_SHRINKING_WINDOW_RETRY_LOOKUPS; + while (!attempt.token && retryLength >= 1 && retryLookupsRemaining > 0) { + retryLookupsRemaining -= 1; const retry = await findTokenAt(i, retryLength); if (retry.token) { attempt = retry; @@ -1530,17 +1581,37 @@ ${YOMITAN_SCANNING_HELPERS} retryLength = Math.min(retryLength - 1, retry.matchedLength - 1); } if (attempt.token) { + flushUnparsedRun(unparsedRunStart, i); + unparsedRunStart = null; tokens.push(attempt.token); i += attempt.matchedLength; continue; } + if (unparsedRunStart === null) { unparsedRunStart = i; } i += String.fromCodePoint(text.codePointAt(i)).length; } + flushUnparsedRun(unparsedRunStart, text.length); return tokens; + }; + return true; + })(); +`; + +function buildYomitanScanCallScript(params: YomitanScanRequestParams): string { + return ` + (async () => { + if (typeof globalThis.__subminerYomitanScan !== "function") { + return ${JSON.stringify(YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL)}; + } + return await globalThis.__subminerYomitanScan(${JSON.stringify(params)}); })(); `; } +async function installYomitanScanRuntime(parserWindow: BrowserWindow): Promise { + await parserWindow.webContents.executeJavaScript(YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT, true); +} + export async function requestYomitanParseResults( text: string, deps: YomitanParserRuntimeDeps, @@ -1635,6 +1706,20 @@ export async function requestYomitanParseResults( } } +// parseText fallback for when the in-window scanner cannot run (script eval +// failure, unexpected payload). The scanner walk is the primary tokenizer and +// emits its own filler runs, so this extra full parse only happens on errors. +async function requestYomitanParseFallbackTokens( + text: string, + deps: YomitanParserRuntimeDeps, + logger: LoggerLike, +): Promise { + const parseResults = await requestYomitanParseResults(text, deps, logger); + const selectedTokens = selectYomitanParseTokens(parseResults, () => false, 'headword'); + const parseScanTokens = selectedTokens?.map(toYomitanScanToken) ?? null; + return parseScanTokens && parseScanTokens.length > 0 ? parseScanTokens : null; +} + export async function requestYomitanScanTokens( text: string, deps: YomitanParserRuntimeDeps, @@ -1655,10 +1740,6 @@ export async function requestYomitanScanTokens( return null; } - const parseResults = await requestYomitanParseResults(text, deps, logger); - const selectedParseTokens = selectYomitanParseTokens(parseResults, () => false, 'headword'); - const parseScanTokens = selectedParseTokens?.map(toYomitanScanToken) ?? null; - const metadata = await requestYomitanProfileMetadata(parserWindow, logger); const profileIndex = metadata?.profileIndex ?? 0; const scanLength = metadata?.scanLength ?? DEFAULT_YOMITAN_SCAN_LENGTH; @@ -1669,44 +1750,41 @@ export async function requestYomitanScanTokens( name.startsWith(CHARACTER_DICTIONARY_TITLE_PREFIX), ); + const callScript = buildYomitanScanCallScript({ + text, + profileIndex, + scanLength, + includeNameMatchMetadata, + greedyNameScanEnabled, + currentCharacterDictionaryMediaId: + typeof options?.currentCharacterDictionaryMediaId === 'number' && + Number.isFinite(options.currentCharacterDictionaryMediaId) && + options.currentCharacterDictionaryMediaId > 0 + ? Math.floor(options.currentCharacterDictionaryMediaId) + : null, + dictionaryPriorityByName: metadata?.dictionaryPriorityByName ?? {}, + dictionaryFrequencyModeByName: metadata?.dictionaryFrequencyModeByName ?? {}, + cacheEpoch: getYomitanScanCacheEpoch(parserWindow), + }); + try { - const rawResult = await parserWindow.webContents.executeJavaScript( - buildYomitanScanningScript( - text, - profileIndex, - scanLength, - includeNameMatchMetadata, - greedyNameScanEnabled, - typeof options?.currentCharacterDictionaryMediaId === 'number' && - Number.isFinite(options.currentCharacterDictionaryMediaId) && - options.currentCharacterDictionaryMediaId > 0 - ? Math.floor(options.currentCharacterDictionaryMediaId) - : null, - metadata?.dictionaryPriorityByName ?? {}, - metadata?.dictionaryFrequencyModeByName ?? {}, - ), - true, - ); + let rawResult = await parserWindow.webContents.executeJavaScript(callScript, true); + if (rawResult === YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL) { + // First request for this window, or the page reloaded and dropped the + // installed runtime: install and retry once. + await installYomitanScanRuntime(parserWindow); + rawResult = await parserWindow.webContents.executeJavaScript(callScript, true); + } if (isScanTokenArray(rawResult)) { - if (parseScanTokens && parseScanTokens.length > 0) { - return mergeScannerTokensIntoParseTokens(parseScanTokens, rawResult); - } - return rawResult; + // Filler-only results carry no dictionary match; keep the historical + // contract of returning null so callers fall back to raw text. + return rawResult.some((token) => token.isUnparsedRun !== true) ? rawResult : null; } - if (Array.isArray(rawResult)) { - const selectedTokens = selectYomitanParseTokens(rawResult, () => false, 'headword'); - return selectedTokens?.map(toYomitanScanToken) ?? null; - } - if (parseScanTokens && parseScanTokens.length > 0) { - return parseScanTokens; - } - return null; + logger.error('Yomitan scanner returned an unexpected payload; using parseText fallback.'); + return await requestYomitanParseFallbackTokens(text, deps, logger); } catch (err) { - if (parseScanTokens && parseScanTokens.length > 0) { - return parseScanTokens; - } logger.error('Yomitan scanner request failed:', (err as Error).message); - return null; + return await requestYomitanParseFallbackTokens(text, deps, logger); } } diff --git a/src/main.ts b/src/main.ts index 9b28f23f..e0ac39a1 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1816,7 +1816,7 @@ function withCurrentSubtitleTiming(payload: SubtitleData): SubtitleData { endTime: appState.mpvClient?.currentSubEnd ?? null, }; } -function emitSubtitlePayload(payload: SubtitleData): void { +function emitSubtitlePayload(payload: SubtitleData, options?: { resumePrefetch?: boolean }): void { const timedPayload = withCurrentSubtitleTiming(payload); const currentSubtitleData = appState.currentSubtitleData; const isAnnotationUpgrade = isSubtitleAnnotationUpgrade(currentSubtitleData, timedPayload); @@ -1833,7 +1833,12 @@ function emitSubtitlePayload(payload: SubtitleData): void { } annotationSubtitleWsService.broadcast(timedPayload, frequencyOptions); autoplayReadyGate.maybeSignalPluginAutoplayReady(timedPayload, { forceWhilePaused: true }); - subtitlePrefetchService?.resume(); + // resumePrefetch: false marks a provisional pre-tokenization emit; prefetch + // stays paused until the tokenized payload for the line lands so it does not + // compete with the on-screen line for the single Yomitan parser window. + if (options?.resumePrefetch !== false) { + subtitlePrefetchService?.resume(); + } } function getCurrentAutoplaySubtitlePayload(): SubtitleData | null { const payload = appState.currentSubtitleData; @@ -1926,7 +1931,7 @@ const autoplaySubtitlePrimingRuntime = createAutoplaySubtitlePrimingRuntime({ appState.activeParsedSubtitleMediaPath = mediaPath; }, subtitleProcessingController, - emitSubtitlePayload: (payload) => emitSubtitlePayload(payload), + emitSubtitlePayload: (payload, options) => emitSubtitlePayload(payload, options), getSubtitlePrefetchService: () => subtitlePrefetchService, getLastObservedTimePos: () => lastObservedTimePos, getVisibleOverlayVisible: () => overlayManager.getVisibleOverlayVisible(), @@ -4372,8 +4377,9 @@ const { emitSubtitlePayload(payload); }, onSubtitleChange: (text) => { + // Pause only; restarting the prefetch run here would discard in-flight + // tokenization work on every line. Real seeks restart via onTimePosUpdate. subtitlePrefetchService?.pause(); - subtitlePrefetchService?.onSeek(lastObservedTimePos); subtitleProcessingController.onSubtitleChange(text); }, refreshDiscordPresence: () => { diff --git a/src/main/main-wiring.test.ts b/src/main/main-wiring.test.ts index 60358e1d..1e25ed39 100644 --- a/src/main/main-wiring.test.ts +++ b/src/main/main-wiring.test.ts @@ -223,7 +223,7 @@ test('update overlay notification action triggers install flow', () => { assert.match(runtimeSource, /fallbackClient\.openNoteInBrowser\(noteId\)/); }); -test('subtitle change re-prioritizes prefetch around live playback before tokenizing current line', () => { +test('subtitle change pauses prefetch without restarting its run before tokenizing current line', () => { const source = readMainSource(); const actionBlock = source.match( /onSubtitleChange:\s*\(text\)\s*=>\s*\{(?[\s\S]*?)\n \},\n refreshDiscordPresence:/, @@ -231,14 +231,12 @@ test('subtitle change re-prioritizes prefetch around live playback before tokeni assert.ok(actionBlock); assert.match(actionBlock, /subtitlePrefetchService\?\.pause\(\);/); - assert.match(actionBlock, /subtitlePrefetchService\?\.onSeek\(lastObservedTimePos\);/); + // Restarting the run per line (onSeek) discards in-flight prefetch work; + // only real seeks restart via onTimePosUpdate. + assert.doesNotMatch(actionBlock, /subtitlePrefetchService\?\.onSeek\(/); assert.match(actionBlock, /subtitleProcessingController\.onSubtitleChange\(text\);/); assert.ok( actionBlock.indexOf('subtitlePrefetchService?.pause();') < - actionBlock.indexOf('subtitlePrefetchService?.onSeek(lastObservedTimePos);'), - ); - assert.ok( - actionBlock.indexOf('subtitlePrefetchService?.onSeek(lastObservedTimePos);') < actionBlock.indexOf('subtitleProcessingController.onSubtitleChange(text);'), ); }); @@ -593,7 +591,7 @@ test('YouTube media cache lifecycle routes through configured status notificatio test('subtitle broadcasts share one frequency options snapshot per emitted payload', () => { const source = readMainSource(); const emitBlock = source.match( - /function emitSubtitlePayload\(payload: SubtitleData\): void \{(?[\s\S]*?)\n\}/, + /function emitSubtitlePayload\([\s\S]*?\): void \{(?[\s\S]*?)\n\}/, )?.groups?.body; const frequencyOptionsSnapshot = emitBlock?.match( /const frequencyDictionary = configService\.getConfig\(\)\.subtitleStyle\.frequencyDictionary;(?[\s\S]*?)\n \};/, diff --git a/src/main/runtime/autoplay-subtitle-priming-runtime.test.ts b/src/main/runtime/autoplay-subtitle-priming-runtime.test.ts index 41bbddb6..f3413fe5 100644 --- a/src/main/runtime/autoplay-subtitle-priming-runtime.test.ts +++ b/src/main/runtime/autoplay-subtitle-priming-runtime.test.ts @@ -96,10 +96,10 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su onSubtitleChange: (text) => calls.push(`change:${text}`), refreshCurrentSubtitle: (text) => calls.push(`refresh:${text ?? ''}`), }, - emitSubtitlePayload: (payload) => calls.push(`emit:${payload.text}`), + emitSubtitlePayload: (payload, options) => + calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`), getSubtitlePrefetchService: () => ({ pause: () => calls.push('prefetch:pause'), - onSeek: (timePos) => calls.push(`prefetch:seek:${timePos}`), }), getLastObservedTimePos: () => 12, getVisibleOverlayVisible: () => true, @@ -120,7 +120,7 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su 'request:time-pos', 'set:起動字幕', 'prefetch:pause', - 'emit:起動字幕', + 'emit:起動字幕:resume=false', 'change:起動字幕', ]); }); @@ -154,10 +154,10 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before onSubtitleChange: (text) => calls.push(`change:${text}`), refreshCurrentSubtitle: (text) => calls.push(`refresh:${text ?? ''}`), }, - emitSubtitlePayload: (payload) => calls.push(`emit:${payload.text}`), + emitSubtitlePayload: (payload, options) => + calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`), getSubtitlePrefetchService: () => ({ pause: () => calls.push('prefetch:pause'), - onSeek: (timePos) => calls.push(`prefetch:seek:${timePos}`), }), getLastObservedTimePos: () => 12, getVisibleOverlayVisible: () => true, @@ -175,7 +175,7 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before 'request:sub-text', 'set:起動字幕', 'prefetch:pause', - 'emit:起動字幕', + 'emit:起動字幕:resume=false', 'change:起動字幕', ]); }); diff --git a/src/main/runtime/autoplay-subtitle-priming-runtime.ts b/src/main/runtime/autoplay-subtitle-priming-runtime.ts index aef9d437..46ec61d7 100644 --- a/src/main/runtime/autoplay-subtitle-priming-runtime.ts +++ b/src/main/runtime/autoplay-subtitle-priming-runtime.ts @@ -17,7 +17,6 @@ type AutoplaySubtitlePrimingMpvClient = { type AutoplaySubtitlePrimingPrefetchService = { pause: () => void; - onSeek: (timePos: number) => void; }; export interface AutoplaySubtitlePrimingRuntimeDeps { @@ -33,7 +32,7 @@ export interface AutoplaySubtitlePrimingRuntimeDeps { onSubtitleChange: (text: string) => void; refreshCurrentSubtitle: (text: string) => void; }; - emitSubtitlePayload: (payload: SubtitleData) => void; + emitSubtitlePayload: (payload: SubtitleData, options?: { resumePrefetch?: boolean }) => void; getSubtitlePrefetchService: () => AutoplaySubtitlePrimingPrefetchService | null; getLastObservedTimePos: () => number; getVisibleOverlayVisible: () => boolean; @@ -108,7 +107,9 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi return true; } - emitSubtitlePayload({ text, tokens: null }); + // Provisional raw emit: keep prefetch paused until the tokenized payload + // for this line is delivered by the processing controller. + emitSubtitlePayload({ text, tokens: null }, { resumePrefetch: false }); subtitleProcessingController.onSubtitleChange(text); return true; } @@ -154,12 +155,10 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi consumeCachedSubtitle: (text) => subtitleProcessingController.consumeCachedSubtitle(text), onSubtitleChange: (text) => { deps.getSubtitlePrefetchService()?.pause(); - deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos()); subtitleProcessingController.onSubtitleChange(text); }, refreshCurrentSubtitle: (text) => { deps.getSubtitlePrefetchService()?.pause(); - deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos()); subtitleProcessingController.refreshCurrentSubtitle(text); }, deferUncachedRefresh: true, @@ -205,7 +204,6 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi return; } deps.getSubtitlePrefetchService()?.pause(); - deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos()); subtitleProcessingController.refreshCurrentSubtitle(text); }, VISIBLE_OVERLAY_SUBTITLE_REFRESH_AFTER_FIRST_PAINT_DELAY_MS); visibleOverlaySubtitleRefreshAfterFirstPaintTimer.unref?.(); diff --git a/src/main/runtime/composers/mpv-runtime-composer.ts b/src/main/runtime/composers/mpv-runtime-composer.ts index 22f8ede9..c2744e61 100644 --- a/src/main/runtime/composers/mpv-runtime-composer.ts +++ b/src/main/runtime/composers/mpv-runtime-composer.ts @@ -225,24 +225,35 @@ export function composeMpvRuntimeHandlers< } return tokenizationWarmupInFlight; }; + // Built once and reused for every tokenization: per-call rebuilds create + // fresh closures, which defeats identity-keyed caches downstream (the JLPT + // lookup cache keys on the getJlptLevel function, and the mecab availability + // WeakSet keys on the runtime deps instance). + let cachedTokenizerRuntimeDeps: TTokenizerRuntimeDeps | null = null; + const getTokenizerRuntimeDeps = (): TTokenizerRuntimeDeps => { + if (cachedTokenizerRuntimeDeps) { + return cachedTokenizerRuntimeDeps; + } + const tokenizerMainDeps = buildTokenizerDepsHandler(); + const baseOnTokenizationReady = tokenizerMainDeps.onTokenizationReady; + tokenizerMainDeps.onTokenizationReady = (tokenizedText: string): void => { + if (!shouldWarmupAnnotationDictionaries()) { + baseOnTokenizationReady?.(tokenizedText); + return; + } + markTokenizationPlaybackReady(); + baseOnTokenizationReady?.(tokenizedText); + if (!tokenizationWarmupCompleted) { + void prewarmSubtitleDictionaries({ showLoadingOsd: true }).catch(() => {}); + } + }; + cachedTokenizerRuntimeDeps = options.tokenizer.createTokenizerRuntimeDeps(tokenizerMainDeps); + return cachedTokenizerRuntimeDeps; + }; const tokenizeSubtitle = async (text: string): Promise => { if (!tokenizationWarmupCompleted) void startTokenizationWarmups(); await ensureTokenizationPrerequisites(); - const tokenizerMainDeps = buildTokenizerDepsHandler(); - if (shouldWarmupAnnotationDictionaries()) { - const onTokenizationReady = tokenizerMainDeps.onTokenizationReady; - tokenizerMainDeps.onTokenizationReady = (tokenizedText: string): void => { - markTokenizationPlaybackReady(); - onTokenizationReady?.(tokenizedText); - if (!tokenizationWarmupCompleted) { - void prewarmSubtitleDictionaries({ showLoadingOsd: true }).catch(() => {}); - } - }; - } - return options.tokenizer.tokenizeSubtitle( - text, - options.tokenizer.createTokenizerRuntimeDeps(tokenizerMainDeps), - ); + return options.tokenizer.tokenizeSubtitle(text, getTokenizerRuntimeDeps()); }; const launchBackgroundWarmupTask = createLaunchBackgroundWarmupTaskFromStartup(