From a2e49b369b2a373109974a1fc5f865bef1737eff Mon Sep 17 00:00:00 2001 From: sudacode Date: Wed, 8 Jul 2026 22:26:11 -0700 Subject: [PATCH] fix(tokenizer): merge scanner metadata per token instead of all-or-nothi - Replace hasSameTokenSpans + full-discard with mergeScannerTokensIntoParseTokens - Grafts isNameMatch/frequencyRank/etc onto matching parse spans; filler chunks degrade only themselves - Fixes name annotations dropping for entire subtitle lines containing unmatched interjections --- changes/name-match-span-merge.md | 4 + .../tokenizer/yomitan-parser-runtime.test.ts | 128 ++++++++++++++++++ .../tokenizer/yomitan-parser-runtime.ts | 32 +++-- 3 files changed, 151 insertions(+), 13 deletions(-) create mode 100644 changes/name-match-span-merge.md diff --git a/changes/name-match-span-merge.md b/changes/name-match-span-merge.md new file mode 100644 index 00000000..6edf6c14 --- /dev/null +++ b/changes/name-match-span-merge.md @@ -0,0 +1,4 @@ +type: fixed +area: overlay + +- Fixed character-name annotations dropping for an entire subtitle line when it contained any chunk the dictionary scanner could not match (e.g. an interjection like やほっ before a name): scanner metadata is now merged per token into the parseText segmentation instead of being discarded on any span mismatch. diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts index ed5f6208..3c2e2dbd 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts @@ -820,6 +820,134 @@ test('requestYomitanScanTokens keeps scanner metadata when parse spans agree', a ]); }); +test('requestYomitanScanTokens keeps scanner metadata for matching spans when parse segmentation has filler chunks', async () => { + const deps = createDeps(async (script) => { + if (script.includes('optionsGetFull')) { + return { + profileCurrent: 0, + profiles: [ + { + options: { + scanning: { length: 40 }, + }, + }, + ], + }; + } + if (script.includes('parseText')) { + return [ + { + source: 'scanning-parser', + index: 0, + content: [ + [ + { + text: 'や', + reading: '', + headwords: [[{ term: 'や' }]], + }, + ], + [ + { + text: 'ほ', + reading: '', + headwords: [[{ term: '帆' }]], + }, + ], + [ + { + text: 'っ ', + reading: '', + }, + ], + [ + { + text: 'ミナト', + reading: '', + headwords: [[{ term: 'ミナト' }]], + }, + ], + ], + }, + ]; + } + // The termsFind scanner skips the unmatched っ+space chunk, so its spans + // do not line up 1:1 with the parseText segmentation above. + return [ + { + surface: 'や', + reading: 'や', + headword: 'や', + headwordReading: 'や', + startPos: 0, + endPos: 1, + frequencyRank: 57, + }, + { + surface: 'ほ', + reading: 'ほ', + headword: '帆', + headwordReading: 'ほ', + startPos: 1, + endPos: 2, + frequencyRank: 15414, + }, + { + surface: 'ミナト', + reading: 'ミナト', + headword: 'ミナト', + headwordReading: 'みなと', + startPos: 4, + endPos: 7, + isNameMatch: true, + frequencyRank: 75133, + }, + ]; + }); + + const result = await requestYomitanScanTokens('やほっ ミナト', deps, { + error: () => undefined, + }); + + assert.deepEqual(result, [ + { + surface: 'や', + reading: 'や', + headword: 'や', + headwordReading: 'や', + startPos: 0, + endPos: 1, + frequencyRank: 57, + }, + { + surface: 'ほ', + reading: 'ほ', + headword: '帆', + headwordReading: 'ほ', + startPos: 1, + endPos: 2, + frequencyRank: 15414, + }, + { + surface: 'っ ', + reading: '', + headword: 'っ ', + startPos: 2, + endPos: 4, + }, + { + surface: 'ミナト', + reading: 'ミナト', + headword: 'ミナト', + headwordReading: 'みなと', + startPos: 4, + endPos: 7, + isNameMatch: true, + frequencyRank: 75133, + }, + ]); +}); + test('requestYomitanScanTokens falls back to left-to-right termsFind scanning', async () => { const scripts: string[] = []; const deps = createDeps(async (script) => { diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.ts b/src/core/services/tokenizer/yomitan-parser-runtime.ts index 37664337..18c9a535 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.ts @@ -105,20 +105,26 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] { ); } -function hasSameTokenSpans(left: YomitanScanToken[], right: YomitanScanToken[]): boolean { - if (left.length !== right.length) { - return false; +function scanTokenSpanKey(token: YomitanScanToken): string { + return `${token.startPos}:${token.endPos}:${token.surface}`; +} + +// parseText segmentation is authoritative (it emits filler chunks for text the +// termsFind scanner skips), but only the termsFind scanner carries annotation +// metadata (isNameMatch, frequencyRank, headwordReading, wordClasses). Graft +// scanner tokens onto the parseText segmentation per matching span so one +// unmatched chunk degrades only itself instead of dropping the whole line's +// metadata. +function mergeScannerTokensIntoParseTokens( + parseScanTokens: YomitanScanToken[], + scannerTokens: YomitanScanToken[], +): YomitanScanToken[] { + const scannerTokensBySpan = new Map(); + for (const token of scannerTokens) { + scannerTokensBySpan.set(scanTokenSpanKey(token), token); } - return left.every((token, index) => { - const other = right[index]; - return ( - other !== undefined && - token.surface === other.surface && - token.startPos === other.startPos && - token.endPos === other.endPos - ); - }); + return parseScanTokens.map((token) => scannerTokensBySpan.get(scanTokenSpanKey(token)) ?? token); } function makeTermReadingCacheKey(term: string, reading: string | null): string { @@ -1514,7 +1520,7 @@ export async function requestYomitanScanTokens( ); if (isScanTokenArray(rawResult)) { if (parseScanTokens && parseScanTokens.length > 0) { - return hasSameTokenSpans(parseScanTokens, rawResult) ? rawResult : parseScanTokens; + return mergeScannerTokensIntoParseTokens(parseScanTokens, rawResult); } return rawResult; }