From b14f977e3375c3b17d45c85f75e553afc3ecac42 Mon Sep 17 00:00:00 2001 From: sudacode Date: Sun, 5 Jul 2026 17:02:44 -0700 Subject: [PATCH] fix(tokenizer): stop partial furigana readings from marking words known (#137) --- .../known-word-partial-reading-highlight.md | 4 ++ .../tokenizer/annotation-stage.test.ts | 42 ++++++++++++ .../services/tokenizer/annotation-stage.ts | 30 ++++++++ .../tokenizer/yomitan-parser-runtime.test.ts | 68 ++++++++++++++++++- .../tokenizer/yomitan-parser-runtime.ts | 8 ++- 5 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 changes/known-word-partial-reading-highlight.md diff --git a/changes/known-word-partial-reading-highlight.md b/changes/known-word-partial-reading-highlight.md new file mode 100644 index 00000000..6616358d --- /dev/null +++ b/changes/known-word-partial-reading-highlight.md @@ -0,0 +1,4 @@ +type: fixed +area: overlay + +- Fixed never-mined compound words (e.g. 待ち合わせてる) being highlighted green as known: subtitle tokens now carry complete readings instead of kanji-only furigana joins, and the known-word reading fallback rejects readings that don't cover the token surface. Stored word readings in the stats database are no longer truncated for new lines. diff --git a/src/core/services/tokenizer/annotation-stage.test.ts b/src/core/services/tokenizer/annotation-stage.test.ts index 46f73448..d971c2c8 100644 --- a/src/core/services/tokenizer/annotation-stage.test.ts +++ b/src/core/services/tokenizer/annotation-stage.test.ts @@ -123,6 +123,48 @@ test('annotateTokens falls back to reading for known-word matches when headword assert.equal(result[0]?.frequencyRank, 1895); }); +test('annotateTokens ignores partial furigana readings for known-word fallback', () => { + const tokens = [ + makeToken({ + surface: '待ち合わせてる', + headword: '待ち合わせる', + reading: 'まあ', + partOfSpeech: PartOfSpeech.verb, + endPos: 7, + }), + ]; + + const result = annotateTokens( + tokens, + makeDeps({ + isKnownWord: (text) => text === 'まあ', + }), + ); + + assert.equal(result[0]?.isKnown, false); +}); + +test('annotateTokens reading fallback still matches kana surfaces with complete readings', () => { + const tokens = [ + makeToken({ + surface: 'ください', + headword: '下さい', + reading: 'ください', + partOfSpeech: PartOfSpeech.verb, + endPos: 4, + }), + ]; + + const result = annotateTokens( + tokens, + makeDeps({ + isKnownWord: (text) => text === 'ください', + }), + ); + + assert.equal(result[0]?.isKnown, true); +}); + test('annotateTokens excludes frequency for particle/bound_auxiliary and pos1 exclusions', () => { const tokens = [ makeToken({ diff --git a/src/core/services/tokenizer/annotation-stage.ts b/src/core/services/tokenizer/annotation-stage.ts index 3189e98f..ad3e6cb9 100644 --- a/src/core/services/tokenizer/annotation-stage.ts +++ b/src/core/services/tokenizer/annotation-stage.ts @@ -635,6 +635,32 @@ export function stripSubtitleAnnotationMetadata( return sharedStripSubtitleAnnotationMetadata(token, options); } +// Furigana-derived readings can be partial (kanji readings only, e.g. まあ for +// 待ち合わせてる); matching those against known words produces false positives, +// so the reading fallback requires a reading that plausibly covers the surface: +// at least as many characters as the surface, with the surface's kana appearing +// in order within the reading. +function isCompleteReadingForSurface(surface: string, reading: string): boolean { + const surfaceChars = [...normalizeJlptTextForExclusion(surface)]; + const readingChars = [...normalizeJlptTextForExclusion(reading)]; + if (readingChars.length < surfaceChars.length) { + return false; + } + + let cursor = 0; + for (const char of surfaceChars) { + if (!isKanaChar(char)) { + continue; + } + const foundAt = readingChars.indexOf(char, cursor); + if (foundAt === -1) { + return false; + } + cursor = foundAt + 1; + } + return true; +} + function computeTokenKnownStatus( token: MergedToken, isKnownWord: (text: string) => boolean, @@ -650,6 +676,10 @@ function computeTokenKnownStatus( return false; } + if (!isCompleteReadingForSurface(token.surface, normalizedReading)) { + return false; + } + return normalizedReading !== matchText.trim() && isKnownWord(normalizedReading); } diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts index 1db5c8cd..a2b4784a 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts @@ -964,7 +964,7 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF assert.deepEqual(result, [ { surface: '潜み', - reading: 'ひそ', + reading: 'ひそみ', headword: '潜む', startPos: 0, endPos: 2, @@ -974,6 +974,72 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF ]); }); +test('requestYomitanScanTokens emits complete readings for kanji-kana compounds', async () => { + let scannerScript = ''; + const deps = createDeps(async (script) => { + if (script.includes('termsFind')) { + scannerScript = script; + return []; + } + if (script.includes('optionsGetFull')) { + return { + profileCurrent: 0, + profiles: [ + { + options: { + scanning: { length: 40 }, + dictionaries: [{ name: 'JPDBv2㋕', enabled: true, id: 0 }], + }, + }, + ], + }; + } + return null; + }); + + await requestYomitanScanTokens('待ち合わせてる', deps, { + error: () => undefined, + }); + + const result = await runInjectedYomitanScript(scannerScript, (action, params) => { + if (action !== 'termsFind') { + throw new Error(`unexpected action: ${action}`); + } + + const text = (params as { text?: string } | undefined)?.text ?? ''; + if (!text.startsWith('待ち合わせてる')) { + return { originalTextLength: 0, dictionaryEntries: [] }; + } + + return { + originalTextLength: 7, + dictionaryEntries: [ + { + headwords: [ + { + term: '待ち合わせる', + reading: 'まちあわせる', + sources: [{ originalText: '待ち合わせてる', isPrimary: true, matchType: 'exact' }], + }, + ], + }, + ], + }; + }); + + assert.deepEqual(result, [ + { + surface: '待ち合わせてる', + reading: 'まちあわせてる', + headword: '待ち合わせる', + startPos: 0, + endPos: 7, + isNameMatch: false, + frequencyRank: undefined, + }, + ]); +}); + test('requestYomitanScanTokens uses frequency from later exact-match entry when first exact entry has none', async () => { let scannerScript = ''; const deps = createDeps(async (script) => { diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.ts b/src/core/services/tokenizer/yomitan-parser-runtime.ts index 1835b755..8497e610 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.ts @@ -817,6 +817,12 @@ const YOMITAN_SCANNING_HELPERS = String.raw` function isCodePointKana(codePoint) { return isCodePointInRanges(codePoint, KANA_RANGES); } function isCodePointJapanese(codePoint) { return isCodePointInRanges(codePoint, JAPANESE_RANGES); } function createFuriganaSegment(text, reading) { return {text, reading}; } + function getSegmentReadingContribution(segment) { + if (typeof segment.reading === "string" && segment.reading.length > 0) { return segment.reading; } + const segmentText = typeof segment.text === "string" ? segment.text : ""; + const isKanaOnly = segmentText.length > 0 && [...segmentText].every((char) => isCodePointKana(char.codePointAt(0))); + return isKanaOnly ? segmentText : ""; + } function getProlongedHiragana(previousCharacter) { switch (previousCharacter) { case "あ": case "か": case "が": case "さ": case "ざ": case "た": case "だ": case "な": case "は": case "ば": case "ぱ": case "ま": case "や": case "ら": case "わ": case "ぁ": case "ゃ": case "ゎ": return "あ"; @@ -1310,7 +1316,7 @@ ${YOMITAN_SCANNING_HELPERS} const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source); const tokenPayload = { surface: segments.map((segment) => segment.text).join("") || source, - reading: segments.map((segment) => typeof segment.reading === "string" ? segment.reading : "").join(""), + reading: segments.map(getSegmentReadingContribution).join(""), headword: preferredHeadword.term, startPos: i, endPos: i + originalTextLength,