From 6c251502b34ce929deb41fd35e91411fdc4d8cc5 Mon Sep 17 00:00:00 2001 From: sudacode Date: Thu, 9 Jul 2026 00:28:38 -0700 Subject: [PATCH] fix(tokenizer): exclude unparsed-run tokens from annotations and N+1 (#153) --- changes/known-word-reading-aware-matching.md | 6 ++- changes/particle-known-word-highlight.md | 3 +- changes/stats-known-words-v3-cache.md | 4 -- changes/unparsed-token-hover-lookup.md | 2 +- src/core/services/tokenizer.test.ts | 34 +++++++++++++++ src/core/services/tokenizer.ts | 1 + .../tokenizer/annotation-stage.test.ts | 42 +++++++++++++++++++ .../tokenizer/parser-selection-stage.test.ts | 16 ++++--- .../tokenizer/parser-selection-stage.ts | 11 +++-- .../tokenizer/subtitle-annotation-filter.ts | 7 ++++ .../tokenizer/yomitan-parser-runtime.test.ts | 1 + .../tokenizer/yomitan-parser-runtime.ts | 41 ++++++++++-------- src/types/subtitle.ts | 6 +++ 13 files changed, 138 insertions(+), 36 deletions(-) delete mode 100644 changes/stats-known-words-v3-cache.md diff --git a/changes/known-word-reading-aware-matching.md b/changes/known-word-reading-aware-matching.md index 398ec904..9f9a9335 100644 --- a/changes/known-word-reading-aware-matching.md +++ b/changes/known-word-reading-aware-matching.md @@ -1,4 +1,6 @@ -type: fixed +type: changed area: overlay -- Fixed words being highlighted green as known when a same-spelled Anki card taught a different reading (e.g. とこ parsed as 床 "bed" matching a known 床/ゆか "floor" card). The known-word cache now stores each card's word together with its reading and only matches when the token's reading agrees; cards without a reading field keep matching in any reading as before. +- New reading-aware subtitle parsing and known-word matching: the known-word cache now stores each Anki card's word together with its reading (cache format v3), and a token only gets the known-word highlight when its parsed reading agrees with the card. The cache and the stats server upgrade automatically. +- Fixes words being highlighted green as known when a same-spelled Anki card taught a different reading (e.g. とこ parsed as 床 "bed" no longer matches a known 床/ゆか "floor" card). Cards without a reading field keep matching in any reading as before. +- Single-kana grammar tokens (よ in 全然いいよ, standalone え) no longer borrow the reading of an unrelated card (such as 夜 or 絵) and get painted as known: reading-only matching requires at least two kana, while single-kana cards still match by their word field. diff --git a/changes/particle-known-word-highlight.md b/changes/particle-known-word-highlight.md index 6be0ee61..22c3f467 100644 --- a/changes/particle-known-word-highlight.md +++ b/changes/particle-known-word-highlight.md @@ -1,5 +1,4 @@ -type: fixed +type: changed area: overlay -- Fixed single-kana grammar tokens counting as known words by borrowing the reading of an unrelated Anki note (よ in 全然いいよ matched a card read よ such as 夜, standalone え matched 絵), which painted them with the known-word highlight. Reading-only known-word matching now requires at least two kana; single-kana cards still match by their word field, and tokens genuinely present in the known-words cache (e.g. です) keep their highlight. - Standalone suffix tokens (MeCab pos2 接尾, e.g. さん, れる) are now excluded from JLPT/frequency/N+1 annotations by default, matching how particles and interjections are treated. Cache-backed known-word highlighting still applies; override via the pos2 exclusion config if you want them annotated. diff --git a/changes/stats-known-words-v3-cache.md b/changes/stats-known-words-v3-cache.md deleted file mode 100644 index 3b06934b..00000000 --- a/changes/stats-known-words-v3-cache.md +++ /dev/null @@ -1,4 +0,0 @@ -type: fixed -area: stats - -- Session known-word counts no longer show 0 everywhere. The stats server's known-word cache parser only understood the v1/v2 cache formats, so after the reading-aware v3 cache upgrade it silently treated the cache as missing; it now flattens v3 note entries into the headword set. diff --git a/changes/unparsed-token-hover-lookup.md b/changes/unparsed-token-hover-lookup.md index 49217f8c..f28eb237 100644 --- a/changes/unparsed-token-hover-lookup.md +++ b/changes/unparsed-token-hover-lookup.md @@ -1,4 +1,4 @@ type: fixed area: overlay -- Fixed subtitle text that Yomitan's parser cannot match (e.g. the truncated volitional in とこ戻ろ…) being rendered as plain, non-interactive text: it was invisible to hover/lookup and excluded from the n+1 word count, which could wrongly mark a sentence as n+1. Unparsed runs are now kept as hoverable tokens matching Yomitan's own segmentation; bracketed SFX/speaker captions and punctuation-only runs are still skipped. +- Fixed subtitle text that Yomitan's parser cannot match (truncated inflections like the volitional in とこ戻ろ…, elongation runs like ぅ～/ぉ〜) being rendered as plain, non-interactive text with no hover lookup. Unparsed runs are now kept as hoverable tokens matching Yomitan's own segmentation, while being fully ignored by frequency/JLPT highlighting, the N+1 candidate math, and vocabulary stats. Bracketed SFX/speaker captions and punctuation-only runs are still skipped. diff --git a/src/core/services/tokenizer.test.ts b/src/core/services/tokenizer.test.ts index 007a88e3..23deee51 100644 --- a/src/core/services/tokenizer.test.ts +++ b/src/core/services/tokenizer.test.ts @@ -28,6 +28,7 @@ interface YomitanTokenInput { frequencyRank?: number; isNameMatch?: boolean; wordClasses?: string[]; + isUnparsedRun?: boolean; } function makeDepsFromYomitanTokens( @@ -60,6 +61,7 @@ function makeDepsFromYomitanTokens( isNameMatch: token.isNameMatch ?? false, frequencyRank: token.frequencyRank, wordClasses: token.wordClasses, + isUnparsedRun: token.isUnparsedRun, }; }); }, @@ -4223,6 +4225,38 @@ test('tokenizeSubtitle clears all annotations for explanatory pondering endings' ); }); +test('tokenizeSubtitle ignores unparsed-run tokens for annotations and N+1', async () => { + // もう いるぅ～！: the ぅ～ elongation has no Yomitan dictionary entry; it must + // not become the sole N+1 candidate or receive frequency/JLPT annotations. + const result = await tokenizeSubtitle( + 'もう いるぅ～！', + makeDepsFromYomitanTokens( + [ + { surface: 'もう', reading: 'もう', headword: 'もう' }, + { surface: 'いる', reading: 'いる', headword: 'いる' }, + { surface: 'ぅ～', reading: '', headword: 'ぅ～', isUnparsedRun: true, frequencyRank: 999 }, + ], + { + getFrequencyDictionaryEnabled: () => true, + getJlptLevel: (text) => (text === 'ぅ～' ? 'N5' : null), + isKnownWord: (text) => text === 'もう' || text === 'いる', + getMinSentenceWordsForNPlusOne: () => 2, + tokenizeWithMecab: async () => null, + }, + ), + ); + + const filler = result.tokens?.find((token) => token.surface === 'ぅ～'); + assert.ok(filler); + assert.equal(filler?.isNPlusOneTarget, false); + assert.equal(filler?.frequencyRank, undefined); + assert.equal(filler?.jlptLevel, undefined); + assert.equal( + result.tokens?.some((token) => token.isNPlusOneTarget), + false, + ); +}); + test('tokenizeSubtitle keeps frequency for content-led merged token with trailing colloquial suffixes', async () => { const result = await tokenizeSubtitle( '張り切ってんじゃ', diff --git a/src/core/services/tokenizer.ts b/src/core/services/tokenizer.ts index 7d37d647..0d75f773 100644 --- a/src/core/services/tokenizer.ts +++ b/src/core/services/tokenizer.ts @@ -734,6 +734,7 @@ async function parseWithYomitanInternalParser( isNPlusOneTarget: false, isNameMatch: token.isNameMatch ?? false, frequencyRank: token.frequencyRank, + isUnparsedRun: token.isUnparsedRun === true ? true : undefined, }; }), ); diff --git a/src/core/services/tokenizer/annotation-stage.test.ts b/src/core/services/tokenizer/annotation-stage.test.ts index b18fd259..da144134 100644 --- a/src/core/services/tokenizer/annotation-stage.test.ts +++ b/src/core/services/tokenizer/annotation-stage.test.ts @@ -208,6 +208,48 @@ test('annotateTokens hides known-word marks while still using known words for N+ assert.equal(result[2]?.isNPlusOneTarget, true); }); +test('shouldExcludeTokenFromSubtitleAnnotations excludes unparsed-run tokens', () => { + // 戻 from 「…とこ戻ろ…」: Yomitan had no dictionary entry, headword falls back + // to the surface. Without the flag the token passes every other filter. + const unflagged = makeToken({ surface: '戻', headword: '戻', reading: '' }); + assert.equal(shouldExcludeTokenFromSubtitleAnnotations(unflagged), false); + + const flagged = makeToken({ surface: '戻', headword: '戻', reading: '', isUnparsedRun: true }); + assert.equal(shouldExcludeTokenFromSubtitleAnnotations(flagged), true); + assert.equal(shouldExcludeTokenFromVocabularyPersistence(flagged), true); +}); + +test('annotateTokens ignores unparsed-run tokens for annotations and N+1', () => { + // もう いるぅ～！-style line: the elongation run is the only unknown token and + // used to become the sole N+1 candidate despite having no dictionary entry. + const tokens = [ + makeToken({ surface: 'みんな', headword: '皆', reading: 'みんな', startPos: 0, endPos: 3 }), + makeToken({ surface: 'とこ', headword: '所', reading: 'とこ', startPos: 3, endPos: 5 }), + makeToken({ + surface: '戻', + headword: '戻', + reading: '', + startPos: 5, + endPos: 6, + isUnparsedRun: true, + frequencyRank: 12, + }), + ]; + + const result = annotateTokens( + tokens, + makeDeps({ + isKnownWord: (text) => text === '皆' || text === '所', + getJlptLevel: (text) => (text === '戻' ? 'N5' : null), + }), + { minSentenceWordsForNPlusOne: 2 }, + ); + + assert.equal(result[2]?.isNPlusOneTarget, false); + assert.equal(result[2]?.jlptLevel, undefined); + assert.equal(result[2]?.frequencyRank, undefined); +}); + test('annotateTokens falls back to reading for known-word matches when headword lookup misses', () => { const tokens = [ makeToken({ diff --git a/src/core/services/tokenizer/parser-selection-stage.test.ts b/src/core/services/tokenizer/parser-selection-stage.test.ts index f190fbe6..8d532015 100644 --- a/src/core/services/tokenizer/parser-selection-stage.test.ts +++ b/src/core/services/tokenizer/parser-selection-stage.test.ts @@ -146,13 +146,17 @@ test('emits unparsed non-caption text as a token with surface headword', () => { const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword'); assert.deepEqual( - tokens?.map((token) => ({ surface: token.surface, headword: token.headword })), + tokens?.map((token) => ({ + surface: token.surface, + headword: token.headword, + isUnparsedRun: token.isUnparsedRun ?? false, + })), [ - { surface: 'みんな', headword: '皆' }, - { surface: 'の', headword: 'の' }, - { surface: 'とこ', headword: '所' }, - { surface: '戻', headword: '戻' }, - { surface: 'ろ', headword: '櫓' }, + { surface: 'みんな', headword: '皆', isUnparsedRun: false }, + { surface: 'の', headword: 'の', isUnparsedRun: false }, + { surface: 'とこ', headword: '所', isUnparsedRun: false }, + { surface: '戻', headword: '戻', isUnparsedRun: true }, + { surface: 'ろ', headword: '櫓', isUnparsedRun: false }, ], ); }); diff --git a/src/core/services/tokenizer/parser-selection-stage.ts b/src/core/services/tokenizer/parser-selection-stage.ts index 0b5fee79..5d508de7 100644 --- a/src/core/services/tokenizer/parser-selection-stage.ts +++ b/src/core/services/tokenizer/parser-selection-stage.ts @@ -239,6 +239,7 @@ export function mapYomitanParseResultItemToMergedTokens( headword: string, start: number, end: number, + isUnparsedRun = false, ): void => { tokens.push({ surface, @@ -254,6 +255,7 @@ export function mapYomitanParseResultItemToMergedTokens( const matchText = resolveKnownWordText(surface, headword, knownWordMatchMode); return matchText ? isKnownWord(matchText) : false; })(), + ...(isUnparsedRun ? { isUnparsedRun: true } : {}), }); }; @@ -275,10 +277,11 @@ export function mapYomitanParseResultItemToMergedTokens( previousToken.reading += combinedReading; previousToken.endPos = end; } else if (shouldEmitUnparsedRunAsToken(combinedSurface)) { - // Yomitan couldn't parse this run (e.g. 戻ろ… truncated volitional). Keep it - // as a token with its surface as headword so it stays hoverable and counts in - // the n+1 math — matching what the embedded Yomitan actually returns. - pushToken(combinedSurface, combinedReading, combinedSurface, combinedStart, end); + // Yomitan couldn't parse this run (e.g. 戻ろ… truncated volitional, ぅ～ + // elongations). Keep it as a token with its surface as headword so it stays + // hoverable, but flag it so annotation/N+1/vocab logic ignores it — there is + // no dictionary entry behind it. + pushToken(combinedSurface, combinedReading, combinedSurface, combinedStart, end, true); } } else { hasDictionaryMatch = true; diff --git a/src/core/services/tokenizer/subtitle-annotation-filter.ts b/src/core/services/tokenizer/subtitle-annotation-filter.ts index 67adb0b6..7141ca9f 100644 --- a/src/core/services/tokenizer/subtitle-annotation-filter.ts +++ b/src/core/services/tokenizer/subtitle-annotation-filter.ts @@ -474,6 +474,13 @@ export function shouldExcludeTokenFromSubtitleAnnotations( token: MergedToken, options: SubtitleAnnotationFilterOptions = {}, ): boolean { + // No Yomitan dictionary entry backs this token (ぅ～ elongations, truncated + // inflections) — it exists only to stay hoverable and must never receive + // annotations or count in the N+1 math. + if (token.isUnparsedRun === true) { + return true; + } + const pos1Exclusions = resolvePos1Exclusions(options); const pos2Exclusions = resolvePos2Exclusions(options); const normalizedPos1 = normalizePosTag(token.pos1); diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts index a51645ef..e5a6c6a1 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts @@ -934,6 +934,7 @@ test('requestYomitanScanTokens keeps scanner metadata for matching spans when pa headword: 'っ ', startPos: 2, endPos: 4, + isUnparsedRun: true, }, { surface: 'ミナト', diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.ts b/src/core/services/tokenizer/yomitan-parser-runtime.ts index 39ec52b2..5997c65f 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.ts @@ -57,6 +57,7 @@ export interface YomitanScanToken { isNameMatch?: boolean; frequencyRank?: number; wordClasses?: string[]; + isUnparsedRun?: boolean; } interface YomitanProfileMetadata { @@ -110,6 +111,27 @@ function scanTokenSpanKey(token: YomitanScanToken): string { return `${token.startPos}:${token.endPos}:${token.surface}`; } +// Maps a parse-selected token to the scanner-token shape carried out of the +// parser runtime. Shared by both selectYomitanParseTokens fallback paths so the +// projected fields stay in sync as the shape changes. +function toYomitanScanToken(token: { + surface: string; + reading: string; + headword: string; + startPos: number; + endPos: number; + isUnparsedRun?: boolean; +}): YomitanScanToken { + return { + surface: token.surface, + reading: token.reading, + headword: token.headword, + startPos: token.startPos, + endPos: token.endPos, + ...(token.isUnparsedRun === true ? { isUnparsedRun: true } : {}), + }; +} + // parseText segmentation is authoritative (it emits filler chunks for text the // termsFind scanner skips), but only the termsFind scanner carries annotation // metadata (isNameMatch, frequencyRank, headwordReading, wordClasses). Graft @@ -1611,14 +1633,7 @@ export async function requestYomitanScanTokens( const parseResults = await requestYomitanParseResults(text, deps, logger); const selectedParseTokens = selectYomitanParseTokens(parseResults, () => false, 'headword'); - const parseScanTokens = - selectedParseTokens?.map((token) => ({ - surface: token.surface, - reading: token.reading, - headword: token.headword, - startPos: token.startPos, - endPos: token.endPos, - })) ?? null; + const parseScanTokens = selectedParseTokens?.map(toYomitanScanToken) ?? null; const metadata = await requestYomitanProfileMetadata(parserWindow, logger); const profileIndex = metadata?.profileIndex ?? 0; @@ -1656,15 +1671,7 @@ export async function requestYomitanScanTokens( } if (Array.isArray(rawResult)) { const selectedTokens = selectYomitanParseTokens(rawResult, () => false, 'headword'); - return ( - selectedTokens?.map((token) => ({ - surface: token.surface, - reading: token.reading, - headword: token.headword, - startPos: token.startPos, - endPos: token.endPos, - })) ?? null - ); + return selectedTokens?.map(toYomitanScanToken) ?? null; } if (parseScanTokens && parseScanTokens.length > 0) { return parseScanTokens; diff --git a/src/types/subtitle.ts b/src/types/subtitle.ts index 2e775f34..2535c6ea 100644 --- a/src/types/subtitle.ts +++ b/src/types/subtitle.ts @@ -40,6 +40,12 @@ export interface MergedToken { isMerged: boolean; isKnown: boolean; isNPlusOneTarget: boolean; + /** + * Text Yomitan had no dictionary entry for (e.g. ぅ～ elongation runs, + * truncated inflections). Kept as a token so it stays hoverable, but + * ignored by annotation, N+1, and vocabulary-stats logic. + */ + isUnparsedRun?: boolean; isNameMatch?: boolean; characterImage?: CharacterNameImage; jlptLevel?: JlptLevel;