diff --git a/changes/lexicalized-expression-frequency.md b/changes/lexicalized-expression-frequency.md new file mode 100644 index 00000000..b1384b46 --- /dev/null +++ b/changes/lexicalized-expression-frequency.md @@ -0,0 +1,4 @@ +type: fixed +area: overlay + +- Fixed lexicalized kana expressions such as `かといって` losing frequency annotations when their merged MeCab parts included particles. diff --git a/docs-site/subtitle-annotations.md b/docs-site/subtitle-annotations.md index 719a0598..73f51b53 100644 --- a/docs-site/subtitle-annotations.md +++ b/docs-site/subtitle-annotations.md @@ -90,7 +90,7 @@ SubMiner looks up each token's `frequencyRank` from `term_meta_bank_*.json` file When `sourcePath` is omitted, SubMiner searches default install/runtime locations for `frequency-dictionary` directories automatically. ::: info -Frequency highlighting skips tokens that look like non-lexical noise (kana reduplication, short kana endings like `っ`), even when dictionary ranks exist. +Frequency highlighting skips tokens that look like non-lexical noise (kana reduplication, short kana endings like `っ`), even when dictionary ranks exist. For merged kana tokens, SubMiner keeps a rank when the dictionary headword reading covers the full token (for example, `かと言って` / `かといって`), while grammar wrapped around a shorter lemma remains unannotated. ::: ::: info diff --git a/src/core/services/tokenizer/annotation-stage.test.ts b/src/core/services/tokenizer/annotation-stage.test.ts index 0e7492c2..d0d19b95 100644 --- a/src/core/services/tokenizer/annotation-stage.test.ts +++ b/src/core/services/tokenizer/annotation-stage.test.ts @@ -1713,7 +1713,9 @@ test('annotateTokens excludes kana-only composite function/content tokens from f const tokens = [ makeToken({ surface: 'になれば', + reading: 'になれば', headword: 'なる', + headwordReading: 'なる', pos1: '助詞|動詞', pos2: '格助詞|自立|接続助詞', startPos: 0, @@ -1730,6 +1732,28 @@ test('annotateTokens excludes kana-only composite function/content tokens from f assert.equal(result[0]?.isNPlusOneTarget, false); }); +test('annotateTokens keeps frequency for mixed kana tokens whose headword reading covers the token', () => { + const tokens = [ + makeToken({ + surface: 'かといって', + reading: 'カトイッテ', + headword: 'かと言って', + headwordReading: 'かといって', + pos1: '助詞|助詞|動詞|助詞', + pos2: '副助詞|格助詞|自立|接続助詞', + startPos: 0, + endPos: 5, + frequencyRank: 4898, + }), + ]; + + const result = annotateTokens(tokens, makeDeps(), { + minSentenceWordsForNPlusOne: 1, + }); + + assert.equal(result[0]?.frequencyRank, 4898); +}); + test('annotateTokens excludes composite tokens when all component pos tags are excluded', () => { const tokens = [ makeToken({ diff --git a/src/core/services/tokenizer/annotation-stage.ts b/src/core/services/tokenizer/annotation-stage.ts index 7631fa23..9fd347a8 100644 --- a/src/core/services/tokenizer/annotation-stage.ts +++ b/src/core/services/tokenizer/annotation-stage.ts @@ -550,10 +550,20 @@ function isKanaOnlyMixedFunctionContentToken( } const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1)); - return ( + const hasMixedFunctionContentParts = pos1Parts.length >= 2 && pos1Parts.some((part) => pos1Exclusions.has(part)) && - pos1Parts.some((part) => !pos1Exclusions.has(part)) + pos1Parts.some((part) => !pos1Exclusions.has(part)); + if (!hasMixedFunctionContentParts) { + return false; + } + + const normalizedReading = normalizeJlptTextForExclusion(token.reading); + const normalizedHeadwordReading = normalizeJlptTextForExclusion(token.headwordReading ?? ''); + return ( + !normalizedReading || + !normalizedHeadwordReading || + normalizedReading !== normalizedHeadwordReading ); }