fix(tokenizer): keep frequency rank for lexicalized kana expressions (#156)

changes/lexicalized-expression-frequency.md

@@ -0,0 +1,4 @@
+type: fixed
+area: overlay
+
+- Fixed lexicalized kana expressions such as `かといって` losing frequency annotations when their merged MeCab parts included particles.

docs-site/subtitle-annotations.md

@@ -90,7 +90,7 @@ SubMiner looks up each token's `frequencyRank` from `term_meta_bank_*.json` file
 When `sourcePath` is omitted, SubMiner searches default install/runtime locations for `frequency-dictionary` directories automatically.
 
 ::: info
-Frequency highlighting skips tokens that look like non-lexical noise (kana reduplication, short kana endings like `っ`), even when dictionary ranks exist.
+Frequency highlighting skips tokens that look like non-lexical noise (kana reduplication, short kana endings like `っ`), even when dictionary ranks exist. For merged kana tokens, SubMiner keeps a rank when the dictionary headword reading covers the full token (for example, `かと言って` / `かといって`), while grammar wrapped around a shorter lemma remains unannotated.
 :::
 
 ::: info

src/core/services/tokenizer/annotation-stage.test.ts

@@ -1713,7 +1713,9 @@ test('annotateTokens excludes kana-only composite function/content tokens from f
   const tokens = [
     makeToken({
       surface: 'になれば',
+      reading: 'になれば',
       headword: 'なる',
+      headwordReading: 'なる',
       pos1: '助詞|動詞',
       pos2: '格助詞|自立|接続助詞',
       startPos: 0,
@@ -1730,6 +1732,28 @@ test('annotateTokens excludes kana-only composite function/content tokens from f
   assert.equal(result[0]?.isNPlusOneTarget, false);
 });
 
+test('annotateTokens keeps frequency for mixed kana tokens whose headword reading covers the token', () => {
+  const tokens = [
+    makeToken({
+      surface: 'かといって',
+      reading: 'カトイッテ',
+      headword: 'かと言って',
+      headwordReading: 'かといって',
+      pos1: '助詞|助詞|動詞|助詞',
+      pos2: '副助詞|格助詞|自立|接続助詞',
+      startPos: 0,
+      endPos: 5,
+      frequencyRank: 4898,
+    }),
+  ];
+
+  const result = annotateTokens(tokens, makeDeps(), {
+    minSentenceWordsForNPlusOne: 1,
+  });
+
+  assert.equal(result[0]?.frequencyRank, 4898);
+});
+
 test('annotateTokens excludes composite tokens when all component pos tags are excluded', () => {
   const tokens = [
     makeToken({

src/core/services/tokenizer/annotation-stage.ts

@@ -550,10 +550,20 @@ function isKanaOnlyMixedFunctionContentToken(
   }
 
   const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
-  return (
+  const hasMixedFunctionContentParts =
     pos1Parts.length >= 2 &&
     pos1Parts.some((part) => pos1Exclusions.has(part)) &&
-    pos1Parts.some((part) => !pos1Exclusions.has(part))
+    pos1Parts.some((part) => !pos1Exclusions.has(part));
+  if (!hasMixedFunctionContentParts) {
+    return false;
+  }
+
+  const normalizedReading = normalizeJlptTextForExclusion(token.reading);
+  const normalizedHeadwordReading = normalizeJlptTextForExclusion(token.headwordReading ?? '');
+  return (
+    !normalizedReading ||
+    !normalizedHeadwordReading ||
+    normalizedReading !== normalizedHeadwordReading
   );
 }