fix(tokenizer): stop partial furigana readings from marking words known (#137)

changes/known-word-partial-reading-highlight.md

@@ -0,0 +1,4 @@
+type: fixed
+area: overlay
+
+- Fixed never-mined compound words (e.g. 待ち合わせてる) being highlighted green as known: subtitle tokens now carry complete readings instead of kanji-only furigana joins, and the known-word reading fallback rejects readings that don't cover the token surface. Stored word readings in the stats database are no longer truncated for new lines.

src/core/services/tokenizer/annotation-stage.test.ts

@@ -123,6 +123,48 @@ test('annotateTokens falls back to reading for known-word matches when headword
   assert.equal(result[0]?.frequencyRank, 1895);
 });
 
+test('annotateTokens ignores partial furigana readings for known-word fallback', () => {
+  const tokens = [
+    makeToken({
+      surface: '待ち合わせてる',
+      headword: '待ち合わせる',
+      reading: 'まあ',
+      partOfSpeech: PartOfSpeech.verb,
+      endPos: 7,
+    }),
+  ];
+
+  const result = annotateTokens(
+    tokens,
+    makeDeps({
+      isKnownWord: (text) => text === 'まあ',
+    }),
+  );
+
+  assert.equal(result[0]?.isKnown, false);
+});
+
+test('annotateTokens reading fallback still matches kana surfaces with complete readings', () => {
+  const tokens = [
+    makeToken({
+      surface: 'ください',
+      headword: '下さい',
+      reading: 'ください',
+      partOfSpeech: PartOfSpeech.verb,
+      endPos: 4,
+    }),
+  ];
+
+  const result = annotateTokens(
+    tokens,
+    makeDeps({
+      isKnownWord: (text) => text === 'ください',
+    }),
+  );
+
+  assert.equal(result[0]?.isKnown, true);
+});
+
 test('annotateTokens excludes frequency for particle/bound_auxiliary and pos1 exclusions', () => {
   const tokens = [
     makeToken({

src/core/services/tokenizer/annotation-stage.ts

@@ -635,6 +635,32 @@ export function stripSubtitleAnnotationMetadata(
   return sharedStripSubtitleAnnotationMetadata(token, options);
 }
 
+// Furigana-derived readings can be partial (kanji readings only, e.g. まあ for
+// 待ち合わせてる); matching those against known words produces false positives,
+// so the reading fallback requires a reading that plausibly covers the surface:
+// at least as many characters as the surface, with the surface's kana appearing
+// in order within the reading.
+function isCompleteReadingForSurface(surface: string, reading: string): boolean {
+  const surfaceChars = [...normalizeJlptTextForExclusion(surface)];
+  const readingChars = [...normalizeJlptTextForExclusion(reading)];
+  if (readingChars.length < surfaceChars.length) {
+    return false;
+  }
+
+  let cursor = 0;
+  for (const char of surfaceChars) {
+    if (!isKanaChar(char)) {
+      continue;
+    }
+    const foundAt = readingChars.indexOf(char, cursor);
+    if (foundAt === -1) {
+      return false;
+    }
+    cursor = foundAt + 1;
+  }
+  return true;
+}
+
 function computeTokenKnownStatus(
   token: MergedToken,
   isKnownWord: (text: string) => boolean,
@@ -650,6 +676,10 @@ function computeTokenKnownStatus(
     return false;
   }
 
+  if (!isCompleteReadingForSurface(token.surface, normalizedReading)) {
+    return false;
+  }
+
   return normalizedReading !== matchText.trim() && isKnownWord(normalizedReading);
 }
 

src/core/services/tokenizer/yomitan-parser-runtime.test.ts

@@ -964,7 +964,7 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF
   assert.deepEqual(result, [
     {
       surface: '潜み',
-      reading: 'ひそ',
+      reading: 'ひそみ',
       headword: '潜む',
       startPos: 0,
       endPos: 2,
@@ -974,6 +974,72 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF
   ]);
 });
 
+test('requestYomitanScanTokens emits complete readings for kanji-kana compounds', async () => {
+  let scannerScript = '';
+  const deps = createDeps(async (script) => {
+    if (script.includes('termsFind')) {
+      scannerScript = script;
+      return [];
+    }
+    if (script.includes('optionsGetFull')) {
+      return {
+        profileCurrent: 0,
+        profiles: [
+          {
+            options: {
+              scanning: { length: 40 },
+              dictionaries: [{ name: 'JPDBv2㋕', enabled: true, id: 0 }],
+            },
+          },
+        ],
+      };
+    }
+    return null;
+  });
+
+  await requestYomitanScanTokens('待ち合わせてる', deps, {
+    error: () => undefined,
+  });
+
+  const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
+    if (action !== 'termsFind') {
+      throw new Error(`unexpected action: ${action}`);
+    }
+
+    const text = (params as { text?: string } | undefined)?.text ?? '';
+    if (!text.startsWith('待ち合わせてる')) {
+      return { originalTextLength: 0, dictionaryEntries: [] };
+    }
+
+    return {
+      originalTextLength: 7,
+      dictionaryEntries: [
+        {
+          headwords: [
+            {
+              term: '待ち合わせる',
+              reading: 'まちあわせる',
+              sources: [{ originalText: '待ち合わせてる', isPrimary: true, matchType: 'exact' }],
+            },
+          ],
+        },
+      ],
+    };
+  });
+
+  assert.deepEqual(result, [
+    {
+      surface: '待ち合わせてる',
+      reading: 'まちあわせてる',
+      headword: '待ち合わせる',
+      startPos: 0,
+      endPos: 7,
+      isNameMatch: false,
+      frequencyRank: undefined,
+    },
+  ]);
+});
+
 test('requestYomitanScanTokens uses frequency from later exact-match entry when first exact entry has none', async () => {
   let scannerScript = '';
   const deps = createDeps(async (script) => {

src/core/services/tokenizer/yomitan-parser-runtime.ts

@@ -817,6 +817,12 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
       function isCodePointKana(codePoint) { return isCodePointInRanges(codePoint, KANA_RANGES); }
       function isCodePointJapanese(codePoint) { return isCodePointInRanges(codePoint, JAPANESE_RANGES); }
       function createFuriganaSegment(text, reading) { return {text, reading}; }
+      function getSegmentReadingContribution(segment) {
+        if (typeof segment.reading === "string" && segment.reading.length > 0) { return segment.reading; }
+        const segmentText = typeof segment.text === "string" ? segment.text : "";
+        const isKanaOnly = segmentText.length > 0 && [...segmentText].every((char) => isCodePointKana(char.codePointAt(0)));
+        return isKanaOnly ? segmentText : "";
+      }
       function getProlongedHiragana(previousCharacter) {
         switch (previousCharacter) {
           case "あ": case "か": case "が": case "さ": case "ざ": case "た": case "だ": case "な": case "は": case "ば": case "ぱ": case "ま": case "や": case "ら": case "わ": case "ぁ": case "ゃ": case "ゎ": return "あ";
@@ -1310,7 +1316,7 @@ ${YOMITAN_SCANNING_HELPERS}
             const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
             const tokenPayload = {
               surface: segments.map((segment) => segment.text).join("") || source,
-              reading: segments.map((segment) => typeof segment.reading === "string" ? segment.reading : "").join(""),
+              reading: segments.map(getSegmentReadingContribution).join(""),
               headword: preferredHeadword.term,
               startPos: i,
               endPos: i + originalTextLength,