feat(anki): reading-aware known-word matching (cache v3) (#142)

src/core/services/tokenizer.ts

@@ -42,7 +42,7 @@ export interface TokenizerServiceDeps {
   setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
   getYomitanParserInitPromise: () => Promise<boolean> | null;
   setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
-  isKnownWord: (text: string) => boolean;
+  isKnownWord: (text: string, reading?: string) => boolean;
   getKnownWordMatchMode: () => NPlusOneMatchMode;
   getKnownWordsEnabled?: () => boolean;
   getJlptLevel: (text: string) => JlptLevel | null;
@@ -77,7 +77,7 @@ export interface TokenizerDepsRuntimeOptions {
   setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
   getYomitanParserInitPromise: () => Promise<boolean> | null;
   setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
-  isKnownWord: (text: string) => boolean;
+  isKnownWord: (text: string, reading?: string) => boolean;
   getKnownWordMatchMode: () => NPlusOneMatchMode;
   getKnownWordsEnabled?: () => boolean;
   getJlptLevel: (text: string) => JlptLevel | null;
@@ -129,7 +129,7 @@ const INVISIBLE_SEPARATOR_PATTERN = /[\u200b\u2060\ufeff]/g;
 function getKnownWordLookup(
   deps: TokenizerServiceDeps,
   options: TokenizerAnnotationOptions,
-): (text: string) => boolean {
+): (text: string, reading?: string) => boolean {
   if (!options.knownWordsEnabled && !options.nPlusOneEnabled) {
     return () => false;
   }
@@ -723,6 +723,7 @@ async function parseWithYomitanInternalParser(
         surface: token.surface,
         reading: token.reading,
         headword: token.headword,
+        headwordReading: token.headwordReading,
         startPos: token.startPos,
         endPos: token.endPos,
         partOfSpeech: posMetadata.partOfSpeech,

src/core/services/tokenizer/annotation-stage.test.ts

@@ -56,6 +56,70 @@ test('annotateTokens known-word match mode uses headword vs surface', () => {
   assert.equal(surfaceResult[0]?.isKnown, false);
 });
 
+test('annotateTokens passes dictionary-form reading so spelling collisions stay unknown', () => {
+  // とこ (colloquial ところ) resolves to headword 床/とこ; a known 床/ゆか card
+  // must not mark it known (#138 regression).
+  const cache = new Map([['床', 'ゆか']]);
+  const isKnownWord = (text: string, reading?: string): boolean => {
+    if (!cache.has(text)) {
+      return false;
+    }
+    return reading === undefined || cache.get(text) === reading;
+  };
+  const tokens = [
+    makeToken({
+      surface: 'とこ',
+      headword: '床',
+      reading: 'とこ',
+      headwordReading: 'とこ',
+      endPos: 2,
+    }),
+  ];
+
+  const result = annotateTokens(tokens, makeDeps({ isKnownWord }));
+
+  assert.equal(result[0]?.isKnown, false);
+});
+
+test('annotateTokens keeps inflected known words matched via headword reading', () => {
+  const isKnownWord = (text: string, reading?: string): boolean =>
+    text === '行く' && (reading === undefined || reading === 'いく');
+  const tokens = [
+    makeToken({
+      surface: '行きたい',
+      headword: '行く',
+      reading: 'いきたい',
+      headwordReading: 'いく',
+      partOfSpeech: PartOfSpeech.verb,
+      endPos: 4,
+    }),
+  ];
+
+  const result = annotateTokens(tokens, makeDeps({ isKnownWord }));
+
+  assert.equal(result[0]?.isKnown, true);
+});
+
+test('annotateTokens omits reading for headword match when token lacks headword reading and is inflected', () => {
+  // MeCab tokens have no dictionary-form reading; the surface reading of an
+  // inflected form must not be compared against the note's dictionary reading.
+  const isKnownWord = (text: string, reading?: string): boolean =>
+    text === '食べる' && reading === undefined;
+  const tokens = [
+    makeToken({
+      surface: '食べた',
+      headword: '食べる',
+      reading: 'タベタ',
+      partOfSpeech: PartOfSpeech.verb,
+      endPos: 3,
+    }),
+  ];
+
+  const result = annotateTokens(tokens, makeDeps({ isKnownWord }));
+
+  assert.equal(result[0]?.isKnown, true);
+});
+
 test('annotateTokens marks known words when N+1 is disabled', () => {
   const tokens = [
     makeToken({ surface: '私', headword: '私', startPos: 0, endPos: 1 }),

src/core/services/tokenizer/annotation-stage.ts

@@ -25,7 +25,7 @@ const jlptLevelLookupCaches = new WeakMap<
 >();
 
 export interface AnnotationStageDeps {
-  isKnownWord: (text: string) => boolean;
+  isKnownWord: (text: string, reading?: string) => boolean;
   knownWordMatchMode: NPlusOneMatchMode;
   getJlptLevel: (text: string) => JlptLevel | null;
 }
@@ -661,26 +661,57 @@ function isCompleteReadingForSurface(surface: string, reading: string): boolean
   return true;
 }
 
+// Returns the token's trimmed reading only when it plausibly covers the surface
+// (see isCompleteReadingForSurface); undefined otherwise. Shared so the
+// known-word reading disambiguation and the reading fallback stay in sync if the
+// validity rule changes.
+function resolveCompleteTokenReading(token: MergedToken): string | undefined {
+  const normalizedReading = token.reading.trim();
+  if (!normalizedReading || !isCompleteReadingForSurface(token.surface, normalizedReading)) {
+    return undefined;
+  }
+  return normalizedReading;
+}
+
+// Reading to disambiguate the known-word text match, or undefined when the
+// token has no reading that describes the match text: in headword mode an
+// inflected surface's reading does not match the dictionary form's reading,
+// and partial furigana readings (see isCompleteReadingForSurface) would cause
+// false negatives. Undefined falls back to text-only matching (fail-open).
+function resolveKnownWordReadingForMatch(
+  token: MergedToken,
+  knownWordMatchMode: NPlusOneMatchMode,
+): string | undefined {
+  if (knownWordMatchMode === 'headword') {
+    const headwordReading = token.headwordReading?.trim();
+    if (headwordReading) {
+      return headwordReading;
+    }
+    if (token.surface !== token.headword) {
+      return undefined;
+    }
+  }
+
+  return resolveCompleteTokenReading(token);
+}
+
 function computeTokenKnownStatus(
   token: MergedToken,
-  isKnownWord: (text: string) => boolean,
+  isKnownWord: (text: string, reading?: string) => boolean,
   knownWordMatchMode: NPlusOneMatchMode,
 ): boolean {
   const matchText = resolveKnownWordText(token.surface, token.headword, knownWordMatchMode);
-  if (token.isKnown || (matchText ? isKnownWord(matchText) : false)) {
+  const matchReading = resolveKnownWordReadingForMatch(token, knownWordMatchMode);
+  if (token.isKnown || (matchText ? isKnownWord(matchText, matchReading) : false)) {
     return true;
   }
 
-  const normalizedReading = token.reading.trim();
-  if (!normalizedReading) {
+  const fallbackReading = resolveCompleteTokenReading(token);
+  if (!fallbackReading) {
     return false;
   }
 
-  if (!isCompleteReadingForSurface(token.surface, normalizedReading)) {
-    return false;
-  }
-
-  return normalizedReading !== matchText.trim() && isKnownWord(normalizedReading);
+  return fallbackReading !== matchText.trim() && isKnownWord(fallbackReading);
 }
 
 function filterTokenFrequencyRank(

src/core/services/tokenizer/yomitan-parser-runtime.test.ts

@@ -966,6 +966,7 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF
       surface: '潜み',
       reading: 'ひそみ',
       headword: '潜む',
+      headwordReading: 'ひそむ',
       startPos: 0,
       endPos: 2,
       isNameMatch: false,
@@ -1032,6 +1033,7 @@ test('requestYomitanScanTokens emits complete readings for kanji-kana compounds'
       surface: '待ち合わせてる',
       reading: 'まちあわせてる',
       headword: '待ち合わせる',
+      headwordReading: 'まちあわせる',
       startPos: 0,
       endPos: 7,
       isNameMatch: false,
@@ -1139,6 +1141,7 @@ test('requestYomitanScanTokens uses frequency from later exact-match entry when
       surface: '者',
       reading: 'もの',
       headword: '者',
+      headwordReading: 'もの',
       startPos: 0,
       endPos: 1,
       isNameMatch: false,
@@ -1240,6 +1243,7 @@ test('requestYomitanScanTokens can use frequency from later exact secondary-matc
       surface: '者',
       reading: 'もの',
       headword: '者',
+      headwordReading: 'もの',
       startPos: 0,
       endPos: 1,
       isNameMatch: false,
@@ -1340,6 +1344,7 @@ test('requestYomitanScanTokens uses exact frequency entry when selected reading
     surface: '第二',
     reading: 'だいに',
     headword: '第二',
+    headwordReading: 'だいに',
     startPos: 0,
     endPos: 2,
     isNameMatch: false,

src/core/services/tokenizer/yomitan-parser-runtime.ts

@@ -51,6 +51,7 @@ export interface YomitanScanToken {
   surface: string;
   reading: string;
   headword: string;
+  headwordReading?: string;
   startPos: number;
   endPos: number;
   isNameMatch?: boolean;
@@ -92,6 +93,7 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
         typeof entry.surface === 'string' &&
         typeof entry.reading === 'string' &&
         typeof entry.headword === 'string' &&
+        (entry.headwordReading === undefined || typeof entry.headwordReading === 'string') &&
         typeof entry.startPos === 'number' &&
         typeof entry.endPos === 'number' &&
         (entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean') &&
@@ -1318,6 +1320,7 @@ ${YOMITAN_SCANNING_HELPERS}
               surface: segments.map((segment) => segment.text).join("") || source,
               reading: segments.map(getSegmentReadingContribution).join(""),
               headword: preferredHeadword.term,
+              headwordReading: reading || undefined,
               startPos: i,
               endPos: i + originalTextLength,
               isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,