fix(tokenizer): block kanji reading collisions and name/generic length t

- Add allowReadingOnlyMatch:false to kanji token known-word lookups so 渓谷/けいこく no longer matches a mined 警告/けいこく card via reading-only index
- Greedy name pre-pass yields when a strictly longer generic word starts at the same position (空 no longer splits 空気; ties still go to the name)

changes/greedy-name-parsing-priority.md

@@ -1,4 +1,4 @@
 type: fixed
 area: overlay
 
-- Fixed character names being swallowed by longer generic dictionary matches during subtitle tokenization (e.g. ヨータ in 美姫とヨータ was lost because とヨー normalizes to とよう and matched 渡洋), so the name never tokenized and got no highlight, portrait, or hover lookup. Character-dictionary matches now claim their spans in a greedy pre-pass before the left-to-right scan, and re-segmented name spans override the parse tokenization on merge. The pre-pass runs only when a SubMiner character dictionary is enabled in the active Yomitan profile.
+- Fixed character names being swallowed by longer generic dictionary matches during subtitle tokenization (e.g. ヨータ in 美姫とヨータ was lost because とヨー normalizes to とよう and matched 渡洋), so the name never tokenized and got no highlight, portrait, or hover lookup. Character-dictionary matches now claim their spans in a greedy pre-pass before the left-to-right scan, and re-segmented name spans override the parse tokenization on merge. A name only claims its span when no strictly longer generic word starts at the same position, so a character named 空 no longer splits 空気 (ties still go to the name). The pre-pass runs only when a SubMiner character dictionary is enabled in the active Yomitan profile.

changes/known-word-reading-aware-matching.md

@@ -4,3 +4,4 @@ area: overlay
 - New reading-aware subtitle parsing and known-word matching: the known-word cache now stores each Anki card's word together with its reading (cache format v3), and a token only gets the known-word highlight when its parsed reading agrees with the card. The cache and the stats server upgrade automatically.
 - Fixes words being highlighted green as known when a same-spelled Anki card taught a different reading (e.g. とこ parsed as 床 "bed" no longer matches a known 床/ゆか "floor" card). Cards without a reading field keep matching in any reading as before.
 - Single-kana grammar tokens (よ in 全然いいよ, standalone え) no longer borrow the reading of an unrelated card (such as 夜 or 絵) and get painted as known: reading-only matching requires at least two kana, while single-kana cards still match by their word field.
+- Kanji words no longer count as known just because a different mined kanji word shares their reading (渓谷/けいこく no longer matches a known 警告/けいこく card). Words mined in kana form still highlight their kanji spelling in subtitles, and kana subtitle text still matches cards mined in kanji form.

src/anki-integration.ts

@@ -703,8 +703,12 @@ export class AnkiIntegration {
     });
   }
 
-  isKnownWord(text: string, reading?: string): boolean {
-    return this.knownWordCache.isKnownWord(text, reading);
+  isKnownWord(
+    text: string,
+    reading?: string,
+    options?: { allowReadingOnlyMatch?: boolean },
+  ): boolean {
+    return this.knownWordCache.isKnownWord(text, reading, options);
   }
 
   getKnownWordMatchMode(): NPlusOneMatchMode {

src/anki-integration/known-word-cache.test.ts

@@ -734,6 +734,42 @@ test('KnownWordCacheManager disambiguates known words by note reading', async ()
   }
 });
 
+test('KnownWordCacheManager suppresses reading-only matches when disallowed', async () => {
+  const config: AnkiConnectConfig = {
+    fields: {
+      word: 'Word',
+    },
+    knownWords: {
+      highlightEnabled: true,
+    },
+  };
+  const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
+
+  try {
+    clientState.findNotesResult = [1];
+    clientState.notesInfoResult = [
+      {
+        noteId: 1,
+        fields: {
+          Word: { value: '警告' },
+          'Word Reading': { value: 'けいこく' },
+        },
+      },
+    ];
+
+    await manager.refresh(true);
+
+    // Reading-only match stays available for kana subtitle text…
+    assert.equal(manager.isKnownWord('けいこく'), true);
+    // …but a kanji token's reading (渓谷/けいこく) must not borrow 警告's.
+    assert.equal(manager.isKnownWord('けいこく', undefined, { allowReadingOnlyMatch: false }), false);
+    // Mined word texts still match regardless of the flag.
+    assert.equal(manager.isKnownWord('警告', undefined, { allowReadingOnlyMatch: false }), true);
+  } finally {
+    cleanup();
+  }
+});
+
 test('KnownWordCacheManager does not match single-kana text by reading alone', async () => {
   const config: AnkiConnectConfig = {
     fields: {

src/anki-integration/known-word-cache.ts

@@ -142,7 +142,11 @@ export class KnownWordCacheManager {
     );
   }
 
-  isKnownWord(text: string, reading?: string): boolean {
+  isKnownWord(
+    text: string,
+    reading?: string,
+    options?: { allowReadingOnlyMatch?: boolean },
+  ): boolean {
     if (!this.isKnownWordCacheEnabled()) {
       return false;
     }
@@ -163,6 +167,14 @@ export class KnownWordCacheManager {
       );
     }
 
+    // Callers that look up a kanji token's reading (not subtitle text) must
+    // opt out of the reading-only fallback: readingCounts holds readings of
+    // every note including kanji words, so 渓谷's けいこく would match a
+    // mined 警告/けいこく.
+    if (options?.allowReadingOnlyMatch === false) {
+      return false;
+    }
+
     // Reading-only fallback, except for single-kana text: particles and
     // interjections (よ, ね, え…) would otherwise borrow the reading of an
     // unrelated note (夜「よ」, 絵「え」) and count as known.

src/core/services/tokenizer.ts

@@ -33,6 +33,15 @@ type MecabTokenEnrichmentFn = (
   mecabTokens: MergedToken[] | null,
 ) => Promise<MergedToken[]>;
 
+// allowReadingOnlyMatch: false suppresses the cache's reading-only index for
+// lookups that pass a kanji token's reading as the text (see
+// computeTokenKnownStatus in annotation-stage).
+export type KnownWordLookupFn = (
+  text: string,
+  reading?: string,
+  options?: { allowReadingOnlyMatch?: boolean },
+) => boolean;
+
 export interface TokenizerServiceDeps {
   getYomitanExt: () => Extension | null;
   getYomitanSession?: () => Session | null;
@@ -42,7 +51,7 @@ export interface TokenizerServiceDeps {
   setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
   getYomitanParserInitPromise: () => Promise<boolean> | null;
   setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
-  isKnownWord: (text: string, reading?: string) => boolean;
+  isKnownWord: KnownWordLookupFn;
   getKnownWordMatchMode: () => NPlusOneMatchMode;
   getKnownWordsEnabled?: () => boolean;
   getJlptLevel: (text: string) => JlptLevel | null;
@@ -77,7 +86,7 @@ export interface TokenizerDepsRuntimeOptions {
   setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
   getYomitanParserInitPromise: () => Promise<boolean> | null;
   setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
-  isKnownWord: (text: string, reading?: string) => boolean;
+  isKnownWord: KnownWordLookupFn;
   getKnownWordMatchMode: () => NPlusOneMatchMode;
   getKnownWordsEnabled?: () => boolean;
   getJlptLevel: (text: string) => JlptLevel | null;
@@ -129,7 +138,7 @@ const INVISIBLE_SEPARATOR_PATTERN = /[\u200b\u2060\ufeff]/g;
 function getKnownWordLookup(
   deps: TokenizerServiceDeps,
   options: TokenizerAnnotationOptions,
-): (text: string, reading?: string) => boolean {
+): KnownWordLookupFn {
   if (!options.knownWordsEnabled && !options.nPlusOneEnabled) {
     return () => false;
   }

src/core/services/tokenizer/annotation-stage.test.ts

@@ -273,6 +273,29 @@ test('annotateTokens falls back to reading for known-word matches when headword
   assert.equal(result[0]?.frequencyRank, 1895);
 });
 
+test('annotateTokens reading fallback does not match kanji tokens sharing a mined reading', () => {
+  const tokens = [
+    makeToken({
+      surface: '渓谷',
+      headword: '渓谷',
+      reading: 'けいこく',
+      endPos: 2,
+    }),
+  ];
+
+  const result = annotateTokens(
+    tokens,
+    makeDeps({
+      // Mimics the cache with a mined 警告/けいこく: けいこく matches through
+      // the reading-only index unless the lookup opts out of it.
+      isKnownWord: (text, _reading, options) =>
+        text === '警告' || (options?.allowReadingOnlyMatch !== false && text === 'けいこく'),
+    }),
+  );
+
+  assert.equal(result[0]?.isKnown, false);
+});
+
 test('annotateTokens ignores partial furigana readings for known-word fallback', () => {
   const tokens = [
     makeToken({

src/core/services/tokenizer/annotation-stage.ts

@@ -26,7 +26,11 @@ const jlptLevelLookupCaches = new WeakMap<
 >();
 
 export interface AnnotationStageDeps {
-  isKnownWord: (text: string, reading?: string) => boolean;
+  isKnownWord: (
+    text: string,
+    reading?: string,
+    options?: { allowReadingOnlyMatch?: boolean },
+  ) => boolean;
   knownWordMatchMode: NPlusOneMatchMode;
   getJlptLevel: (text: string) => JlptLevel | null;
 }
@@ -654,7 +658,7 @@ function resolveKnownWordReadingForMatch(
 
 function computeTokenKnownStatus(
   token: MergedToken,
-  isKnownWord: (text: string, reading?: string) => boolean,
+  isKnownWord: AnnotationStageDeps['isKnownWord'],
   knownWordMatchMode: NPlusOneMatchMode,
 ): boolean {
   const matchText = resolveKnownWordText(token.surface, token.headword, knownWordMatchMode);
@@ -668,7 +672,14 @@ function computeTokenKnownStatus(
     return false;
   }
 
-  return fallbackReading !== matchText.trim() && isKnownWord(fallbackReading);
+  // This fallback covers words mined in kana (token 大体, mined word だいたい),
+  // so the reading must only match mined word texts — the cache's reading-only
+  // index would let any kanji token match an unrelated note that shares its
+  // reading (渓谷/けいこく vs a mined 警告/けいこく).
+  return (
+    fallbackReading !== matchText.trim() &&
+    isKnownWord(fallbackReading, undefined, { allowReadingOnlyMatch: false })
+  );
 }
 
 function filterTokenFrequencyRank(

src/core/services/tokenizer/yomitan-parser-runtime.test.ts

@@ -1995,6 +1995,107 @@ test('requestYomitanScanTokens greedily tokenizes character names before longer
   );
 });
 
+test('requestYomitanScanTokens lets a longer generic word beat a shorter name at the same position', async () => {
+  let scannerScript = '';
+  const deps = createDeps(async (script) => {
+    if (script.includes('termsFind')) {
+      scannerScript = script;
+      return [];
+    }
+    if (script.includes('optionsGetFull')) {
+      return {
+        profileCurrent: 0,
+        profiles: [
+          {
+            options: {
+              scanning: { length: 40 },
+              dictionaries: [
+                { name: 'JMdict', enabled: true },
+                { name: 'SubMiner Character Dictionary (AniList 130298)', enabled: true },
+              ],
+            },
+          },
+        ],
+      };
+    }
+    return null;
+  });
+
+  await requestYomitanScanTokens(
+    '空気変わって',
+    deps,
+    { error: () => undefined },
+    { includeNameMatchMetadata: true },
+  );
+
+  assert.match(scannerScript, /const greedyNameScanEnabled = true;/);
+
+  const nameEntry = (term: string, reading: string) => ({
+    headwords: [
+      {
+        term,
+        reading,
+        sources: [{ originalText: term, isPrimary: true, matchType: 'exact' }],
+      },
+    ],
+    definitions: [
+      {
+        dictionary: 'SubMiner Character Dictionary (AniList 130298)',
+        dictionaryAlias: 'SubMiner Character Dictionary (AniList 130298)',
+      },
+    ],
+  });
+  const jmdictEntry = (term: string, reading: string, originalText: string) => ({
+    headwords: [
+      {
+        term,
+        reading,
+        sources: [{ originalText, isPrimary: true, matchType: 'exact' }],
+      },
+    ],
+    definitions: [{ dictionary: 'JMdict', dictionaryAlias: 'JMdict' }],
+  });
+
+  const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
+    if (action !== 'termsFind') {
+      throw new Error(`unexpected action: ${action}`);
+    }
+    const text = (params as { text?: string } | undefined)?.text ?? '';
+    if (text.startsWith('空気')) {
+      // A character named 空 matches here, but the generic 空気 is longer and
+      // must win the position.
+      return {
+        originalTextLength: 2,
+        dictionaryEntries: [nameEntry('空', 'くう'), jmdictEntry('空気', 'くうき', '空気')],
+      };
+    }
+    if (text.startsWith('変わって')) {
+      return {
+        originalTextLength: 4,
+        dictionaryEntries: [jmdictEntry('変わる', 'かわる', '変わって')],
+      };
+    }
+    return { originalTextLength: 0, dictionaryEntries: [] };
+  });
+
+  assert.equal(Array.isArray(result), true);
+  assert.deepEqual(
+    (result as Array<Record<string, unknown>>).map(
+      ({ surface, headword, startPos, endPos, isNameMatch }) => ({
+        surface,
+        headword,
+        startPos,
+        endPos,
+        isNameMatch,
+      }),
+    ),
+    [
+      { surface: '空気', headword: '空気', startPos: 0, endPos: 2, isNameMatch: false },
+      { surface: '変わって', headword: '変わる', startPos: 2, endPos: 6, isNameMatch: false },
+    ],
+  );
+});
+
 test('requestYomitanScanTokens skips greedy name scan without an enabled character dictionary', async () => {
   let scannerScript = '';
   const deps = createDeps(async (script) => {

src/core/services/tokenizer/yomitan-parser-runtime.ts

@@ -1304,6 +1304,22 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
         }
         return best;
       }
+      function findLongestGenericMatchLength(dictionaryEntries, textWindow) {
+        let best = 0;
+        for (const dictionaryEntry of dictionaryEntries || []) {
+          if (isNameDictionaryEntry(dictionaryEntry)) { continue; }
+          const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
+          for (const headword of headwords) {
+            for (const src of headword?.sources || []) {
+              if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
+              const originalText = typeof src.originalText === 'string' ? src.originalText : '';
+              if (!originalText || !textWindow.startsWith(originalText)) { continue; }
+              if (originalText.length > best) { best = originalText.length; }
+            }
+          }
+        }
+        return best;
+      }
       function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
         const currentMediaDictionaryEntries =
           currentCharacterDictionaryMediaId === null
@@ -1455,8 +1471,16 @@ ${YOMITAN_SCANNING_HELPERS}
           }
           const result = await termsFindAt(namePos, ${scanLength});
           const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
-          const nameMatch = findLongestNameMatch(dictionaryEntries, text.substring(namePos, namePos + ${scanLength}));
-          if (!nameMatch) {
+          const textWindow = text.substring(namePos, namePos + ${scanLength});
+          const nameMatch = findLongestNameMatch(dictionaryEntries, textWindow);
+          // A name only claims its span when no strictly longer generic word
+          // starts at the same position (a character named 空 must not split
+          // 空気). Ties go to the name. Generic matches that start earlier and
+          // overlap the name are still blocked by the reservation.
+          if (
+            !nameMatch ||
+            findLongestGenericMatchLength(dictionaryEntries, textWindow) > nameMatch.sourceLength
+          ) {
             namePos += String.fromCodePoint(codePoint).length;
             continue;
           }

src/main.ts

@@ -4652,7 +4652,8 @@ const {
       setYomitanParserInitPromise: (promise) => {
         appState.yomitanParserInitPromise = promise;
       },
-      isKnownWord: (text, reading) => Boolean(appState.ankiIntegration?.isKnownWord(text, reading)),
+      isKnownWord: (text, reading, options) =>
+        Boolean(appState.ankiIntegration?.isKnownWord(text, reading, options)),
       recordLookup: (hit) => {
         ensureImmersionTrackerStarted();
         appState.immersionTracker?.recordLookup(hit);

src/main/runtime/subtitle-tokenization-main-deps.ts

@@ -37,8 +37,8 @@ export function createBuildTokenizerDepsMainHandler(deps: TokenizerMainDeps) {
     getYomitanParserInitPromise: () => deps.getYomitanParserInitPromise(),
     setYomitanParserInitPromise: (promise: Promise<boolean> | null) =>
       deps.setYomitanParserInitPromise(promise),
-    isKnownWord: (text: string, reading?: string) => {
-      const hit = deps.isKnownWord(text, reading);
+    isKnownWord: (text, reading, options) => {
+      const hit = deps.isKnownWord(text, reading, options);
       deps.recordLookup(hit);
       return hit;
     },