fix(tokenizer): exclude unparsed-run tokens from annotations and N+1 (#153)

changes/known-word-reading-aware-matching.md

@@ -1,4 +1,6 @@
-type: fixed
+type: changed
 area: overlay
 
-- Fixed words being highlighted green as known when a same-spelled Anki card taught a different reading (e.g. とこ parsed as 床 "bed" matching a known 床/ゆか "floor" card). The known-word cache now stores each card's word together with its reading and only matches when the token's reading agrees; cards without a reading field keep matching in any reading as before.
+- New reading-aware subtitle parsing and known-word matching: the known-word cache now stores each Anki card's word together with its reading (cache format v3), and a token only gets the known-word highlight when its parsed reading agrees with the card. The cache and the stats server upgrade automatically.
+- Fixes words being highlighted green as known when a same-spelled Anki card taught a different reading (e.g. とこ parsed as 床 "bed" no longer matches a known 床/ゆか "floor" card). Cards without a reading field keep matching in any reading as before.
+- Single-kana grammar tokens (よ in 全然いいよ, standalone え) no longer borrow the reading of an unrelated card (such as 夜 or 絵) and get painted as known: reading-only matching requires at least two kana, while single-kana cards still match by their word field.

changes/particle-known-word-highlight.md

@@ -1,5 +1,4 @@
-type: fixed
+type: changed
 area: overlay
 
-- Fixed single-kana grammar tokens counting as known words by borrowing the reading of an unrelated Anki note (よ in 全然いいよ matched a card read よ such as 夜, standalone え matched 絵), which painted them with the known-word highlight. Reading-only known-word matching now requires at least two kana; single-kana cards still match by their word field, and tokens genuinely present in the known-words cache (e.g. です) keep their highlight.
 - Standalone suffix tokens (MeCab pos2 接尾, e.g. さん, れる) are now excluded from JLPT/frequency/N+1 annotations by default, matching how particles and interjections are treated. Cache-backed known-word highlighting still applies; override via the pos2 exclusion config if you want them annotated.

changes/stats-known-words-v3-cache.md

@@ -1,4 +0,0 @@
-type: fixed
-area: stats
-
-- Session known-word counts no longer show 0 everywhere. The stats server's known-word cache parser only understood the v1/v2 cache formats, so after the reading-aware v3 cache upgrade it silently treated the cache as missing; it now flattens v3 note entries into the headword set.

changes/unparsed-token-hover-lookup.md

@@ -1,4 +1,4 @@
 type: fixed
 area: overlay
 
-- Fixed subtitle text that Yomitan's parser cannot match (e.g. the truncated volitional in とこ戻ろ…) being rendered as plain, non-interactive text: it was invisible to hover/lookup and excluded from the n+1 word count, which could wrongly mark a sentence as n+1. Unparsed runs are now kept as hoverable tokens matching Yomitan's own segmentation; bracketed SFX/speaker captions and punctuation-only runs are still skipped.
+- Fixed subtitle text that Yomitan's parser cannot match (truncated inflections like the volitional in とこ戻ろ…, elongation runs like ぅ～/ぉ〜) being rendered as plain, non-interactive text with no hover lookup. Unparsed runs are now kept as hoverable tokens matching Yomitan's own segmentation, while being fully ignored by frequency/JLPT highlighting, the N+1 candidate math, and vocabulary stats. Bracketed SFX/speaker captions and punctuation-only runs are still skipped.

src/core/services/tokenizer.test.ts

@@ -28,6 +28,7 @@ interface YomitanTokenInput {
   frequencyRank?: number;
   isNameMatch?: boolean;
   wordClasses?: string[];
+  isUnparsedRun?: boolean;
 }
 
 function makeDepsFromYomitanTokens(
@@ -60,6 +61,7 @@ function makeDepsFromYomitanTokens(
                 isNameMatch: token.isNameMatch ?? false,
                 frequencyRank: token.frequencyRank,
                 wordClasses: token.wordClasses,
+                isUnparsedRun: token.isUnparsedRun,
               };
             });
           },
@@ -4223,6 +4225,38 @@ test('tokenizeSubtitle clears all annotations for explanatory pondering endings'
   );
 });
 
+test('tokenizeSubtitle ignores unparsed-run tokens for annotations and N+1', async () => {
+  // もう いるぅ～！: the ぅ～ elongation has no Yomitan dictionary entry; it must
+  // not become the sole N+1 candidate or receive frequency/JLPT annotations.
+  const result = await tokenizeSubtitle(
+    'もう いるぅ～！',
+    makeDepsFromYomitanTokens(
+      [
+        { surface: 'もう', reading: 'もう', headword: 'もう' },
+        { surface: 'いる', reading: 'いる', headword: 'いる' },
+        { surface: 'ぅ～', reading: '', headword: 'ぅ～', isUnparsedRun: true, frequencyRank: 999 },
+      ],
+      {
+        getFrequencyDictionaryEnabled: () => true,
+        getJlptLevel: (text) => (text === 'ぅ～' ? 'N5' : null),
+        isKnownWord: (text) => text === 'もう' || text === 'いる',
+        getMinSentenceWordsForNPlusOne: () => 2,
+        tokenizeWithMecab: async () => null,
+      },
+    ),
+  );
+
+  const filler = result.tokens?.find((token) => token.surface === 'ぅ～');
+  assert.ok(filler);
+  assert.equal(filler?.isNPlusOneTarget, false);
+  assert.equal(filler?.frequencyRank, undefined);
+  assert.equal(filler?.jlptLevel, undefined);
+  assert.equal(
+    result.tokens?.some((token) => token.isNPlusOneTarget),
+    false,
+  );
+});
+
 test('tokenizeSubtitle keeps frequency for content-led merged token with trailing colloquial suffixes', async () => {
   const result = await tokenizeSubtitle(
     '張り切ってんじゃ',

src/core/services/tokenizer.ts

@@ -734,6 +734,7 @@ async function parseWithYomitanInternalParser(
         isNPlusOneTarget: false,
         isNameMatch: token.isNameMatch ?? false,
         frequencyRank: token.frequencyRank,
+        isUnparsedRun: token.isUnparsedRun === true ? true : undefined,
       };
     }),
   );

src/core/services/tokenizer/annotation-stage.test.ts

@@ -208,6 +208,48 @@ test('annotateTokens hides known-word marks while still using known words for N+
   assert.equal(result[2]?.isNPlusOneTarget, true);
 });
 
+test('shouldExcludeTokenFromSubtitleAnnotations excludes unparsed-run tokens', () => {
+  // 戻 from 「…とこ戻ろ…」: Yomitan had no dictionary entry, headword falls back
+  // to the surface. Without the flag the token passes every other filter.
+  const unflagged = makeToken({ surface: '戻', headword: '戻', reading: '' });
+  assert.equal(shouldExcludeTokenFromSubtitleAnnotations(unflagged), false);
+
+  const flagged = makeToken({ surface: '戻', headword: '戻', reading: '', isUnparsedRun: true });
+  assert.equal(shouldExcludeTokenFromSubtitleAnnotations(flagged), true);
+  assert.equal(shouldExcludeTokenFromVocabularyPersistence(flagged), true);
+});
+
+test('annotateTokens ignores unparsed-run tokens for annotations and N+1', () => {
+  // もう いるぅ～！-style line: the elongation run is the only unknown token and
+  // used to become the sole N+1 candidate despite having no dictionary entry.
+  const tokens = [
+    makeToken({ surface: 'みんな', headword: '皆', reading: 'みんな', startPos: 0, endPos: 3 }),
+    makeToken({ surface: 'とこ', headword: '所', reading: 'とこ', startPos: 3, endPos: 5 }),
+    makeToken({
+      surface: '戻',
+      headword: '戻',
+      reading: '',
+      startPos: 5,
+      endPos: 6,
+      isUnparsedRun: true,
+      frequencyRank: 12,
+    }),
+  ];
+
+  const result = annotateTokens(
+    tokens,
+    makeDeps({
+      isKnownWord: (text) => text === '皆' || text === '所',
+      getJlptLevel: (text) => (text === '戻' ? 'N5' : null),
+    }),
+    { minSentenceWordsForNPlusOne: 2 },
+  );
+
+  assert.equal(result[2]?.isNPlusOneTarget, false);
+  assert.equal(result[2]?.jlptLevel, undefined);
+  assert.equal(result[2]?.frequencyRank, undefined);
+});
+
 test('annotateTokens falls back to reading for known-word matches when headword lookup misses', () => {
   const tokens = [
     makeToken({

src/core/services/tokenizer/parser-selection-stage.test.ts

@@ -146,13 +146,17 @@ test('emits unparsed non-caption text as a token with surface headword', () => {
 
   const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
   assert.deepEqual(
-    tokens?.map((token) => ({ surface: token.surface, headword: token.headword })),
+    tokens?.map((token) => ({
+      surface: token.surface,
+      headword: token.headword,
+      isUnparsedRun: token.isUnparsedRun ?? false,
+    })),
     [
-      { surface: 'みんな', headword: '皆' },
+      { surface: 'みんな', headword: '皆', isUnparsedRun: false },
-      { surface: 'の', headword: 'の' },
+      { surface: 'の', headword: 'の', isUnparsedRun: false },
-      { surface: 'とこ', headword: '所' },
+      { surface: 'とこ', headword: '所', isUnparsedRun: false },
-      { surface: '戻', headword: '戻' },
+      { surface: '戻', headword: '戻', isUnparsedRun: true },
-      { surface: 'ろ', headword: '櫓' },
+      { surface: 'ろ', headword: '櫓', isUnparsedRun: false },
     ],
   );
 });

src/core/services/tokenizer/parser-selection-stage.ts

@@ -239,6 +239,7 @@ export function mapYomitanParseResultItemToMergedTokens(
       headword: string,
       start: number,
       end: number,
+      isUnparsedRun = false,
     ): void => {
       tokens.push({
         surface,
@@ -254,6 +255,7 @@ export function mapYomitanParseResultItemToMergedTokens(
           const matchText = resolveKnownWordText(surface, headword, knownWordMatchMode);
           return matchText ? isKnownWord(matchText) : false;
         })(),
+        ...(isUnparsedRun ? { isUnparsedRun: true } : {}),
       });
     };
 
@@ -275,10 +277,11 @@ export function mapYomitanParseResultItemToMergedTokens(
           previousToken.reading += combinedReading;
           previousToken.endPos = end;
         } else if (shouldEmitUnparsedRunAsToken(combinedSurface)) {
-          // Yomitan couldn't parse this run (e.g. 戻ろ… truncated volitional). Keep it
+          // Yomitan couldn't parse this run (e.g. 戻ろ… truncated volitional, ぅ～
-          // as a token with its surface as headword so it stays hoverable and counts in
+          // elongations). Keep it as a token with its surface as headword so it stays
-          // the n+1 math — matching what the embedded Yomitan actually returns.
+          // hoverable, but flag it so annotation/N+1/vocab logic ignores it — there is
-          pushToken(combinedSurface, combinedReading, combinedSurface, combinedStart, end);
+          // no dictionary entry behind it.
+          pushToken(combinedSurface, combinedReading, combinedSurface, combinedStart, end, true);
         }
       } else {
         hasDictionaryMatch = true;

src/core/services/tokenizer/subtitle-annotation-filter.ts

@@ -474,6 +474,13 @@ export function shouldExcludeTokenFromSubtitleAnnotations(
   token: MergedToken,
   options: SubtitleAnnotationFilterOptions = {},
 ): boolean {
+  // No Yomitan dictionary entry backs this token (ぅ～ elongations, truncated
+  // inflections) — it exists only to stay hoverable and must never receive
+  // annotations or count in the N+1 math.
+  if (token.isUnparsedRun === true) {
+    return true;
+  }
+
   const pos1Exclusions = resolvePos1Exclusions(options);
   const pos2Exclusions = resolvePos2Exclusions(options);
   const normalizedPos1 = normalizePosTag(token.pos1);

src/core/services/tokenizer/yomitan-parser-runtime.test.ts

@@ -934,6 +934,7 @@ test('requestYomitanScanTokens keeps scanner metadata for matching spans when pa
       headword: 'っ ',
       startPos: 2,
       endPos: 4,
+      isUnparsedRun: true,
     },
     {
       surface: 'ミナト',

src/core/services/tokenizer/yomitan-parser-runtime.ts

@@ -57,6 +57,7 @@ export interface YomitanScanToken {
   isNameMatch?: boolean;
   frequencyRank?: number;
   wordClasses?: string[];
+  isUnparsedRun?: boolean;
 }
 
 interface YomitanProfileMetadata {
@@ -110,6 +111,27 @@ function scanTokenSpanKey(token: YomitanScanToken): string {
   return `${token.startPos}:${token.endPos}:${token.surface}`;
 }
 
+// Maps a parse-selected token to the scanner-token shape carried out of the
+// parser runtime. Shared by both selectYomitanParseTokens fallback paths so the
+// projected fields stay in sync as the shape changes.
+function toYomitanScanToken(token: {
+  surface: string;
+  reading: string;
+  headword: string;
+  startPos: number;
+  endPos: number;
+  isUnparsedRun?: boolean;
+}): YomitanScanToken {
+  return {
+    surface: token.surface,
+    reading: token.reading,
+    headword: token.headword,
+    startPos: token.startPos,
+    endPos: token.endPos,
+    ...(token.isUnparsedRun === true ? { isUnparsedRun: true } : {}),
+  };
+}
+
 // parseText segmentation is authoritative (it emits filler chunks for text the
 // termsFind scanner skips), but only the termsFind scanner carries annotation
 // metadata (isNameMatch, frequencyRank, headwordReading, wordClasses). Graft
@@ -1611,14 +1633,7 @@ export async function requestYomitanScanTokens(
 
   const parseResults = await requestYomitanParseResults(text, deps, logger);
   const selectedParseTokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
-  const parseScanTokens =
+  const parseScanTokens = selectedParseTokens?.map(toYomitanScanToken) ?? null;
-    selectedParseTokens?.map((token) => ({
-      surface: token.surface,
-      reading: token.reading,
-      headword: token.headword,
-      startPos: token.startPos,
-      endPos: token.endPos,
-    })) ?? null;
 
   const metadata = await requestYomitanProfileMetadata(parserWindow, logger);
   const profileIndex = metadata?.profileIndex ?? 0;
@@ -1656,15 +1671,7 @@ export async function requestYomitanScanTokens(
     }
     if (Array.isArray(rawResult)) {
       const selectedTokens = selectYomitanParseTokens(rawResult, () => false, 'headword');
-      return (
+      return selectedTokens?.map(toYomitanScanToken) ?? null;
-        selectedTokens?.map((token) => ({
-          surface: token.surface,
-          reading: token.reading,
-          headword: token.headword,
-          startPos: token.startPos,
-          endPos: token.endPos,
-        })) ?? null
-      );
     }
     if (parseScanTokens && parseScanTokens.length > 0) {
       return parseScanTokens;

src/types/subtitle.ts

@@ -40,6 +40,12 @@ export interface MergedToken {
   isMerged: boolean;
   isKnown: boolean;
   isNPlusOneTarget: boolean;
+  /**
+   * Text Yomitan had no dictionary entry for (e.g. ぅ～ elongation runs,
+   * truncated inflections). Kept as a token so it stays hoverable, but
+   * ignored by annotation, N+1, and vocabulary-stats logic.
+   */
+  isUnparsedRun?: boolean;
   isNameMatch?: boolean;
   characterImage?: CharacterNameImage;
   jlptLevel?: JlptLevel;