fix(tokenizer): keep unparsed Yomitan tokens hoverable (#145)

changes/unparsed-token-hover-lookup.md

@@ -0,0 +1,4 @@
+type: fixed
+area: overlay
+
+- Fixed subtitle text that Yomitan's parser cannot match (e.g. the truncated volitional in とこ戻ろ…) being rendered as plain, non-interactive text: it was invisible to hover/lookup and excluded from the n+1 word count, which could wrongly mark a sentence as n+1. Unparsed runs are now kept as hoverable tokens matching Yomitan's own segmentation; bracketed SFX/speaker captions and punctuation-only runs are still skipped.

src/core/services/tokenizer.test.ts

@@ -1821,7 +1821,7 @@ test('tokenizeSubtitle returns null tokens when mecab throws', async () => {
   assert.deepEqual(result, { text: '猫です', tokens: null });
 });
 
-test('tokenizeSubtitle uses Yomitan parser result when available and drops no-headword groups', async () => {
+test('tokenizeSubtitle uses Yomitan parser result and keeps no-headword groups as surface tokens', async () => {
   const parserWindow = {
     isDestroyed: () => false,
     webContents: {
@@ -1859,10 +1859,12 @@ test('tokenizeSubtitle uses Yomitan parser result when available and drops no-he
   );
 
   assert.equal(result.text, '猫です');
-  assert.equal(result.tokens?.length, 1);
+  assert.equal(result.tokens?.length, 2);
   assert.equal(result.tokens?.[0]?.surface, '猫');
   assert.equal(result.tokens?.[0]?.reading, 'ねこ');
   assert.equal(result.tokens?.[0]?.isKnown, false);
+  assert.equal(result.tokens?.[1]?.surface, 'です');
+  assert.equal(result.tokens?.[1]?.headword, 'です');
 });
 
 test('tokenizeSubtitle logs selected Yomitan groups when debug toggle is enabled', async () => {

src/core/services/tokenizer/parser-selection-stage.test.ts

@@ -128,6 +128,61 @@ test('drops scanning parser tokens which have no dictionary headword', () => {
   );
 });
 
+// Regression: 「…とこ戻ろ…」 — Yomitan cannot deinflect the truncated volitional 戻ろ
+// (the bare ろ rule is ichidan-only, 戻る is godan), so 戻 comes back with no headword
+// while ろ matches an unrelated term (櫓). Dropping 戻 made it a plain text node:
+// unhoverable, unannotated, and invisible to the n+1 candidate count.
+test('emits unparsed non-caption text as a token with surface headword', () => {
+  const parseResults = [
+    makeParseItem('scanning-parser', [
+      [{ text: 'みんな', reading: 'みんな', headword: '皆' }],
+      [{ text: 'の', reading: 'の', headword: 'の' }],
+      [{ text: 'とこ', reading: 'とこ', headword: '所' }],
+      [{ text: '戻', reading: '' }],
+      [{ text: 'ろ', reading: 'ろ', headword: '櫓' }],
+      [{ text: '…', reading: '' }],
+    ]),
+  ];
+
+  const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
+  assert.deepEqual(
+    tokens?.map((token) => ({ surface: token.surface, headword: token.headword })),
+    [
+      { surface: 'みんな', headword: '皆' },
+      { surface: 'の', headword: 'の' },
+      { surface: 'とこ', headword: '所' },
+      { surface: '戻', headword: '戻' },
+      { surface: 'ろ', headword: '櫓' },
+    ],
+  );
+});
+
+test('still drops punctuation-only and whitespace-only unparsed runs', () => {
+  const parseResults = [
+    makeParseItem('scanning-parser', [
+      [{ text: '猫', reading: 'ねこ', headword: '猫' }],
+      [{ text: '…', reading: '' }],
+      [{ text: ' ', reading: '' }],
+      [{ text: '犬', reading: 'いぬ', headword: '犬' }],
+    ]),
+  ];
+
+  const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
+  assert.equal(tokens?.map((token) => token.surface).join(','), '猫,犬');
+});
+
+test('candidate with only unparsed tokens still yields no dictionary match', () => {
+  const parseResults = [
+    makeParseItem('scanning-parser', [
+      [{ text: '戻', reading: '' }],
+      [{ text: '轟', reading: '' }],
+    ]),
+  ];
+
+  const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
+  assert.equal(tokens, null);
+});
+
 test('prefers the longest dictionary headword across merged segments', () => {
   const parseResults = [
     makeParseItem('scanning-parser', [

src/core/services/tokenizer/parser-selection-stage.ts

@@ -151,6 +151,39 @@ function isStandaloneGrammarEndingSegment(segment: YomitanParseSegment): boolean
   );
 }
 
+// Caption-style asides (SFX/speaker labels) start with a bracket and should stay
+// non-interactive; dropping punctuation-only runs also keeps the source-text gap that
+// sentence-boundary detection relies on (e.g. a dropped 「…」).
+const CAPTION_OPENING_BRACKETS = new Set([
+  '(',
+  '（',
+  '[',
+  '［',
+  '{',
+  '｛',
+  '「',
+  '『',
+  '【',
+  '〈',
+  '《',
+  '≪',
+  '＜',
+  '<',
+]);
+
+function hasLookupWorthyText(text: string): boolean {
+  return /[\p{L}\p{N}]/u.test(text);
+}
+
+function isCaptionLikeUnparsedText(text: string): boolean {
+  const firstChar = Array.from(text.trim())[0];
+  return firstChar !== undefined && CAPTION_OPENING_BRACKETS.has(firstChar);
+}
+
+function shouldEmitUnparsedRunAsToken(text: string): boolean {
+  return hasLookupWorthyText(text) && !isCaptionLikeUnparsedText(text);
+}
+
 function shouldMergeKanaContinuation(
   previousToken: MergedToken | undefined,
   continuationSurface: string,
@@ -241,6 +274,11 @@ export function mapYomitanParseResultItemToMergedTokens(
           previousToken.surface += combinedSurface;
           previousToken.reading += combinedReading;
           previousToken.endPos = end;
+        } else if (shouldEmitUnparsedRunAsToken(combinedSurface)) {
+          // Yomitan couldn't parse this run (e.g. 戻ろ… truncated volitional). Keep it
+          // as a token with its surface as headword so it stays hoverable and counts in
+          // the n+1 math — matching what the embedded Yomitan actually returns.
+          pushToken(combinedSurface, combinedReading, combinedSurface, combinedStart, end);
         }
       } else {
         hasDictionaryMatch = true;