fix(tokenizer): merge scanner metadata per token instead of all-or-nothi

- Replace hasSameTokenSpans + full-discard with mergeScannerTokensIntoParseTokens
- Grafts isNameMatch/frequencyRank/etc onto matching parse spans; filler chunks degrade only themselves
- Fixes name annotations dropping for entire subtitle lines containing unmatched interjections

changes/name-match-span-merge.md

@@ -0,0 +1,4 @@
+type: fixed
+area: overlay
+
+- Fixed character-name annotations dropping for an entire subtitle line when it contained any chunk the dictionary scanner could not match (e.g. an interjection like やほっ before a name): scanner metadata is now merged per token into the parseText segmentation instead of being discarded on any span mismatch.

src/core/services/tokenizer/yomitan-parser-runtime.test.ts

@@ -820,6 +820,134 @@ test('requestYomitanScanTokens keeps scanner metadata when parse spans agree', a
   ]);
 });
 
+test('requestYomitanScanTokens keeps scanner metadata for matching spans when parse segmentation has filler chunks', async () => {
+  const deps = createDeps(async (script) => {
+    if (script.includes('optionsGetFull')) {
+      return {
+        profileCurrent: 0,
+        profiles: [
+          {
+            options: {
+              scanning: { length: 40 },
+            },
+          },
+        ],
+      };
+    }
+    if (script.includes('parseText')) {
+      return [
+        {
+          source: 'scanning-parser',
+          index: 0,
+          content: [
+            [
+              {
+                text: 'や',
+                reading: '',
+                headwords: [[{ term: 'や' }]],
+              },
+            ],
+            [
+              {
+                text: 'ほ',
+                reading: '',
+                headwords: [[{ term: '帆' }]],
+              },
+            ],
+            [
+              {
+                text: 'っ ',
+                reading: '',
+              },
+            ],
+            [
+              {
+                text: 'ミナト',
+                reading: '',
+                headwords: [[{ term: 'ミナト' }]],
+              },
+            ],
+          ],
+        },
+      ];
+    }
+    // The termsFind scanner skips the unmatched っ+space chunk, so its spans
+    // do not line up 1:1 with the parseText segmentation above.
+    return [
+      {
+        surface: 'や',
+        reading: 'や',
+        headword: 'や',
+        headwordReading: 'や',
+        startPos: 0,
+        endPos: 1,
+        frequencyRank: 57,
+      },
+      {
+        surface: 'ほ',
+        reading: 'ほ',
+        headword: '帆',
+        headwordReading: 'ほ',
+        startPos: 1,
+        endPos: 2,
+        frequencyRank: 15414,
+      },
+      {
+        surface: 'ミナト',
+        reading: 'ミナト',
+        headword: 'ミナト',
+        headwordReading: 'みなと',
+        startPos: 4,
+        endPos: 7,
+        isNameMatch: true,
+        frequencyRank: 75133,
+      },
+    ];
+  });
+
+  const result = await requestYomitanScanTokens('やほっ ミナト', deps, {
+    error: () => undefined,
+  });
+
+  assert.deepEqual(result, [
+    {
+      surface: 'や',
+      reading: 'や',
+      headword: 'や',
+      headwordReading: 'や',
+      startPos: 0,
+      endPos: 1,
+      frequencyRank: 57,
+    },
+    {
+      surface: 'ほ',
+      reading: 'ほ',
+      headword: '帆',
+      headwordReading: 'ほ',
+      startPos: 1,
+      endPos: 2,
+      frequencyRank: 15414,
+    },
+    {
+      surface: 'っ ',
+      reading: '',
+      headword: 'っ ',
+      startPos: 2,
+      endPos: 4,
+    },
+    {
+      surface: 'ミナト',
+      reading: 'ミナト',
+      headword: 'ミナト',
+      headwordReading: 'みなと',
+      startPos: 4,
+      endPos: 7,
+      isNameMatch: true,
+      frequencyRank: 75133,
+    },
+  ]);
+});
+
 test('requestYomitanScanTokens falls back to left-to-right termsFind scanning', async () => {
   const scripts: string[] = [];
   const deps = createDeps(async (script) => {

src/core/services/tokenizer/yomitan-parser-runtime.ts

@@ -105,20 +105,26 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
   );
 }
 
-function hasSameTokenSpans(left: YomitanScanToken[], right: YomitanScanToken[]): boolean {
+function scanTokenSpanKey(token: YomitanScanToken): string {
-  if (left.length !== right.length) {
+  return `${token.startPos}:${token.endPos}:${token.surface}`;
-    return false;
 }
 
-  return left.every((token, index) => {
+// parseText segmentation is authoritative (it emits filler chunks for text the
-    const other = right[index];
+// termsFind scanner skips), but only the termsFind scanner carries annotation
-    return (
+// metadata (isNameMatch, frequencyRank, headwordReading, wordClasses). Graft
-      other !== undefined &&
+// scanner tokens onto the parseText segmentation per matching span so one
-      token.surface === other.surface &&
+// unmatched chunk degrades only itself instead of dropping the whole line's
-      token.startPos === other.startPos &&
+// metadata.
-      token.endPos === other.endPos
+function mergeScannerTokensIntoParseTokens(
-    );
+  parseScanTokens: YomitanScanToken[],
-  });
+  scannerTokens: YomitanScanToken[],
+): YomitanScanToken[] {
+  const scannerTokensBySpan = new Map<string, YomitanScanToken>();
+  for (const token of scannerTokens) {
+    scannerTokensBySpan.set(scanTokenSpanKey(token), token);
+  }
+
+  return parseScanTokens.map((token) => scannerTokensBySpan.get(scanTokenSpanKey(token)) ?? token);
 }
 
 function makeTermReadingCacheKey(term: string, reading: string | null): string {
@@ -1514,7 +1520,7 @@ export async function requestYomitanScanTokens(
     );
     if (isScanTokenArray(rawResult)) {
       if (parseScanTokens && parseScanTokens.length > 0) {
-        return hasSameTokenSpans(parseScanTokens, rawResult) ? rawResult : parseScanTokens;
+        return mergeScannerTokensIntoParseTokens(parseScanTokens, rawResult);
       }
       return rawResult;
     }