fix(tokenizer): greedy name pre-pass to prevent generic matches swallowing character names (#151)

changes/greedy-name-parsing-priority.md

@@ -0,0 +1,4 @@
+type: fixed
+area: overlay
+
+- Fixed character names being swallowed by longer generic dictionary matches during subtitle tokenization (e.g. ヨータ in 美姫とヨータ was lost because とヨー normalizes to とよう and matched 渡洋), so the name never tokenized and got no highlight, portrait, or hover lookup. Character-dictionary matches now claim their spans in a greedy pre-pass before the left-to-right scan, and re-segmented name spans override the parse tokenization on merge. The pre-pass runs only when a SubMiner character dictionary is enabled in the active Yomitan profile.

src/core/services/tokenizer/yomitan-parser-runtime.test.ts

@@ -1889,6 +1889,263 @@ test('requestYomitanScanTokens accepts SubMiner character entries with structure
   assert.equal((result as Array<{ isNameMatch?: boolean }>)[0]?.isNameMatch, true);
 });
 
+test('requestYomitanScanTokens greedily tokenizes character names before longer generic matches', async () => {
+  let scannerScript = '';
+  const deps = createDeps(async (script) => {
+    if (script.includes('termsFind')) {
+      scannerScript = script;
+      return [];
+    }
+    if (script.includes('optionsGetFull')) {
+      return {
+        profileCurrent: 0,
+        profiles: [
+          {
+            options: {
+              scanning: { length: 40 },
+              dictionaries: [
+                { name: 'JMdict', enabled: true },
+                { name: 'SubMiner Character Dictionary (AniList 130298)', enabled: true },
+              ],
+            },
+          },
+        ],
+      };
+    }
+    return null;
+  });
+
+  await requestYomitanScanTokens(
+    '美姫とヨータ',
+    deps,
+    { error: () => undefined },
+    { includeNameMatchMetadata: true },
+  );
+
+  assert.match(scannerScript, /const greedyNameScanEnabled = true;/);
+
+  const nameEntry = (term: string, reading: string) => ({
+    headwords: [
+      {
+        term,
+        reading,
+        sources: [{ originalText: term, isPrimary: true, matchType: 'exact' }],
+      },
+    ],
+    definitions: [
+      {
+        dictionary: 'SubMiner Character Dictionary (AniList 130298)',
+        dictionaryAlias: 'SubMiner Character Dictionary (AniList 130298)',
+      },
+    ],
+  });
+  const jmdictEntry = (term: string, reading: string, originalText: string) => ({
+    headwords: [
+      {
+        term,
+        reading,
+        sources: [{ originalText, isPrimary: true, matchType: 'exact' }],
+      },
+    ],
+    definitions: [{ dictionary: 'JMdict', dictionaryAlias: 'JMdict' }],
+  });
+
+  const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
+    if (action !== 'termsFind') {
+      throw new Error(`unexpected action: ${action}`);
+    }
+    const text = (params as { text?: string } | undefined)?.text ?? '';
+    if (text.startsWith('美姫')) {
+      return { originalTextLength: 2, dictionaryEntries: [nameEntry('美姫', 'みき')] };
+    }
+    if (text.startsWith('とヨータ')) {
+      // Greedy generic match: とヨー normalizes to とよう (渡洋). Without the
+      // name pre-pass this consumes the ヨ of ヨータ.
+      return {
+        originalTextLength: 3,
+        dictionaryEntries: [jmdictEntry('渡洋', 'とよう', 'とヨー'), jmdictEntry('と', 'と', 'と')],
+      };
+    }
+    if (text.startsWith('ヨータ')) {
+      return { originalTextLength: 3, dictionaryEntries: [nameEntry('ヨータ', 'よーた')] };
+    }
+    if (text === 'と') {
+      return { originalTextLength: 1, dictionaryEntries: [jmdictEntry('と', 'と', 'と')] };
+    }
+    return { originalTextLength: 0, dictionaryEntries: [] };
+  });
+
+  assert.equal(Array.isArray(result), true);
+  assert.deepEqual(
+    (result as Array<Record<string, unknown>>).map(
+      ({ surface, headword, startPos, endPos, isNameMatch }) => ({
+        surface,
+        headword,
+        startPos,
+        endPos,
+        isNameMatch,
+      }),
+    ),
+    [
+      { surface: '美姫', headword: '美姫', startPos: 0, endPos: 2, isNameMatch: true },
+      { surface: 'と', headword: 'と', startPos: 2, endPos: 3, isNameMatch: false },
+      { surface: 'ヨータ', headword: 'ヨータ', startPos: 3, endPos: 6, isNameMatch: true },
+    ],
+  );
+});
+
+test('requestYomitanScanTokens skips greedy name scan without an enabled character dictionary', async () => {
+  let scannerScript = '';
+  const deps = createDeps(async (script) => {
+    if (script.includes('termsFind')) {
+      scannerScript = script;
+      return [];
+    }
+    if (script.includes('optionsGetFull')) {
+      return {
+        profileCurrent: 0,
+        profiles: [
+          {
+            options: {
+              scanning: { length: 40 },
+              dictionaries: [{ name: 'JMdict', enabled: true }],
+            },
+          },
+        ],
+      };
+    }
+    return null;
+  });
+
+  await requestYomitanScanTokens(
+    'アクア',
+    deps,
+    { error: () => undefined },
+    { includeNameMatchMetadata: true },
+  );
+
+  assert.match(scannerScript, /const greedyNameScanEnabled = false;/);
+});
+
+test('requestYomitanScanTokens replaces parseText segmentation where greedy name tokens re-segment', async () => {
+  const deps = createDeps(async (script) => {
+    if (script.includes('optionsGetFull')) {
+      return {
+        profileCurrent: 0,
+        profiles: [
+          {
+            options: {
+              scanning: { length: 40 },
+              dictionaries: [
+                { name: 'JMdict', enabled: true },
+                { name: 'SubMiner Character Dictionary (AniList 130298)', enabled: true },
+              ],
+            },
+          },
+        ],
+      };
+    }
+    if (script.includes('parseText')) {
+      // parseText walks greedily too, so it merges と with ヨー into 渡洋 and
+      // strands the タ.
+      return [
+        {
+          source: 'scanning-parser',
+          index: 0,
+          content: [
+            [
+              {
+                text: '美姫',
+                reading: 'みき',
+                headwords: [[{ term: '美姫' }]],
+              },
+            ],
+            [
+              {
+                text: 'とヨー',
+                reading: 'とよう',
+                headwords: [[{ term: '渡洋' }]],
+              },
+            ],
+            [
+              {
+                text: 'タ',
+                reading: '',
+              },
+            ],
+          ],
+        },
+      ];
+    }
+    return [
+      {
+        surface: '美姫',
+        reading: 'みき',
+        headword: '美姫',
+        headwordReading: 'みき',
+        startPos: 0,
+        endPos: 2,
+        isNameMatch: true,
+      },
+      {
+        surface: 'と',
+        reading: 'と',
+        headword: 'と',
+        headwordReading: 'と',
+        startPos: 2,
+        endPos: 3,
+        isNameMatch: false,
+      },
+      {
+        surface: 'ヨータ',
+        reading: 'ヨータ',
+        headword: 'ヨータ',
+        headwordReading: 'よーた',
+        startPos: 3,
+        endPos: 6,
+        isNameMatch: true,
+      },
+    ];
+  });
+
+  const result = await requestYomitanScanTokens(
+    '美姫とヨータ',
+    deps,
+    { error: () => undefined },
+    { includeNameMatchMetadata: true },
+  );
+
+  assert.deepEqual(result, [
+    {
+      surface: '美姫',
+      reading: 'みき',
+      headword: '美姫',
+      headwordReading: 'みき',
+      startPos: 0,
+      endPos: 2,
+      isNameMatch: true,
+    },
+    {
+      surface: 'と',
+      reading: 'と',
+      headword: 'と',
+      headwordReading: 'と',
+      startPos: 2,
+      endPos: 3,
+      isNameMatch: false,
+    },
+    {
+      surface: 'ヨータ',
+      reading: 'ヨータ',
+      headword: 'ヨータ',
+      headwordReading: 'よーた',
+      startPos: 3,
+      endPos: 6,
+      isNameMatch: true,
+    },
+  ]);
+});
+
 test('requestYomitanScanTokens preserves matched headword word classes', async () => {
   let scannerScript = '';
   const deps = createDeps(async (script) => {

src/core/services/tokenizer/yomitan-parser-runtime.ts

@@ -73,6 +73,7 @@ export interface YomitanAddNoteResult {
 }
 
 const DEFAULT_YOMITAN_SCAN_LENGTH = 40;
+const CHARACTER_DICTIONARY_TITLE_PREFIX = 'SubMiner Character Dictionary';
 const yomitanProfileMetadataByWindow = new WeakMap<BrowserWindow, YomitanProfileMetadata>();
 const yomitanProfileDiagnosticsLoggedByWindow = new WeakSet<BrowserWindow>();
 const yomitanFrequencyCacheByWindow = new WeakMap<
@@ -115,6 +116,12 @@ function scanTokenSpanKey(token: YomitanScanToken): string {
 // scanner tokens onto the parseText segmentation per matching span so one
 // unmatched chunk degrades only itself instead of dropping the whole line's
 // metadata.
+//
+// Exception: character-name tokens. The greedy name scan can re-segment text
+// around a name (e.g. とヨータ → と + ヨータ instead of とヨー + タ), so
+// parseText segmentation cannot be authoritative there. Each name span is
+// expanded until it aligns with token boundaries in both segmentations, then
+// the parse tokens inside are replaced with the scanner tokens.
 function mergeScannerTokensIntoParseTokens(
   parseScanTokens: YomitanScanToken[],
   scannerTokens: YomitanScanToken[],
@@ -123,8 +130,44 @@ function mergeScannerTokensIntoParseTokens(
   for (const token of scannerTokens) {
     scannerTokensBySpan.set(scanTokenSpanKey(token), token);
   }
+  const graftedTokens = parseScanTokens.map(
+    (token) => scannerTokensBySpan.get(scanTokenSpanKey(token)) ?? token,
+  );
 
-  return parseScanTokens.map((token) => scannerTokensBySpan.get(scanTokenSpanKey(token)) ?? token);
+  const nameTokens = scannerTokens.filter((token) => token.isNameMatch === true);
+  if (nameTokens.length === 0) {
+    return graftedTokens;
+  }
+
+  const regions = nameTokens.map((token) => ({ start: token.startPos, end: token.endPos }));
+  const allTokens = [...parseScanTokens, ...scannerTokens];
+  let expanded = true;
+  while (expanded) {
+    expanded = false;
+    for (const region of regions) {
+      for (const token of allTokens) {
+        const overlaps = token.startPos < region.end && token.endPos > region.start;
+        const extendsBeyond = token.startPos < region.start || token.endPos > region.end;
+        if (overlaps && extendsBeyond) {
+          region.start = Math.min(region.start, token.startPos);
+          region.end = Math.max(region.end, token.endPos);
+          expanded = true;
+        }
+      }
+    }
+  }
+
+  const isInsideNameRegion = (token: YomitanScanToken): boolean =>
+    regions.some((region) => token.startPos >= region.start && token.endPos <= region.end);
+
+  const merged = graftedTokens.filter((token) => !isInsideNameRegion(token));
+  for (const token of scannerTokens) {
+    if (isInsideNameRegion(token)) {
+      merged.push(token);
+    }
+  }
+  merged.sort((a, b) => a.startPos - b.startPos || a.endPos - b.endPos);
+  return merged;
 }
 
 function makeTermReadingCacheKey(term: string, reading: string | null): string {
@@ -1110,7 +1153,6 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
         }
         return best;
       }
-      function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
       function normalizeWordClasses(headword) {
           if (!Array.isArray(headword?.wordClasses)) { return undefined; }
           const classes = headword.wordClasses.filter((wordClass) => typeof wordClass === "string" && wordClass.trim().length > 0);
@@ -1152,7 +1194,7 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
           if (!includeNameMatchMetadata || !entry || typeof entry !== 'object') {
             return false;
           }
-          return getDictionaryEntryNames(entry).some((name) => name.startsWith("SubMiner Character Dictionary"));
+          return getDictionaryEntryNames(entry).some((name) => name.startsWith(${JSON.stringify(CHARACTER_DICTIONARY_TITLE_PREFIX)}));
         }
         function parseSubMinerMediaIdFromString(value) {
           const imageMatch = value.match(/\bimg\/m(\d+)-/i);
@@ -1160,7 +1202,7 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
             const parsed = Number.parseInt(imageMatch[1], 10);
             if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
           }
-          const titleMatch = value.match(/SubMiner Character Dictionary[^\d]*(?:AniList\s*)?(\d+)/i);
+          const titleMatch = value.match(/${CHARACTER_DICTIONARY_TITLE_PREFIX}[^\d]*(?:AniList\s*)?(\d+)/i);
           if (titleMatch) {
             const parsed = Number.parseInt(titleMatch[1], 10);
             if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
@@ -1221,6 +1263,26 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
           const mediaIds = getSubMinerMediaIds(entry);
           return mediaIds.size === 0 || mediaIds.has(currentCharacterDictionaryMediaId);
       }
+      function findLongestNameMatch(dictionaryEntries, textWindow) {
+        let best = null;
+        for (const dictionaryEntry of dictionaryEntries || []) {
+          if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; }
+          const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
+          for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
+            const headword = headwords[headwordIndex];
+            for (const src of headword?.sources || []) {
+              if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
+              const originalText = typeof src.originalText === 'string' ? src.originalText : '';
+              if (!originalText || !textWindow.startsWith(originalText)) { continue; }
+              if (best === null || originalText.length > best.sourceLength) {
+                best = { dictionaryEntry, headword, headwordIndex, sourceLength: originalText.length };
+              }
+            }
+          }
+        }
+        return best;
+      }
+      function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
         const currentMediaDictionaryEntries =
           currentCharacterDictionaryMediaId === null
             ? (dictionaryEntries || [])
@@ -1267,6 +1329,7 @@ function buildYomitanScanningScript(
   profileIndex: number,
   scanLength: number,
   includeNameMatchMetadata: boolean,
+  greedyNameScanEnabled: boolean,
   currentCharacterDictionaryMediaId: number | null,
   dictionaryPriorityByName: Record<string, number>,
   dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>,
@@ -1293,6 +1356,7 @@ function buildYomitanScanningScript(
         });
 ${YOMITAN_SCANNING_HELPERS}
       const includeNameMatchMetadata = ${includeNameMatchMetadata ? 'true' : 'false'};
+      const greedyNameScanEnabled = ${greedyNameScanEnabled ? 'true' : 'false'};
       const currentCharacterDictionaryMediaId = ${
         currentCharacterDictionaryMediaId !== null
           ? String(currentCharacterDictionaryMediaId)
@@ -1303,26 +1367,17 @@ ${YOMITAN_SCANNING_HELPERS}
       const text = ${JSON.stringify(text)};
       const details = {matchType: "exact", deinflect: true};
       const tokens = [];
-      async function findTokenAt(position, windowLength) {
+      const termsFindCache = new Map();
-        const codePoint = text.codePointAt(position);
+      async function termsFindAt(position, windowLength) {
-        const character = String.fromCodePoint(codePoint);
+        const cacheKey = position + ":" + windowLength;
+        const cached = termsFindCache.get(cacheKey);
+        if (cached) { return cached; }
         const substring = text.substring(position, position + windowLength);
         const result = await invoke("termsFind", { text: substring, details, optionsContext: { index: ${profileIndex} } });
-        const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
+        termsFindCache.set(cacheKey, result);
-        const originalTextLength = typeof result?.originalTextLength === "number" ? result.originalTextLength : 0;
+        return result;
-        if (dictionaryEntries.length === 0 || originalTextLength <= 0 || (originalTextLength === character.length && !isCodePointJapanese(codePoint))) {
-          return { token: null, matchedLength: 0 };
-        }
-        const source = substring.substring(0, originalTextLength);
-        const preferredHeadword = getPreferredHeadword(
-          dictionaryEntries,
-          source,
-          dictionaryPriorityByName,
-          dictionaryFrequencyModeByName
-        );
-        if (!preferredHeadword || typeof preferredHeadword.term !== "string") {
-          return { token: null, matchedLength: originalTextLength };
       }
+      function buildScanToken(position, source, preferredHeadword) {
         const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
         const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
         const tokenPayload = {
@@ -1331,7 +1386,7 @@ ${YOMITAN_SCANNING_HELPERS}
           headword: preferredHeadword.term,
           headwordReading: reading || undefined,
           startPos: position,
-          endPos: position + originalTextLength,
+          endPos: position + source.length,
           isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
           frequencyRank:
             typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
@@ -1341,17 +1396,85 @@ ${YOMITAN_SCANNING_HELPERS}
         if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
           tokenPayload.wordClasses = preferredHeadword.wordClasses;
         }
-        return { token: tokenPayload, matchedLength: originalTextLength };
+        return tokenPayload;
+      }
+      async function findTokenAt(position, windowLength) {
+        const codePoint = text.codePointAt(position);
+        const character = String.fromCodePoint(codePoint);
+        const result = await termsFindAt(position, windowLength);
+        const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
+        const originalTextLength = typeof result?.originalTextLength === "number" ? result.originalTextLength : 0;
+        if (dictionaryEntries.length === 0 || originalTextLength <= 0 || (originalTextLength === character.length && !isCodePointJapanese(codePoint))) {
+          return { token: null, matchedLength: 0 };
+        }
+        const source = text.substring(position, position + originalTextLength);
+        const preferredHeadword = getPreferredHeadword(
+          dictionaryEntries,
+          source,
+          dictionaryPriorityByName,
+          dictionaryFrequencyModeByName
+        );
+        if (!preferredHeadword || typeof preferredHeadword.term !== "string") {
+          return { token: null, matchedLength: originalTextLength };
+        }
+        return { token: buildScanToken(position, source, preferredHeadword), matchedLength: originalTextLength };
+      }
+      // Greedy name pre-pass: character-name matches claim their spans before
+      // the left-to-right walk, so a longer generic match starting earlier
+      // (e.g. とヨー → 渡洋) cannot swallow the start of a name (ヨータ).
+      const nameTokens = [];
+      if (greedyNameScanEnabled) {
+        let namePos = 0;
+        while (namePos < text.length) {
+          const codePoint = text.codePointAt(namePos);
+          if (!isCodePointJapanese(codePoint)) {
+            namePos += String.fromCodePoint(codePoint).length;
+            continue;
+          }
+          const result = await termsFindAt(namePos, ${scanLength});
+          const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
+          const nameMatch = findLongestNameMatch(dictionaryEntries, text.substring(namePos, namePos + ${scanLength}));
+          if (!nameMatch) {
+            namePos += String.fromCodePoint(codePoint).length;
+            continue;
+          }
+          const source = text.substring(namePos, namePos + nameMatch.sourceLength);
+          nameTokens.push(buildScanToken(namePos, source, {
+            term: nameMatch.headword.term,
+            reading: nameMatch.headword.reading,
+            wordClasses: normalizeWordClasses(nameMatch.headword),
+            isNameMatch: true,
+            frequencyRank: getBestFrequencyRank(
+              nameMatch.dictionaryEntry,
+              nameMatch.headwordIndex,
+              dictionaryPriorityByName,
+              dictionaryFrequencyModeByName
+            )
+          }));
+          namePos += nameMatch.sourceLength;
+        }
       }
       let i = 0;
+      let nameIndex = 0;
       while (i < text.length) {
-        let attempt = await findTokenAt(i, ${scanLength});
+        while (nameIndex < nameTokens.length && nameTokens[nameIndex].startPos < i) { nameIndex += 1; }
+        const nextNameToken = nameIndex < nameTokens.length ? nameTokens[nameIndex] : null;
+        if (nextNameToken && nextNameToken.startPos === i) {
+          tokens.push(nextNameToken);
+          i = nextNameToken.endPos;
+          nameIndex += 1;
+          continue;
+        }
+        // Cap the window at the next reserved name span so a generic match
+        // cannot consume into it.
+        const windowLength = nextNameToken ? Math.min(${scanLength}, nextNameToken.startPos - i) : ${scanLength};
+        let attempt = await findTokenAt(i, windowLength);
         // Yomitan text normalization can consume characters (whitespace,
         // punctuation) beyond the matched term, leaving no headword whose
         // source equals the consumed text. Retry with shorter windows so a
         // valid prefix term (e.g. a character name before a paren) still
         // tokenizes instead of the position being skipped.
-        let retryLength = Math.min(attempt.matchedLength, ${scanLength}) - 1;
+        let retryLength = Math.min(attempt.matchedLength, windowLength) - 1;
         while (!attempt.token && retryLength >= 1) {
           const retry = await findTokenAt(i, retryLength);
           if (retry.token) {
@@ -1500,6 +1623,12 @@ export async function requestYomitanScanTokens(
   const metadata = await requestYomitanProfileMetadata(parserWindow, logger);
   const profileIndex = metadata?.profileIndex ?? 0;
   const scanLength = metadata?.scanLength ?? DEFAULT_YOMITAN_SCAN_LENGTH;
+  const includeNameMatchMetadata = options?.includeNameMatchMetadata === true;
+  const greedyNameScanEnabled =
+    includeNameMatchMetadata &&
+    (metadata?.dictionaries ?? []).some((name) =>
+      name.startsWith(CHARACTER_DICTIONARY_TITLE_PREFIX),
+    );
 
   try {
     const rawResult = await parserWindow.webContents.executeJavaScript(
@@ -1507,7 +1636,8 @@ export async function requestYomitanScanTokens(
         text,
         profileIndex,
         scanLength,
-        options?.includeNameMatchMetadata === true,
+        includeNameMatchMetadata,
+        greedyNameScanEnabled,
         typeof options?.currentCharacterDictionaryMediaId === 'number' &&
           Number.isFinite(options.currentCharacterDictionaryMediaId) &&
           options.currentCharacterDictionaryMediaId > 0