perf(tokenizer): single-pass Yomitan scan with cross-line caching and prefetch fixes (#185)

src/core/services/subtitle-processing-controller.test.ts

@@ -539,3 +539,125 @@ test('default cache limit covers a full-length title without evicting', () => {
   assert.equal(controller.hasCachedSubtitle('line-0'), true);
   assert.equal(controller.hasCachedSubtitle('line-1999'), true);
 });
+
+test('onSubtitleChange reports whether processing was scheduled', async () => {
+  const emitted: SubtitleData[] = [];
+  const controller = createSubtitleProcessingController({
+    tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
+    emitSubtitle: (payload) => emitted.push(payload),
+  });
+
+  // New text schedules work, so an emit (and anything gated on it) will follow.
+  assert.equal(controller.onSubtitleChange('字幕'), true);
+  await flushMicrotasks();
+
+  // A repeat emits nothing, so callers must not wait on an emit that is never
+  // coming (subtitle prefetching would stay paused for the rest of the cue).
+  const emittedCount = emitted.length;
+  assert.equal(controller.onSubtitleChange('字幕'), false);
+  await flushMicrotasks();
+  assert.equal(emitted.length, emittedCount);
+});
+
+test('refreshCurrentSubtitle reports the empty-text emit that an in-flight run will deliver', async () => {
+  const emitted: SubtitleData[] = [];
+  let resolveFirst: ((value: SubtitleData | null) => void) | undefined;
+  const controller = createSubtitleProcessingController({
+    tokenizeSubtitle: async (text) => {
+      if (text === '字幕') {
+        return await new Promise<SubtitleData | null>((resolve) => {
+          resolveFirst = resolve;
+        });
+      }
+      return { text, tokens: [] };
+    },
+    emitSubtitle: (payload) => emitted.push(payload),
+  });
+
+  controller.onSubtitleChange('字幕');
+  await flushMicrotasks();
+
+  // Clearing the subtitle while tokenization is in flight: the running loop
+  // picks the empty text up and emits it, so callers gated on that emit (the
+  // prefetch pause) must be told one is coming.
+  assert.equal(controller.refreshCurrentSubtitle(''), true);
+
+  resolveFirst?.({ text: '字幕', tokens: [] });
+  await flushMicrotasks();
+  await flushMicrotasks();
+  // '字幕' is the provisional plain emit the in-flight run already made before
+  // the refresh; '' is the emit the refresh promised.
+  assert.deepEqual(
+    emitted.map((payload) => payload.text),
+    ['字幕', ''],
+  );
+});
+
+test('onProcessingSettled fires once after the queue drains, including runs that emit nothing', async () => {
+  const events: string[] = [];
+  let resolveFirst: ((value: SubtitleData | null) => void) | undefined;
+  let tokenizationFails = false;
+  const controller = createSubtitleProcessingController({
+    tokenizeSubtitle: async (text) => {
+      if (tokenizationFails) {
+        return null;
+      }
+      if (text === '一行目') {
+        return await new Promise<SubtitleData | null>((resolve) => {
+          resolveFirst = resolve;
+        });
+      }
+      return { text, tokens: [] };
+    },
+    emitSubtitle: (payload) => events.push(`emit:${payload.text}`),
+    onProcessingSettled: () => events.push('settled'),
+  });
+
+  controller.onSubtitleChange('一行目');
+  await flushMicrotasks();
+  // A second line arrives before the first finishes: the controller still has
+  // work, so it must not report itself settled between the two.
+  controller.onSubtitleChange('二行目');
+  resolveFirst?.({ text: '一行目', tokens: [] });
+  await flushMicrotasks();
+  await flushMicrotasks();
+
+  assert.deepEqual(events, ['emit:一行目', 'emit:二行目', 'emit:二行目', 'settled']);
+
+  // Tokenization failure on a line already shown plain: nothing is emitted, and
+  // the settle signal is the only way a caller learns the work is over.
+  events.length = 0;
+  tokenizationFails = true;
+  controller.invalidateTokenizationCache();
+  assert.equal(controller.refreshCurrentSubtitle('二行目'), true);
+  await flushMicrotasks();
+  await flushMicrotasks();
+  assert.deepEqual(events, ['settled']);
+});
+
+test('notePlainSubtitleEmitted suppresses the controller repeat of a payload already shown', async () => {
+  const emitted: SubtitleData[] = [];
+  const controller = createSubtitleProcessingController({
+    tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
+    emitSubtitle: (payload) => emitted.push(payload),
+  });
+
+  // Autoplay priming paints the plain line itself, then asks for tokenization.
+  controller.notePlainSubtitleEmitted('字幕');
+  controller.refreshCurrentSubtitle('字幕');
+  await flushMicrotasks();
+
+  assert.deepEqual(emitted, [{ text: '字幕', tokens: [] }]);
+});
+
+test('refreshCurrentSubtitle reports no emit for empty text when nothing is running', async () => {
+  const emitted: SubtitleData[] = [];
+  const controller = createSubtitleProcessingController({
+    tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
+    emitSubtitle: (payload) => emitted.push(payload),
+  });
+
+  assert.equal(controller.refreshCurrentSubtitle(''), false);
+  await flushMicrotasks();
+  assert.deepEqual(emitted, []);
+});

src/core/services/subtitle-processing-controller.ts

@@ -3,6 +3,14 @@ import type { SubtitleData } from '../../types';
 export interface SubtitleProcessingControllerDeps {
   tokenizeSubtitle: (text: string) => Promise<SubtitleData | null>;
   emitSubtitle: (payload: SubtitleData) => void;
+  /**
+   * Fires when the controller runs out of work: every scheduled line has been
+   * processed, whether it ended in an emit, a suppressed duplicate, or a
+   * tokenizer failure. Callers that hold a resource for the duration of
+   * processing (prefetch pausing) release it here rather than on an emit,
+   * which is not guaranteed to happen.
+   */
+  onProcessingSettled?: () => void;
   logDebug?: (message: string) => void;
   now?: () => number;
   cacheLimit?: number;
@@ -17,8 +25,22 @@ export interface SubtitleProcessingControllerDeps {
 export const DEFAULT_SUBTITLE_TOKENIZATION_CACHE_LIMIT = 2500;
 
 export interface SubtitleProcessingController {
-  onSubtitleChange: (text: string) => void;
-  refreshCurrentSubtitle: (textOverride?: string) => void;
+  /**
+   * Returns whether processing is now scheduled or already in flight for this
+   * event. A false return means the controller is idle and will do nothing, so
+   * onProcessingSettled will not fire; callers that pause work for the duration
+   * of processing (such as subtitle prefetching) must release it themselves.
+   */
+  onSubtitleChange: (text: string) => boolean;
+  /** Same contract as onSubtitleChange: whether processing is pending. */
+  refreshCurrentSubtitle: (textOverride?: string) => boolean;
+  /**
+   * Records that this exact text has already been shown plain by someone else
+   * (autoplay priming paints its first frame before scheduling tokenization),
+   * so the controller does not repeat that payload on its way to the tokenized
+   * one.
+   */
+  notePlainSubtitleEmitted: (text: string) => void;
   invalidateTokenizationCache: () => void;
   preCacheTokenization: (text: string, data: SubtitleData) => void;
   consumeCachedSubtitle: (text: string) => SubtitleData | null;
@@ -164,14 +186,20 @@ export function createSubtitleProcessingController(
           (latestText.trim() && cacheGeneration !== lastEmittedGeneration)
         ) {
           processLatest();
+          return;
         }
+        // Nothing left to do: signal completion even when this run emitted
+        // nothing (suppressed duplicate, tokenizer failure), or callers waiting
+        // on the controller would wait forever.
+        deps.onProcessingSettled?.();
       });
   };
 
   return {
     onSubtitleChange: (text: string) => {
       if (text === latestText) {
-        return;
+        // A run already in flight for this text will still emit for it.
+        return processing;
       }
       latestText = text;
       if (
@@ -183,21 +211,28 @@ export function createSubtitleProcessingController(
         lastPlainEmittedText = text;
       }
       processLatest();
+      return true;
     },
     refreshCurrentSubtitle: (textOverride?: string) => {
       if (typeof textOverride === 'string') {
         latestText = textOverride;
       }
       if (!latestText.trim()) {
-        return;
+        // A run in flight will pick this up and emit the empty subtitle, so
+        // the caller is still waiting on an emit.
+        return processing;
       }
-      if (
-        processing ||
-        (latestText === lastEmittedText && cacheGeneration === lastEmittedGeneration)
-      ) {
-        return;
+      if (processing) {
+        return true;
+      }
+      if (latestText === lastEmittedText && cacheGeneration === lastEmittedGeneration) {
+        return false;
       }
       processLatest();
+      return true;
+    },
+    notePlainSubtitleEmitted: (text: string) => {
+      lastPlainEmittedText = text;
     },
     invalidateTokenizationCache: () => {
       tokenizationCache.clear();

src/core/services/tokenizer.test.ts

@@ -2934,44 +2934,12 @@ test('tokenizeSubtitle preserves Yomitan compound token when MeCab components ar
                 return [];
               }
 
-              if (script.includes('parseText')) {
-                return [
-                  {
-                    source: 'scanning-parser',
-                    index: 0,
-                    content: [
-                      [
-                        {
-                          text: '取り組んで',
-                          reading: 'とりくんで',
-                          headwords: [[{ term: '取り組む' }]],
-                        },
-                      ],
-                      [
-                        {
-                          text: 'もらいます',
-                          reading: 'もらいます',
-                          headwords: [[{ term: 'もらう' }]],
-                        },
-                      ],
-                    ],
-                  },
-                ];
-              }
-
               return [
                 {
-                  surface: '取り',
-                  reading: 'とり',
-                  headword: '取る',
+                  surface: '取り組んで',
+                  reading: 'とりくんで',
+                  headword: '取り組む',
                   startPos: 0,
-                  endPos: 2,
-                },
-                {
-                  surface: '組んで',
-                  reading: 'くんで',
-                  headword: '組む',
-                  startPos: 2,
                   endPos: 5,
                 },
                 {

src/core/services/tokenizer.ts

@@ -70,6 +70,7 @@ export interface TokenizerServiceDeps {
   getNameMatchImagesEnabled?: () => boolean;
   getCharacterNameImage?: (term: string) => CharacterNameImage | null;
   getCurrentCharacterDictionaryMediaId?: () => number | null;
+  getCharacterNameCandidates?: () => { key: string; forms: string[] } | null;
   getFrequencyDictionaryEnabled?: () => boolean;
   getFrequencyDictionaryMatchMode?: () => FrequencyDictionaryMatchMode;
   getFrequencyRank?: FrequencyDictionaryLookup;
@@ -106,6 +107,7 @@ export interface TokenizerDepsRuntimeOptions {
   getNameMatchImagesEnabled?: () => boolean;
   getCharacterNameImage?: (term: string) => CharacterNameImage | null;
   getCurrentCharacterDictionaryMediaId?: () => number | null;
+  getCharacterNameCandidates?: () => { key: string; forms: string[] } | null;
   getFrequencyDictionaryEnabled?: () => boolean;
   getFrequencyDictionaryMatchMode?: () => FrequencyDictionaryMatchMode;
   getFrequencyRank?: FrequencyDictionaryLookup;
@@ -266,6 +268,7 @@ export function createTokenizerDepsRuntime(
     getNameMatchImagesEnabled: options.getNameMatchImagesEnabled,
     getCharacterNameImage: options.getCharacterNameImage,
     getCurrentCharacterDictionaryMediaId: options.getCurrentCharacterDictionaryMediaId,
+    getCharacterNameCandidates: options.getCharacterNameCandidates,
     getFrequencyDictionaryEnabled: options.getFrequencyDictionaryEnabled,
     getFrequencyDictionaryMatchMode: options.getFrequencyDictionaryMatchMode ?? (() => 'headword'),
     getFrequencyRank: options.getFrequencyRank,
@@ -716,15 +719,30 @@ function getAnnotationOptions(deps: TokenizerServiceDeps): TokenizerAnnotationOp
   };
 }
 
+// Per-line stage durations for the pipeline debug log; every field is filled in
+// by the stage that awaits the corresponding work.
+interface TokenizationStageTimings {
+  scanMs?: number;
+  mecabMs?: number;
+  frequencyMs?: number;
+  annotateMs?: number;
+}
+
 async function parseWithYomitanInternalParser(
   text: string,
   deps: TokenizerServiceDeps,
   options: TokenizerAnnotationOptions,
+  stageTimings?: TokenizationStageTimings,
 ): Promise<MergedToken[] | null> {
+  const scanStartedAtMs = Date.now();
   const selectedTokens = await requestYomitanScanTokens(text, deps, logger, {
     includeNameMatchMetadata: options.nameMatchEnabled,
     currentCharacterDictionaryMediaId: deps.getCurrentCharacterDictionaryMediaId?.() ?? null,
+    nameCandidates: deps.getCharacterNameCandidates?.() ?? null,
   });
+  if (stageTimings) {
+    stageTimings.scanMs = Date.now() - scanStartedAtMs;
+  }
   if (!selectedTokens || selectedTokens.length === 0) {
     return null;
   }
@@ -757,6 +775,7 @@ async function parseWithYomitanInternalParser(
 
   const frequencyRankPromise: Promise<YomitanFrequencyIndex> = options.frequencyEnabled
     ? (async () => {
+        const frequencyStartedAtMs = Date.now();
         const frequencyMatchMode = options.frequencyMatchMode;
         const termReadingList = buildYomitanFrequencyTermReadingList(
           normalizedSelectedTokens,
@@ -767,12 +786,17 @@ async function parseWithYomitanInternalParser(
           deps,
           logger,
         );
-        return buildYomitanFrequencyIndex(yomitanFrequencies);
+        const frequencyIndex = buildYomitanFrequencyIndex(yomitanFrequencies);
+        if (stageTimings) {
+          stageTimings.frequencyMs = Date.now() - frequencyStartedAtMs;
+        }
+        return frequencyIndex;
       })()
     : Promise.resolve({ byPair: new Map(), byTerm: new Map() });
 
   const mecabEnrichmentPromise: Promise<MergedToken[]> = needsMecabPosEnrichment(options)
     ? (async () => {
+        const mecabStartedAtMs = Date.now();
         try {
           const mecabTokens = await deps.tokenizeWithMecab(text);
           const enrichTokensWithMecab = deps.enrichTokensWithMecab ?? enrichTokensWithMecabAsync;
@@ -786,6 +810,10 @@ async function parseWithYomitanInternalParser(
             `textLength=${text.length}`,
           );
           return normalizedSelectedTokens;
+        } finally {
+          if (stageTimings) {
+            stageTimings.mecabMs = Date.now() - mecabStartedAtMs;
+          }
         }
       })()
     : Promise.resolve(normalizedSelectedTokens);
@@ -876,15 +904,35 @@ export async function tokenizeSubtitle(
   const annotationOptions = getAnnotationOptions(deps);
   annotationOptions.sourceText = tokenizeText;
 
-  const yomitanTokens = await parseWithYomitanInternalParser(tokenizeText, deps, annotationOptions);
+  const stageTimings: TokenizationStageTimings = {};
+  const startedAtMs = Date.now();
+  const logStageTimings = (tokenCount: number): void => {
+    logger.debug(
+      `Subtitle tokenization stages; textLength=${tokenizeText.length}, tokenCount=${tokenCount}, ` +
+        `scanMs=${stageTimings.scanMs ?? '-'}, mecabMs=${stageTimings.mecabMs ?? '-'}, ` +
+        `frequencyMs=${stageTimings.frequencyMs ?? '-'}, annotateMs=${stageTimings.annotateMs ?? '-'}, ` +
+        `totalMs=${Date.now() - startedAtMs}`,
+    );
+  };
+
+  const yomitanTokens = await parseWithYomitanInternalParser(
+    tokenizeText,
+    deps,
+    annotationOptions,
+    stageTimings,
+  );
   if (yomitanTokens && yomitanTokens.length > 0) {
+    const annotateStartedAtMs = Date.now();
     const annotatedTokens = await applyAnnotationStage(yomitanTokens, deps, annotationOptions);
+    stageTimings.annotateMs = Date.now() - annotateStartedAtMs;
     const renderedTokens = applyCharacterNameImages(annotatedTokens, deps, annotationOptions);
+    logStageTimings(renderedTokens.length);
     return {
       text: displayText,
       tokens: renderedTokens.length > 0 ? renderedTokens : null,
     };
   }
 
+  logStageTimings(0);
   return { text: displayText, tokens: null };
 }

src/core/services/tokenizer/character-dictionary-title.ts

@@ -0,0 +1,4 @@
+// Title prefix of the dictionaries SubMiner generates per media. Lives on its
+// own because both the main process and the injected scan runtime match on it,
+// and the injected fragments interpolate it into their own source.
+export const CHARACTER_DICTIONARY_TITLE_PREFIX = 'SubMiner Character Dictionary';

src/core/services/tokenizer/golden-corpus-harness.ts

@@ -366,8 +366,11 @@ export function createReplayMessageStore(messages: GoldenRecordedMessage[]): Rep
   };
 }
 
-async function runInjectedScriptInVm(script: string, store: ReplayMessageStore): Promise<unknown> {
-  return await vm.runInNewContext(script, {
+// One persistent context per fixture, matching the real parser window: the
+// scan runtime installs itself once into globalThis and later per-line call
+// scripts reuse it.
+function createInjectedScriptVm(store: ReplayMessageStore): (script: string) => Promise<unknown> {
+  const context = vm.createContext({
     chrome: {
       runtime: {
         lastError: null,
@@ -393,6 +396,7 @@ async function runInjectedScriptInVm(script: string, store: ReplayMessageStore):
     Set,
     String,
   });
+  return async (script: string) => await vm.runInContext(script, context);
 }
 
 export function createReplayTokenizerDeps(fixture: GoldenFixture): TokenizerServiceDeps {
@@ -400,13 +404,14 @@ export function createReplayTokenizerDeps(fixture: GoldenFixture): TokenizerServ
   const scriptResults = new Map(
     fixture.recording.scripts.map((entry) => [entry.sha256, entry] as const),
   );
+  const runInjectedScriptInVm = createInjectedScriptVm(store);
 
   const parserWindow = {
     isDestroyed: () => false,
     webContents: {
       executeJavaScript: async (script: string) => {
         try {
-          return await runInjectedScriptInVm(script, store);
+          return await runInjectedScriptInVm(script);
         } catch (vmError) {
           const recorded = scriptResults.get(hashInjectedScript(script));
           if (recorded) {

src/core/services/tokenizer/token-classification.test.ts

@@ -8,6 +8,7 @@ import {
   isKanaChar,
   isKanaOnlyText,
   isTokenPos2Excluded,
+  normalizeKana,
 } from './token-classification';
 
 const POS1_EXCLUSIONS = new Set(['助詞']);
@@ -29,6 +30,26 @@ function makeNoun(surface: string): MergedToken {
   };
 }
 
+test('kana normalization folds halfwidth kana, composing the voiced pairs', () => {
+  // ｶ + ﾞ is two code points for one character: without composing them, a
+  // halfwidth word counts as longer than the reading that spells it, which
+  // disqualifies the reading from known-word matching.
+  assert.equal(normalizeKana('ｶﾞｸ'), normalizeKana('ガク'));
+  assert.equal(normalizeKana('ﾊﾟﾝ'), normalizeKana('パン'));
+  assert.equal(normalizeKana('ﾐﾅﾄ'), 'みなと');
+  assert.ok(isKanaOnlyText('ｶﾞｸ'));
+});
+
+test('kana normalization leaves characters other than halfwidth kana alone', () => {
+  // The composition is scoped to the halfwidth runs: applied to the whole
+  // string, NFKC would also rewrite these into something the dictionary, the
+  // known-word list, and the frequency data were never keyed on.
+  assert.equal(normalizeKana('①ｶﾞ'), '①が');
+  assert.equal(normalizeKana('Ａｶﾞ'), 'Ａが');
+  assert.equal(normalizeKana('㍑ｶﾞ'), '㍑が');
+  assert.equal(normalizeKana('fiｶﾞ'), 'fiが');
+});
+
 test('kana classification excludes the katakana-hiragana double hyphen', () => {
   assert.equal(isKanaChar('゠'), false);
   assert.equal(isKanaOnlyText('゠'), false);

src/core/services/tokenizer/token-classification.ts

@@ -4,8 +4,20 @@ const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
 const KATAKANA_CODEPOINT_START = 0x30a1;
 const KATAKANA_CODEPOINT_END = 0x30f6;
 
+// No `u` flag: the range is entirely BMP so it changes nothing here, and
+// Bun's unicode-mode matcher mis-handles this class next to certain ligatures.
+const HALFWIDTH_KANA_RUN = /[\uff66-\uff9f]+/g;
+
+// NFKC over the halfwidth kana only, never the whole string: it composes the
+// voiced pairs (ｶ + ﾞ) into single characters so ｶﾞｸ compares equal to ガク
+// instead of counting one character longer than the word it spells, but run
+// over everything it would also rewrite unrelated text (① → 1, ㍑ → リットル).
+function composeHalfwidthKana(text: string): string {
+  return text.replace(HALFWIDTH_KANA_RUN, (run) => run.normalize('NFKC'));
+}
+
 export function normalizeKana(text: string): string {
-  const raw = text.trim();
+  const raw = composeHalfwidthKana(text).trim();
   if (!raw) {
     return '';
   }

src/core/services/tokenizer/yomitan-dictionary-classification-script.ts

@@ -0,0 +1,150 @@
+// Dictionary classification for the injected scan runtime: which dictionaries
+// an entry came from, and whether it is a SubMiner character entry for the
+// media being watched. Both walk nested entry data, so both are memoized on the
+// entry object by the runtime that hosts them.
+
+import { CHARACTER_DICTIONARY_TITLE_PREFIX } from './character-dictionary-title';
+
+// The prefix is interpolated into generated regex source, so metacharacters in
+// it would change what the pattern matches (or fail to compile).
+const ESCAPED_TITLE_PREFIX = CHARACTER_DICTIONARY_TITLE_PREFIX.replace(
+  /[.*+?^${}()|[\]\\]/g,
+  '\\$&',
+);
+const TITLE_MEDIA_ID_PATTERN = ESCAPED_TITLE_PREFIX + String.raw`[^\d]*(?:AniList\s*)?(\d+)`;
+
+export const YOMITAN_DICTIONARY_CLASSIFICATION_HELPERS = String.raw`
+      function normalizeWordClasses(headword) {
+          if (!Array.isArray(headword?.wordClasses)) { return undefined; }
+          const classes = headword.wordClasses.filter((wordClass) => typeof wordClass === "string" && wordClass.trim().length > 0);
+          return classes.length > 0 ? classes : undefined;
+        }
+        function appendDictionaryNames(target, value) {
+          if (!value || typeof value !== 'object') {
+            return;
+          }
+          const candidates = [
+            value.dictionary,
+            value.dictionaryName,
+            value.name,
+            value.title,
+            value.dictionaryTitle,
+            value.dictionaryAlias
+          ];
+          for (const candidate of candidates) {
+            if (typeof candidate === 'string' && candidate.trim().length > 0) {
+              target.push(candidate.trim());
+            }
+          }
+        }
+        // Memoized on the entry object: termsFind results are cached across
+        // lines, so the same entries come back for every repeated lookup, and
+        // each one is classified several times per scan (name pre-pass,
+        // headword preference, every retry window).
+        function getDictionaryEntryNames(entry) {
+          if (!entry || typeof entry !== 'object') { return []; }
+          const cached = dictionaryEntryNamesCache.get(entry);
+          if (cached !== undefined) { return cached; }
+          const names = [];
+          appendDictionaryNames(names, entry);
+          for (const definition of entry?.definitions || []) {
+            appendDictionaryNames(names, definition);
+          }
+          for (const frequency of entry?.frequencies || []) {
+            appendDictionaryNames(names, frequency);
+          }
+          for (const pronunciation of entry?.pronunciations || []) {
+            appendDictionaryNames(names, pronunciation);
+          }
+          dictionaryEntryNamesCache.set(entry, names);
+          return names;
+        }
+        // Cached per scan rather than per runtime: the answer depends on
+        // includeNameMatchMetadata, which is a per-call parameter.
+        const nameDictionaryEntryCache = new WeakMap();
+        function isNameDictionaryEntry(entry) {
+          if (!includeNameMatchMetadata || !entry || typeof entry !== 'object') {
+            return false;
+          }
+          const cached = nameDictionaryEntryCache.get(entry);
+          if (cached !== undefined) { return cached; }
+          const isName = getDictionaryEntryNames(entry).some((name) => name.startsWith(${JSON.stringify(CHARACTER_DICTIONARY_TITLE_PREFIX)}));
+          nameDictionaryEntryCache.set(entry, isName);
+          return isName;
+        }
+        const TITLE_MEDIA_ID_REGEX = new RegExp(${JSON.stringify(TITLE_MEDIA_ID_PATTERN)}, 'i');
+        function parseSubMinerMediaIdFromString(value) {
+          const imageMatch = value.match(/\bimg\/m(\d+)-/i);
+          if (imageMatch) {
+            const parsed = Number.parseInt(imageMatch[1], 10);
+            if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
+          }
+          const titleMatch = value.match(TITLE_MEDIA_ID_REGEX);
+          if (titleMatch) {
+            const parsed = Number.parseInt(titleMatch[1], 10);
+            if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
+          }
+          return null;
+        }
+        function parseSubMinerMediaIdCandidate(value) {
+          if (typeof value === 'number' && Number.isSafeInteger(value) && value > 0) {
+            return value;
+          }
+          if (typeof value === 'string' && /^\d+$/.test(value.trim())) {
+            const parsed = Number.parseInt(value.trim(), 10);
+            if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
+          }
+          return null;
+        }
+        function collectSubMinerMediaIds(value, target) {
+          if (typeof value === 'string') {
+            const parsed = parseSubMinerMediaIdFromString(value);
+            if (parsed !== null) { target.add(parsed); }
+            return;
+          }
+          if (!value || typeof value !== 'object') {
+            return;
+          }
+          if (Array.isArray(value)) {
+            for (const item of value) { collectSubMinerMediaIds(item, target); }
+            return;
+          }
+          const mediaIdCandidates = [
+            value.subminerMediaId,
+            value.subMinerMediaId,
+            value.characterDictionaryMediaId,
+            value.data?.subminerMediaId,
+            value.data?.subMinerMediaId,
+            value.data?.characterDictionaryMediaId
+          ];
+          for (const candidate of mediaIdCandidates) {
+            const parsed = parseSubMinerMediaIdCandidate(candidate);
+            if (parsed !== null) { target.add(parsed); }
+          }
+          for (const child of Object.values(value)) {
+            collectSubMinerMediaIds(child, target);
+          }
+        }
+        // Walking an entry collects media ids from every nested value, so this
+        // is the most expensive classification step; memoized on the entry for
+        // the same reason as the dictionary names above.
+        function getSubMinerMediaIds(entry) {
+          if (!entry || typeof entry !== 'object') { return EMPTY_MEDIA_ID_SET; }
+          const cached = subMinerMediaIdsCache.get(entry);
+          if (cached !== undefined) { return cached; }
+          const mediaIds = new Set();
+          collectSubMinerMediaIds(entry, mediaIds);
+          subMinerMediaIdsCache.set(entry, mediaIds);
+          return mediaIds;
+        }
+        function isCurrentMediaNameDictionaryEntry(entry) {
+          if (!isNameDictionaryEntry(entry)) {
+            return false;
+          }
+          if (currentCharacterDictionaryMediaId === null) {
+            return true;
+          }
+          const mediaIds = getSubMinerMediaIds(entry);
+          return mediaIds.size === 0 || mediaIds.has(currentCharacterDictionaryMediaId);
+      }
+`;

src/core/services/tokenizer/yomitan-frequency-script.ts

@@ -0,0 +1,135 @@
+// Frequency-rank resolution for the injected scan runtime: reads the many
+// shapes a Yomitan frequency entry can take and picks the best rank for a
+// headword, honouring per-dictionary priority and occurrence-vs-rank mode.
+
+export const YOMITAN_FREQUENCY_HELPERS = String.raw`
+      function parsePositiveFrequencyNumber(value) {
+        if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
+          return Math.max(1, Math.floor(value));
+        }
+        if (typeof value === 'string') {
+          const numericMatch = value.trim().match(/[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?/)?.[0];
+          if (!numericMatch) { return null; }
+          const parsed = Number.parseFloat(numericMatch);
+          if (!Number.isFinite(parsed) || parsed <= 0) { return null; }
+          return Math.max(1, Math.floor(parsed));
+        }
+        if (Array.isArray(value)) {
+          for (const item of value) {
+            const parsed = parsePositiveFrequencyNumber(item);
+            if (parsed !== null) { return parsed; }
+          }
+        }
+        return null;
+      }
+      function parseDisplayFrequencyNumber(value) {
+        if (typeof value === 'string') {
+          const leadingDigits = value.trim().match(/^\d+/)?.[0];
+          if (!leadingDigits) { return null; }
+          const parsed = Number.parseInt(leadingDigits, 10);
+          return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
+        }
+        return parsePositiveFrequencyNumber(value);
+      }
+      function getFrequencyDictionaryName(frequency) {
+        const candidates = [
+          frequency?.dictionary,
+          frequency?.dictionaryName,
+          frequency?.name,
+          frequency?.title,
+          frequency?.dictionaryTitle,
+          frequency?.dictionaryAlias
+        ];
+        for (const candidate of candidates) {
+          if (typeof candidate === 'string' && candidate.trim().length > 0) {
+            return candidate.trim();
+          }
+        }
+        return null;
+      }
+      function getBestFrequencyRank(dictionaryEntry, headwordIndex, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
+        let best = null;
+        const headwordCount = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords.length : 0;
+        for (const frequency of dictionaryEntry?.frequencies || []) {
+          if (!frequency || typeof frequency !== 'object') { continue; }
+          const frequencyHeadwordIndex = frequency.headwordIndex;
+          if (typeof frequencyHeadwordIndex === 'number') {
+            if (frequencyHeadwordIndex !== headwordIndex) { continue; }
+          } else if (headwordCount > 1) {
+            continue;
+          }
+          const dictionary = getFrequencyDictionaryName(frequency);
+          if (!dictionary) { continue; }
+          if (dictionaryFrequencyModeByName[dictionary] === 'occurrence-based') { continue; }
+          const rank =
+            parseDisplayFrequencyNumber(frequency.displayValue) ??
+            parsePositiveFrequencyNumber(frequency.frequency);
+          if (rank === null) { continue; }
+          const priorityRaw = dictionaryPriorityByName[dictionary];
+          const fallbackPriority =
+            typeof frequency.dictionaryIndex === 'number' && Number.isFinite(frequency.dictionaryIndex)
+              ? Math.max(0, Math.floor(frequency.dictionaryIndex))
+              : Number.MAX_SAFE_INTEGER;
+          const priority =
+            typeof priorityRaw === 'number' && Number.isFinite(priorityRaw)
+              ? Math.max(0, Math.floor(priorityRaw))
+              : fallbackPriority;
+          if (best === null || priority < best.priority || (priority === best.priority && rank < best.rank)) {
+            best = { priority, rank };
+          }
+        }
+        return best?.rank ?? null;
+      }
+      function hasExactSource(headword, token, requirePrimary) {
+        for (const src of headword?.sources || []) {
+          if (src.originalText !== token) { continue; }
+          if (requirePrimary && !src.isPrimary) { continue; }
+          if (src.matchType !== 'exact') { continue; }
+          return true;
+        }
+        return false;
+      }
+      function collectExactHeadwordMatches(dictionaryEntries, token, requirePrimary) {
+        const matches = [];
+        for (const dictionaryEntry of dictionaryEntries || []) {
+          const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
+          for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
+            const headword = headwords[headwordIndex];
+            if (!hasExactSource(headword, token, requirePrimary)) { continue; }
+            matches.push({ dictionaryEntry, headword, headwordIndex });
+          }
+        }
+        return matches;
+      }
+      function sameHeadword(match, preferredMatch) {
+        if (!match || !preferredMatch) {
+          return false;
+        }
+        if (match.headword?.term !== preferredMatch.headword?.term) {
+          return false;
+        }
+        const matchReading = typeof match.headword?.reading === 'string' ? match.headword.reading : '';
+        const preferredReading =
+          typeof preferredMatch.headword?.reading === 'string' ? preferredMatch.headword.reading : '';
+        if (!matchReading || !preferredReading) {
+          return true;
+        }
+        return matchReading === preferredReading;
+      }
+      function getBestFrequencyRankForMatches(matches, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
+        let best = null;
+        for (const match of matches) {
+          const rank = getBestFrequencyRank(
+            match.dictionaryEntry,
+            match.headwordIndex,
+            dictionaryPriorityByName,
+            dictionaryFrequencyModeByName
+          );
+          if (rank === null) { continue; }
+          if (best === null || rank < best) {
+            best = rank;
+          }
+        }
+        return best;
+      }
+`;

src/core/services/tokenizer/yomitan-furigana-script.ts

@@ -0,0 +1,170 @@
+// Furigana distribution for the injected scan runtime: splits a headword and
+// its reading into the segments a token carries, including the inflected case
+// where the matched source text differs from the dictionary form.
+
+export const YOMITAN_FURIGANA_HELPERS = String.raw`
+      function createFuriganaSegment(text, reading) { return {text, reading}; }
+      function getSegmentReadingContribution(segment) {
+        if (typeof segment.reading === "string" && segment.reading.length > 0) { return segment.reading; }
+        const segmentText = typeof segment.text === "string" ? segment.text : "";
+        const isKanaOnly = segmentText.length > 0 && [...segmentText].every((char) => isCodePointKana(char.codePointAt(0)));
+        return isKanaOnly ? convertHalfwidthKanaToKatakana(segmentText) : "";
+      }
+      function getProlongedHiragana(previousCharacter) {
+        switch (previousCharacter) {
+          case "あ": case "か": case "が": case "さ": case "ざ": case "た": case "だ": case "な": case "は": case "ば": case "ぱ": case "ま": case "や": case "ら": case "わ": case "ぁ": case "ゃ": case "ゎ": return "あ";
+          case "い": case "き": case "ぎ": case "し": case "じ": case "ち": case "ぢ": case "に": case "ひ": case "び": case "ぴ": case "み": case "り": case "ぃ": return "い";
+          case "う": case "く": case "ぐ": case "す": case "ず": case "つ": case "づ": case "ぬ": case "ふ": case "ぶ": case "ぷ": case "む": case "ゆ": case "る": case "ぅ": case "ゅ": return "う";
+          case "え": case "け": case "げ": case "せ": case "ぜ": case "て": case "で": case "ね": case "へ": case "べ": case "ぺ": case "め": case "れ": case "ぇ": return "え";
+          case "お": case "こ": case "ご": case "そ": case "ぞ": case "と": case "ど": case "の": case "ほ": case "ぼ": case "ぽ": case "も": case "よ": case "ろ": case "を": case "ぉ": case "ょ": return "う";
+          default: return null;
+        }
+      }
+      function getFuriganaKanaSegments(text, reading) {
+        const newSegments = [];
+        let start = 0;
+        let state = (reading[0] === text[0]);
+        for (let i = 1; i < text.length; ++i) {
+          const newState = (reading[i] === text[i]);
+          if (state === newState) { continue; }
+          newSegments.push(createFuriganaSegment(text.substring(start, i), state ? '' : reading.substring(start, i)));
+          state = newState;
+          start = i;
+        }
+        newSegments.push(createFuriganaSegment(text.substring(start), state ? '' : reading.substring(start)));
+        return newSegments;
+      }
+      function convertKatakanaToHiragana(text, keepProlongedSoundMarks = false) {
+        let result = '';
+        const offset = (HIRAGANA_CONVERSION_RANGE[0] - KATAKANA_CONVERSION_RANGE[0]);
+        for (let char of text) {
+          const codePoint = char.codePointAt(0);
+          switch (codePoint) {
+            case KATAKANA_SMALL_KA_CODE_POINT:
+            case KATAKANA_SMALL_KE_CODE_POINT:
+              break;
+            case KANA_PROLONGED_SOUND_MARK_CODE_POINT:
+            case HALFWIDTH_KANA_PROLONGED_SOUND_MARK_CODE_POINT:
+              char = "ー";
+              if (!keepProlongedSoundMarks && result.length > 0) {
+                const char2 = getProlongedHiragana(result[result.length - 1]);
+                if (char2 !== null) { char = char2; }
+              }
+              break;
+            default:
+              if (isCodePointInRange(codePoint, KATAKANA_CONVERSION_RANGE)) {
+                char = String.fromCodePoint(codePoint + offset);
+                break;
+              }
+              // Halfwidth katakana folds too, or a name written that way would
+              // match neither a candidate form nor its own reading.
+              const halfwidthHiragana = convertHalfwidthKanaCodePointToHiragana(codePoint);
+              if (halfwidthHiragana !== null) { char = halfwidthHiragana; }
+              break;
+          }
+          result += char;
+        }
+        return result;
+      }
+      function segmentizeFurigana(reading, readingNormalized, groups, groupsStart) {
+        const groupCount = groups.length - groupsStart;
+        if (groupCount <= 0) { return reading.length === 0 ? [] : null; }
+        const group = groups[groupsStart];
+        const {isKana, text} = group;
+        if (isKana) {
+          if (group.textNormalized !== null && readingNormalized.startsWith(group.textNormalized)) {
+            const segments = segmentizeFurigana(reading.substring(text.length), readingNormalized.substring(text.length), groups, groupsStart + 1);
+            if (segments !== null) {
+              if (reading.startsWith(text)) { segments.unshift(createFuriganaSegment(text, '')); }
+              else { segments.unshift(...getFuriganaKanaSegments(text, reading)); }
+              return segments;
+            }
+          }
+          return null;
+        }
+        let result = null;
+        for (let i = reading.length; i >= text.length; --i) {
+          const segments = segmentizeFurigana(reading.substring(i), readingNormalized.substring(i), groups, groupsStart + 1);
+          if (segments !== null) {
+            if (result !== null) { return null; }
+            segments.unshift(createFuriganaSegment(text, reading.substring(0, i)));
+            result = segments;
+          }
+          if (groupCount === 1) { break; }
+        }
+        return result;
+      }
+      function distributeFurigana(term, reading) {
+        if (reading === term) { return [createFuriganaSegment(term, '')]; }
+        const groups = [];
+        let groupPre = null;
+        let isKanaPre = null;
+        for (const c of term) {
+          const isKana = isCodePointKana(c.codePointAt(0));
+          if (isKana === isKanaPre) { groupPre.text += c; }
+          else {
+            groupPre = {isKana, text: c, textNormalized: null};
+            groups.push(groupPre);
+            isKanaPre = isKana;
+          }
+        }
+        for (const group of groups) {
+          if (group.isKana) { group.textNormalized = convertKatakanaToHiragana(group.text); }
+        }
+        const segments = segmentizeFurigana(reading, convertKatakanaToHiragana(reading), groups, 0);
+        return segments !== null ? segments : [createFuriganaSegment(term, reading)];
+      }
+      function getStemLength(text1, text2) {
+        const minLength = Math.min(text1.length, text2.length);
+        if (minLength === 0) { return 0; }
+        let i = 0;
+        while (true) {
+          const char1 = text1.codePointAt(i);
+          const char2 = text2.codePointAt(i);
+          if (char1 !== char2) { break; }
+          const charLength = String.fromCodePoint(char1).length;
+          i += charLength;
+          if (i >= minLength) {
+            if (i > minLength) { i -= charLength; }
+            break;
+          }
+        }
+        return i;
+      }
+      function distributeFuriganaInflected(term, reading, source) {
+        const termNormalized = convertKatakanaToHiragana(term);
+        const readingNormalized = convertKatakanaToHiragana(reading);
+        const sourceNormalized = convertKatakanaToHiragana(source);
+        let mainText = term;
+        let stemLength = getStemLength(termNormalized, sourceNormalized);
+        const readingStemLength = getStemLength(readingNormalized, sourceNormalized);
+        if (readingStemLength > 0 && readingStemLength >= stemLength) {
+          mainText = reading;
+          stemLength = readingStemLength;
+          reading = source.substring(0, stemLength) + reading.substring(stemLength);
+        }
+        const segments = [];
+        if (stemLength > 0) {
+          mainText = source.substring(0, stemLength) + mainText.substring(stemLength);
+          const segments2 = distributeFurigana(mainText, reading);
+          let consumed = 0;
+          for (const segment of segments2) {
+            const start = consumed;
+            consumed += segment.text.length;
+            if (consumed < stemLength) { segments.push(segment); }
+            else if (consumed === stemLength) { segments.push(segment); break; }
+            else {
+              if (start < stemLength) { segments.push(createFuriganaSegment(mainText.substring(start, stemLength), '')); }
+              break;
+            }
+          }
+        }
+        if (stemLength < source.length) {
+          const remainder = source.substring(stemLength);
+          const last = segments[segments.length - 1];
+          if (last && last.reading.length === 0) { last.text += remainder; }
+          else { segments.push(createFuriganaSegment(remainder, '')); }
+        }
+        return segments;
+      }
+`;

src/core/services/tokenizer/yomitan-kana-script.ts

@@ -0,0 +1,45 @@
+// Kana classification and normalization for the injected scan runtime: the
+// code-point ranges the walk tests every character against, and the folds that
+// let halfwidth and katakana spellings compare equal to their dictionary form.
+import { HAN_CODE_POINT_RANGES } from '../../text/han-code-points';
+
+export const YOMITAN_KANA_HELPERS = String.raw`
+      const HIRAGANA_CONVERSION_RANGE = [0x3041, 0x3096];
+      const KATAKANA_CONVERSION_RANGE = [0x30a1, 0x30f6];
+      const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc;
+      const KATAKANA_SMALL_KA_CODE_POINT = 0x30f5;
+      const KATAKANA_SMALL_KE_CODE_POINT = 0x30f6;
+      const KANA_RANGES = [[0x3040, 0x309f], [0x30a0, 0x30ff], [0xff66, 0xff9f]];
+      const HALFWIDTH_KATAKANA_RANGE = [0xff66, 0xff9d];
+      const HALFWIDTH_KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0xff70;
+      // Folded one code point to one, so every index into a normalized string
+      // still lines up with the original text — the name-candidate prefilter
+      // and the furigana stem matching both index back into it. The standalone
+      // voiced marks (ﾞ ﾟ) have no one-character equivalent and stay as they are.
+      const HALFWIDTH_KATAKANA_TO_HIRAGANA = "をぁぃぅぇぉゃゅょっーあいうえおかきくけこさしすせそたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわん";
+      function convertHalfwidthKanaCodePointToHiragana(codePoint) {
+        if (codePoint < HALFWIDTH_KATAKANA_RANGE[0] || codePoint > HALFWIDTH_KATAKANA_RANGE[1]) { return null; }
+        return HALFWIDTH_KATAKANA_TO_HIRAGANA[codePoint - HALFWIDTH_KATAKANA_RANGE[0]] || null;
+      }
+      // Halfwidth katakana is kana here but not to the rest of the pipeline
+      // (known-word matching and frequency lookups only fold fullwidth), so a
+      // reading taken from halfwidth text is written the way the fullwidth
+      // katakana path already writes it. NFKC rather than the per-code-point
+      // table: this is the one place where nothing indexes back into the
+      // result, so a voiced pair (ｶ + ﾞ) can compose into the single ガ it
+      // means instead of leaving a stray combining mark in the reading. Scoped
+      // to the halfwidth runs, because NFKC over everything else rewrites
+      // characters that have nothing to do with kana (① → 1, ㍑ → リットル).
+      function convertHalfwidthKanaToKatakana(text) {
+        return text.replace(/[ｦ-ﾟ]+/g, (run) => run.normalize("NFKC"));
+      }
+      // Han ranges come from the shared table so the scan walk and the character
+      // dictionary agree on what a kanji is (supplementary planes included).
+      // Halfwidth katakana counts as Japanese text: a name written that way has
+      // to reach the greedy pre-pass, which has its own handling for it.
+      const JAPANESE_RANGES = [[0x3040, 0x30ff], [0xff66, 0xff9f], ...${JSON.stringify(HAN_CODE_POINT_RANGES)}];
+      function isCodePointInRange(codePoint, range) { return codePoint >= range[0] && codePoint <= range[1]; }
+      function isCodePointInRanges(codePoint, ranges) { return ranges.some((range) => isCodePointInRange(codePoint, range)); }
+      function isCodePointKana(codePoint) { return isCodePointInRanges(codePoint, KANA_RANGES); }
+      function isCodePointJapanese(codePoint) { return isCodePointInRanges(codePoint, JAPANESE_RANGES); }
+`;

src/core/services/tokenizer/yomitan-match-selection-script.ts

@@ -0,0 +1,79 @@
+// Match selection for the injected scan runtime: picks the headword a position
+// tokenizes to, and the longest name or generic match in a window, which is how
+// the greedy name pre-pass decides what to reserve.
+
+export const YOMITAN_MATCH_SELECTION_HELPERS = String.raw`
+      function findLongestNameMatch(dictionaryEntries, textWindow) {
+        let best = null;
+        for (const dictionaryEntry of dictionaryEntries || []) {
+          if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; }
+          const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
+          for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
+            const headword = headwords[headwordIndex];
+            for (const src of headword?.sources || []) {
+              if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
+              const originalText = typeof src.originalText === 'string' ? src.originalText : '';
+              if (!originalText || !textWindow.startsWith(originalText)) { continue; }
+              if (best === null || originalText.length > best.sourceLength) {
+                best = { dictionaryEntry, headword, headwordIndex, sourceLength: originalText.length };
+              }
+            }
+          }
+        }
+        return best;
+      }
+      function findLongestGenericMatchLength(dictionaryEntries, textWindow) {
+        let best = 0;
+        for (const dictionaryEntry of dictionaryEntries || []) {
+          if (isNameDictionaryEntry(dictionaryEntry)) { continue; }
+          const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
+          for (const headword of headwords) {
+            for (const src of headword?.sources || []) {
+              if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
+              const originalText = typeof src.originalText === 'string' ? src.originalText : '';
+              if (!originalText || !textWindow.startsWith(originalText)) { continue; }
+              if (originalText.length > best) { best = originalText.length; }
+            }
+          }
+        }
+        return best;
+      }
+      function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
+        const currentMediaDictionaryEntries =
+          currentCharacterDictionaryMediaId === null
+            ? (dictionaryEntries || [])
+            : (dictionaryEntries || []).filter((entry) => {
+                if (!isNameDictionaryEntry(entry)) { return true; }
+                return isCurrentMediaNameDictionaryEntry(entry);
+              });
+        const exactPrimaryMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, true);
+        let matchedNameDictionary = false;
+        if (includeNameMatchMetadata) {
+          // Every match already comes from currentMediaDictionaryEntries, so
+          // classifying its own entry is enough.
+          for (const match of exactPrimaryMatches) {
+            if (!isCurrentMediaNameDictionaryEntry(match.dictionaryEntry)) { continue; }
+            matchedNameDictionary = true;
+            break;
+          }
+        }
+        const preferredMatch = exactPrimaryMatches[0];
+        if (preferredMatch) {
+          const exactFrequencyMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, false)
+            .filter((match) => sameHeadword(match, preferredMatch));
+          return {
+            term: preferredMatch.headword.term,
+            reading: preferredMatch.headword.reading,
+            wordClasses: normalizeWordClasses(preferredMatch.headword),
+            isNameMatch:
+              matchedNameDictionary || isCurrentMediaNameDictionaryEntry(preferredMatch.dictionaryEntry),
+            frequencyRank: getBestFrequencyRankForMatches(
+              exactFrequencyMatches.length > 0 ? exactFrequencyMatches : exactPrimaryMatches,
+              dictionaryPriorityByName,
+              dictionaryFrequencyModeByName
+            )
+          };
+        }
+        return null;
+      }
+`;

src/core/services/tokenizer/yomitan-parser-runtime.ts

@@ -3,6 +3,14 @@ import * as fs from 'fs';
 import * as http from 'http';
 import * as path from 'path';
 import { selectYomitanParseTokens } from './parser-selection-stage';
+import {
+  buildYomitanScanCallScript,
+  buildYomitanScanNameCandidatesScript,
+  CHARACTER_DICTIONARY_TITLE_PREFIX,
+  YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT,
+  YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL,
+  type YomitanFrequencyMode,
+} from './yomitan-scan-runtime-script';
 
 interface LoggerLike {
   error: (message: string, ...args: unknown[]) => void;
@@ -22,8 +30,6 @@ interface YomitanParserRuntimeDeps {
   createYomitanExtensionWindow?: (pageName: string) => Promise<BrowserWindow | null>;
 }
 
-type YomitanFrequencyMode = 'occurrence-based' | 'rank-based';
-
 export interface YomitanDictionaryInfo {
   title: string;
   revision?: string | number;
@@ -74,13 +80,19 @@ export interface YomitanAddNoteResult {
 }
 
 const DEFAULT_YOMITAN_SCAN_LENGTH = 40;
-const CHARACTER_DICTIONARY_TITLE_PREFIX = 'SubMiner Character Dictionary';
 const yomitanProfileMetadataByWindow = new WeakMap<BrowserWindow, YomitanProfileMetadata>();
 const yomitanProfileDiagnosticsLoggedByWindow = new WeakSet<BrowserWindow>();
 const yomitanFrequencyCacheByWindow = new WeakMap<
   BrowserWindow,
   Map<string, YomitanTermFrequency[]>
 >();
+// Epoch passed with every scan request; the in-window termsFind cache clears
+// itself when the epoch changes (dictionary imports, settings changes).
+const yomitanScanCacheEpochByWindow = new WeakMap<BrowserWindow, number>();
+
+function getYomitanScanCacheEpoch(window: BrowserWindow): number {
+  return yomitanScanCacheEpochByWindow.get(window) ?? 0;
+}
 
 function isObject(value: unknown): value is Record<string, unknown> {
   return Boolean(value && typeof value === 'object');
@@ -99,6 +111,7 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
         typeof entry.startPos === 'number' &&
         typeof entry.endPos === 'number' &&
         (entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean') &&
+        (entry.isUnparsedRun === undefined || typeof entry.isUnparsedRun === 'boolean') &&
         (entry.frequencyRank === undefined || typeof entry.frequencyRank === 'number') &&
         (entry.wordClasses === undefined ||
           (Array.isArray(entry.wordClasses) &&
@@ -107,13 +120,9 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
   );
 }
 
-function scanTokenSpanKey(token: YomitanScanToken): string {
-  return `${token.startPos}:${token.endPos}:${token.surface}`;
-}
-
 // Maps a parse-selected token to the scanner-token shape carried out of the
-// parser runtime. Shared by both selectYomitanParseTokens fallback paths so the
-// projected fields stay in sync as the shape changes.
+// parser runtime, used by the parseText fallback path when the in-window
+// scanner is unavailable.
 function toYomitanScanToken(token: {
   surface: string;
   reading: string;
@@ -132,66 +141,6 @@ function toYomitanScanToken(token: {
   };
 }
 
-// parseText segmentation is authoritative (it emits filler chunks for text the
-// termsFind scanner skips), but only the termsFind scanner carries annotation
-// metadata (isNameMatch, frequencyRank, headwordReading, wordClasses). Graft
-// scanner tokens onto the parseText segmentation per matching span so one
-// unmatched chunk degrades only itself instead of dropping the whole line's
-// metadata.
-//
-// Exception: character-name tokens. The greedy name scan can re-segment text
-// around a name (e.g. とヨータ → と + ヨータ instead of とヨー + タ), so
-// parseText segmentation cannot be authoritative there. Each name span is
-// expanded until it aligns with token boundaries in both segmentations, then
-// the parse tokens inside are replaced with the scanner tokens.
-function mergeScannerTokensIntoParseTokens(
-  parseScanTokens: YomitanScanToken[],
-  scannerTokens: YomitanScanToken[],
-): YomitanScanToken[] {
-  const scannerTokensBySpan = new Map<string, YomitanScanToken>();
-  for (const token of scannerTokens) {
-    scannerTokensBySpan.set(scanTokenSpanKey(token), token);
-  }
-  const graftedTokens = parseScanTokens.map(
-    (token) => scannerTokensBySpan.get(scanTokenSpanKey(token)) ?? token,
-  );
-
-  const nameTokens = scannerTokens.filter((token) => token.isNameMatch === true);
-  if (nameTokens.length === 0) {
-    return graftedTokens;
-  }
-
-  const regions = nameTokens.map((token) => ({ start: token.startPos, end: token.endPos }));
-  const allTokens = [...parseScanTokens, ...scannerTokens];
-  let expanded = true;
-  while (expanded) {
-    expanded = false;
-    for (const region of regions) {
-      for (const token of allTokens) {
-        const overlaps = token.startPos < region.end && token.endPos > region.start;
-        const extendsBeyond = token.startPos < region.start || token.endPos > region.end;
-        if (overlaps && extendsBeyond) {
-          region.start = Math.min(region.start, token.startPos);
-          region.end = Math.max(region.end, token.endPos);
-          expanded = true;
-        }
-      }
-    }
-  }
-
-  const isInsideNameRegion = (token: YomitanScanToken): boolean =>
-    regions.some((region) => token.startPos >= region.start && token.endPos <= region.end);
-
-  const merged = graftedTokens.filter((token) => !isInsideNameRegion(token));
-  for (const token of scannerTokens) {
-    if (isInsideNameRegion(token)) {
-      merged.push(token);
-    }
-  }
-  merged.sort((a, b) => a.startPos - b.startPos || a.endPos - b.endPos);
-  return merged;
-}
-
 function makeTermReadingCacheKey(term: string, reading: string | null): string {
   return `${term}\u0000${reading ?? ''}`;
 }
@@ -208,6 +157,7 @@ function getWindowFrequencyCache(window: BrowserWindow): Map<string, YomitanTerm
 function clearWindowCaches(window: BrowserWindow): void {
   yomitanProfileMetadataByWindow.delete(window);
   yomitanFrequencyCacheByWindow.delete(window);
+  yomitanScanCacheEpochByWindow.set(window, getYomitanScanCacheEpoch(window) + 1);
 }
 export function clearYomitanParserCachesForWindow(window: BrowserWindow): void {
   clearWindowCaches(window);
@@ -704,6 +654,10 @@ async function ensureYomitanParserWindow(
       if (readyPromise) {
         await readyPromise;
       }
+      // Eagerly install the scan runtime so the first subtitle line does not
+      // pay the install round trip; failures fall back to the per-request
+      // install-and-retry path.
+      await installYomitanScanRuntime(parserWindow).catch(() => {});
 
       return true;
     } catch (err) {
@@ -877,668 +831,42 @@ async function serveDictionaryZipOnce<T>(
   }
 }
 
-const YOMITAN_SCANNING_HELPERS = String.raw`
-      const HIRAGANA_CONVERSION_RANGE = [0x3041, 0x3096];
-      const KATAKANA_CONVERSION_RANGE = [0x30a1, 0x30f6];
-      const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc;
-      const KATAKANA_SMALL_KA_CODE_POINT = 0x30f5;
-      const KATAKANA_SMALL_KE_CODE_POINT = 0x30f6;
-      const KANA_RANGES = [[0x3040, 0x309f], [0x30a0, 0x30ff]];
-      const JAPANESE_RANGES = [[0x3040, 0x30ff], [0x3400, 0x9fff]];
-      function isCodePointInRange(codePoint, range) { return codePoint >= range[0] && codePoint <= range[1]; }
-      function isCodePointInRanges(codePoint, ranges) { return ranges.some((range) => isCodePointInRange(codePoint, range)); }
-      function isCodePointKana(codePoint) { return isCodePointInRanges(codePoint, KANA_RANGES); }
-      function isCodePointJapanese(codePoint) { return isCodePointInRanges(codePoint, JAPANESE_RANGES); }
-      function createFuriganaSegment(text, reading) { return {text, reading}; }
-      function getSegmentReadingContribution(segment) {
-        if (typeof segment.reading === "string" && segment.reading.length > 0) { return segment.reading; }
-        const segmentText = typeof segment.text === "string" ? segment.text : "";
-        const isKanaOnly = segmentText.length > 0 && [...segmentText].every((char) => isCodePointKana(char.codePointAt(0)));
-        return isKanaOnly ? segmentText : "";
-      }
-      function getProlongedHiragana(previousCharacter) {
-        switch (previousCharacter) {
-          case "あ": case "か": case "が": case "さ": case "ざ": case "た": case "だ": case "な": case "は": case "ば": case "ぱ": case "ま": case "や": case "ら": case "わ": case "ぁ": case "ゃ": case "ゎ": return "あ";
-          case "い": case "き": case "ぎ": case "し": case "じ": case "ち": case "ぢ": case "に": case "ひ": case "び": case "ぴ": case "み": case "り": case "ぃ": return "い";
-          case "う": case "く": case "ぐ": case "す": case "ず": case "つ": case "づ": case "ぬ": case "ふ": case "ぶ": case "ぷ": case "む": case "ゆ": case "る": case "ぅ": case "ゅ": return "う";
-          case "え": case "け": case "げ": case "せ": case "ぜ": case "て": case "で": case "ね": case "へ": case "べ": case "ぺ": case "め": case "れ": case "ぇ": return "え";
-          case "お": case "こ": case "ご": case "そ": case "ぞ": case "と": case "ど": case "の": case "ほ": case "ぼ": case "ぽ": case "も": case "よ": case "ろ": case "を": case "ぉ": case "ょ": return "う";
-          default: return null;
-        }
-      }
-      function getFuriganaKanaSegments(text, reading) {
-        const newSegments = [];
-        let start = 0;
-        let state = (reading[0] === text[0]);
-        for (let i = 1; i < text.length; ++i) {
-          const newState = (reading[i] === text[i]);
-          if (state === newState) { continue; }
-          newSegments.push(createFuriganaSegment(text.substring(start, i), state ? '' : reading.substring(start, i)));
-          state = newState;
-          start = i;
-        }
-        newSegments.push(createFuriganaSegment(text.substring(start), state ? '' : reading.substring(start)));
-        return newSegments;
-      }
-      function convertKatakanaToHiragana(text, keepProlongedSoundMarks = false) {
-        let result = '';
-        const offset = (HIRAGANA_CONVERSION_RANGE[0] - KATAKANA_CONVERSION_RANGE[0]);
-        for (let char of text) {
-          const codePoint = char.codePointAt(0);
-          switch (codePoint) {
-            case KATAKANA_SMALL_KA_CODE_POINT:
-            case KATAKANA_SMALL_KE_CODE_POINT:
-              break;
-            case KANA_PROLONGED_SOUND_MARK_CODE_POINT:
-              if (!keepProlongedSoundMarks && result.length > 0) {
-                const char2 = getProlongedHiragana(result[result.length - 1]);
-                if (char2 !== null) { char = char2; }
-              }
-              break;
-            default:
-              if (isCodePointInRange(codePoint, KATAKANA_CONVERSION_RANGE)) {
-                char = String.fromCodePoint(codePoint + offset);
-              }
-              break;
-          }
-          result += char;
-        }
-        return result;
-      }
-      function segmentizeFurigana(reading, readingNormalized, groups, groupsStart) {
-        const groupCount = groups.length - groupsStart;
-        if (groupCount <= 0) { return reading.length === 0 ? [] : null; }
-        const group = groups[groupsStart];
-        const {isKana, text} = group;
-        if (isKana) {
-          if (group.textNormalized !== null && readingNormalized.startsWith(group.textNormalized)) {
-            const segments = segmentizeFurigana(reading.substring(text.length), readingNormalized.substring(text.length), groups, groupsStart + 1);
-            if (segments !== null) {
-              if (reading.startsWith(text)) { segments.unshift(createFuriganaSegment(text, '')); }
-              else { segments.unshift(...getFuriganaKanaSegments(text, reading)); }
-              return segments;
-            }
-          }
-          return null;
-        }
-        let result = null;
-        for (let i = reading.length; i >= text.length; --i) {
-          const segments = segmentizeFurigana(reading.substring(i), readingNormalized.substring(i), groups, groupsStart + 1);
-          if (segments !== null) {
-            if (result !== null) { return null; }
-            segments.unshift(createFuriganaSegment(text, reading.substring(0, i)));
-            result = segments;
-          }
-          if (groupCount === 1) { break; }
-        }
-        return result;
-      }
-      function distributeFurigana(term, reading) {
-        if (reading === term) { return [createFuriganaSegment(term, '')]; }
-        const groups = [];
-        let groupPre = null;
-        let isKanaPre = null;
-        for (const c of term) {
-          const isKana = isCodePointKana(c.codePointAt(0));
-          if (isKana === isKanaPre) { groupPre.text += c; }
-          else {
-            groupPre = {isKana, text: c, textNormalized: null};
-            groups.push(groupPre);
-            isKanaPre = isKana;
-          }
-        }
-        for (const group of groups) {
-          if (group.isKana) { group.textNormalized = convertKatakanaToHiragana(group.text); }
-        }
-        const segments = segmentizeFurigana(reading, convertKatakanaToHiragana(reading), groups, 0);
-        return segments !== null ? segments : [createFuriganaSegment(term, reading)];
-      }
-      function getStemLength(text1, text2) {
-        const minLength = Math.min(text1.length, text2.length);
-        if (minLength === 0) { return 0; }
-        let i = 0;
-        while (true) {
-          const char1 = text1.codePointAt(i);
-          const char2 = text2.codePointAt(i);
-          if (char1 !== char2) { break; }
-          const charLength = String.fromCodePoint(char1).length;
-          i += charLength;
-          if (i >= minLength) {
-            if (i > minLength) { i -= charLength; }
-            break;
-          }
-        }
-        return i;
-      }
-      function distributeFuriganaInflected(term, reading, source) {
-        const termNormalized = convertKatakanaToHiragana(term);
-        const readingNormalized = convertKatakanaToHiragana(reading);
-        const sourceNormalized = convertKatakanaToHiragana(source);
-        let mainText = term;
-        let stemLength = getStemLength(termNormalized, sourceNormalized);
-        const readingStemLength = getStemLength(readingNormalized, sourceNormalized);
-        if (readingStemLength > 0 && readingStemLength >= stemLength) {
-          mainText = reading;
-          stemLength = readingStemLength;
-          reading = source.substring(0, stemLength) + reading.substring(stemLength);
-        }
-        const segments = [];
-        if (stemLength > 0) {
-          mainText = source.substring(0, stemLength) + mainText.substring(stemLength);
-          const segments2 = distributeFurigana(mainText, reading);
-          let consumed = 0;
-          for (const segment of segments2) {
-            const start = consumed;
-            consumed += segment.text.length;
-            if (consumed < stemLength) { segments.push(segment); }
-            else if (consumed === stemLength) { segments.push(segment); break; }
-            else {
-              if (start < stemLength) { segments.push(createFuriganaSegment(mainText.substring(start, stemLength), '')); }
-              break;
-            }
-          }
-        }
-        if (stemLength < source.length) {
-          const remainder = source.substring(stemLength);
-          const last = segments[segments.length - 1];
-          if (last && last.reading.length === 0) { last.text += remainder; }
-          else { segments.push(createFuriganaSegment(remainder, '')); }
-        }
-        return segments;
-      }
-      function parsePositiveFrequencyNumber(value) {
-        if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
-          return Math.max(1, Math.floor(value));
-        }
-        if (typeof value === 'string') {
-          const numericMatch = value.trim().match(/[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?/)?.[0];
-          if (!numericMatch) { return null; }
-          const parsed = Number.parseFloat(numericMatch);
-          if (!Number.isFinite(parsed) || parsed <= 0) { return null; }
-          return Math.max(1, Math.floor(parsed));
-        }
-        if (Array.isArray(value)) {
-          for (const item of value) {
-            const parsed = parsePositiveFrequencyNumber(item);
-            if (parsed !== null) { return parsed; }
-          }
-        }
-        return null;
-      }
-      function parseDisplayFrequencyNumber(value) {
-        if (typeof value === 'string') {
-          const leadingDigits = value.trim().match(/^\d+/)?.[0];
-          if (!leadingDigits) { return null; }
-          const parsed = Number.parseInt(leadingDigits, 10);
-          return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
-        }
-        return parsePositiveFrequencyNumber(value);
-      }
-      function getFrequencyDictionaryName(frequency) {
-        const candidates = [
-          frequency?.dictionary,
-          frequency?.dictionaryName,
-          frequency?.name,
-          frequency?.title,
-          frequency?.dictionaryTitle,
-          frequency?.dictionaryAlias
-        ];
-        for (const candidate of candidates) {
-          if (typeof candidate === 'string' && candidate.trim().length > 0) {
-            return candidate.trim();
-          }
-        }
-        return null;
-      }
-      function getBestFrequencyRank(dictionaryEntry, headwordIndex, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
-        let best = null;
-        const headwordCount = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords.length : 0;
-        for (const frequency of dictionaryEntry?.frequencies || []) {
-          if (!frequency || typeof frequency !== 'object') { continue; }
-          const frequencyHeadwordIndex = frequency.headwordIndex;
-          if (typeof frequencyHeadwordIndex === 'number') {
-            if (frequencyHeadwordIndex !== headwordIndex) { continue; }
-          } else if (headwordCount > 1) {
-            continue;
-          }
-          const dictionary = getFrequencyDictionaryName(frequency);
-          if (!dictionary) { continue; }
-          if (dictionaryFrequencyModeByName[dictionary] === 'occurrence-based') { continue; }
-          const rank =
-            parseDisplayFrequencyNumber(frequency.displayValue) ??
-            parsePositiveFrequencyNumber(frequency.frequency);
-          if (rank === null) { continue; }
-          const priorityRaw = dictionaryPriorityByName[dictionary];
-          const fallbackPriority =
-            typeof frequency.dictionaryIndex === 'number' && Number.isFinite(frequency.dictionaryIndex)
-              ? Math.max(0, Math.floor(frequency.dictionaryIndex))
-              : Number.MAX_SAFE_INTEGER;
-          const priority =
-            typeof priorityRaw === 'number' && Number.isFinite(priorityRaw)
-              ? Math.max(0, Math.floor(priorityRaw))
-              : fallbackPriority;
-          if (best === null || priority < best.priority || (priority === best.priority && rank < best.rank)) {
-            best = { priority, rank };
-          }
-        }
-        return best?.rank ?? null;
-      }
-      function hasExactSource(headword, token, requirePrimary) {
-        for (const src of headword.sources || []) {
-          if (src.originalText !== token) { continue; }
-          if (requirePrimary && !src.isPrimary) { continue; }
-          if (src.matchType !== 'exact') { continue; }
-          return true;
-        }
-        return false;
-      }
-      function collectExactHeadwordMatches(dictionaryEntries, token, requirePrimary) {
-        const matches = [];
-        for (const dictionaryEntry of dictionaryEntries || []) {
-          const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
-          for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
-            const headword = headwords[headwordIndex];
-            if (!hasExactSource(headword, token, requirePrimary)) { continue; }
-            matches.push({ dictionaryEntry, headword, headwordIndex });
-          }
-        }
-        return matches;
-      }
-      function sameHeadword(match, preferredMatch) {
-        if (!match || !preferredMatch) {
-          return false;
-        }
-        if (match.headword?.term !== preferredMatch.headword?.term) {
-          return false;
-        }
-        const matchReading = typeof match.headword?.reading === 'string' ? match.headword.reading : '';
-        const preferredReading =
-          typeof preferredMatch.headword?.reading === 'string' ? preferredMatch.headword.reading : '';
-        if (!matchReading || !preferredReading) {
-          return true;
-        }
-        return matchReading === preferredReading;
-      }
-      function getBestFrequencyRankForMatches(matches, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
-        let best = null;
-        for (const match of matches) {
-          const rank = getBestFrequencyRank(
-            match.dictionaryEntry,
-            match.headwordIndex,
-            dictionaryPriorityByName,
-            dictionaryFrequencyModeByName
-          );
-          if (rank === null) { continue; }
-          if (best === null || rank < best) {
-            best = rank;
-          }
-        }
-        return best;
-      }
-      function normalizeWordClasses(headword) {
-          if (!Array.isArray(headword?.wordClasses)) { return undefined; }
-          const classes = headword.wordClasses.filter((wordClass) => typeof wordClass === "string" && wordClass.trim().length > 0);
-          return classes.length > 0 ? classes : undefined;
-        }
-        function appendDictionaryNames(target, value) {
-          if (!value || typeof value !== 'object') {
-            return;
-          }
-          const candidates = [
-            value.dictionary,
-            value.dictionaryName,
-            value.name,
-            value.title,
-            value.dictionaryTitle,
-            value.dictionaryAlias
-          ];
-          for (const candidate of candidates) {
-            if (typeof candidate === 'string' && candidate.trim().length > 0) {
-              target.push(candidate.trim());
-            }
-          }
-        }
-        function getDictionaryEntryNames(entry) {
-          const names = [];
-          appendDictionaryNames(names, entry);
-          for (const definition of entry?.definitions || []) {
-            appendDictionaryNames(names, definition);
-          }
-          for (const frequency of entry?.frequencies || []) {
-            appendDictionaryNames(names, frequency);
-          }
-          for (const pronunciation of entry?.pronunciations || []) {
-            appendDictionaryNames(names, pronunciation);
-          }
-          return names;
-        }
-        function isNameDictionaryEntry(entry) {
-          if (!includeNameMatchMetadata || !entry || typeof entry !== 'object') {
-            return false;
-          }
-          return getDictionaryEntryNames(entry).some((name) => name.startsWith(${JSON.stringify(CHARACTER_DICTIONARY_TITLE_PREFIX)}));
-        }
-        function parseSubMinerMediaIdFromString(value) {
-          const imageMatch = value.match(/\bimg\/m(\d+)-/i);
-          if (imageMatch) {
-            const parsed = Number.parseInt(imageMatch[1], 10);
-            if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
-          }
-          const titleMatch = value.match(/${CHARACTER_DICTIONARY_TITLE_PREFIX}[^\d]*(?:AniList\s*)?(\d+)/i);
-          if (titleMatch) {
-            const parsed = Number.parseInt(titleMatch[1], 10);
-            if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
-          }
-          return null;
-        }
-        function parseSubMinerMediaIdCandidate(value) {
-          if (typeof value === 'number' && Number.isSafeInteger(value) && value > 0) {
-            return value;
-          }
-          if (typeof value === 'string' && /^\d+$/.test(value.trim())) {
-            const parsed = Number.parseInt(value.trim(), 10);
-            if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
-          }
-          return null;
-        }
-        function collectSubMinerMediaIds(value, target) {
-          if (typeof value === 'string') {
-            const parsed = parseSubMinerMediaIdFromString(value);
-            if (parsed !== null) { target.add(parsed); }
-            return;
-          }
-          if (!value || typeof value !== 'object') {
-            return;
-          }
-          if (Array.isArray(value)) {
-            for (const item of value) { collectSubMinerMediaIds(item, target); }
-            return;
-          }
-          const mediaIdCandidates = [
-            value.subminerMediaId,
-            value.subMinerMediaId,
-            value.characterDictionaryMediaId,
-            value.data?.subminerMediaId,
-            value.data?.subMinerMediaId,
-            value.data?.characterDictionaryMediaId
-          ];
-          for (const candidate of mediaIdCandidates) {
-            const parsed = parseSubMinerMediaIdCandidate(candidate);
-            if (parsed !== null) { target.add(parsed); }
-          }
-          for (const child of Object.values(value)) {
-            collectSubMinerMediaIds(child, target);
-          }
-        }
-        function getSubMinerMediaIds(entry) {
-          const mediaIds = new Set();
-          collectSubMinerMediaIds(entry, mediaIds);
-          return mediaIds;
-        }
-        function isCurrentMediaNameDictionaryEntry(entry) {
-          if (!isNameDictionaryEntry(entry)) {
-            return false;
-          }
-          if (currentCharacterDictionaryMediaId === null) {
-            return true;
-          }
-          const mediaIds = getSubMinerMediaIds(entry);
-          return mediaIds.size === 0 || mediaIds.has(currentCharacterDictionaryMediaId);
-      }
-      function findLongestNameMatch(dictionaryEntries, textWindow) {
-        let best = null;
-        for (const dictionaryEntry of dictionaryEntries || []) {
-          if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; }
-          const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
-          for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
-            const headword = headwords[headwordIndex];
-            for (const src of headword?.sources || []) {
-              if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
-              const originalText = typeof src.originalText === 'string' ? src.originalText : '';
-              if (!originalText || !textWindow.startsWith(originalText)) { continue; }
-              if (best === null || originalText.length > best.sourceLength) {
-                best = { dictionaryEntry, headword, headwordIndex, sourceLength: originalText.length };
-              }
-            }
-          }
-        }
-        return best;
-      }
-      function findLongestGenericMatchLength(dictionaryEntries, textWindow) {
-        let best = 0;
-        for (const dictionaryEntry of dictionaryEntries || []) {
-          if (isNameDictionaryEntry(dictionaryEntry)) { continue; }
-          const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
-          for (const headword of headwords) {
-            for (const src of headword?.sources || []) {
-              if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
-              const originalText = typeof src.originalText === 'string' ? src.originalText : '';
-              if (!originalText || !textWindow.startsWith(originalText)) { continue; }
-              if (originalText.length > best) { best = originalText.length; }
-            }
-          }
-        }
-        return best;
-      }
-      function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
-        const currentMediaDictionaryEntries =
-          currentCharacterDictionaryMediaId === null
-            ? (dictionaryEntries || [])
-            : (dictionaryEntries || []).filter((entry) => {
-                if (!isNameDictionaryEntry(entry)) { return true; }
-                return isCurrentMediaNameDictionaryEntry(entry);
-              });
-        const exactPrimaryMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, true);
-        let matchedNameDictionary = false;
-        if (includeNameMatchMetadata) {
-          for (const dictionaryEntry of currentMediaDictionaryEntries || []) {
-            if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; }
-            for (const match of exactPrimaryMatches) {
-              if (match.dictionaryEntry !== dictionaryEntry) { continue; }
-              matchedNameDictionary = true;
-              break;
-            }
-            if (matchedNameDictionary) { break; }
-          }
-        }
-        const preferredMatch = exactPrimaryMatches[0];
-        if (preferredMatch) {
-          const exactFrequencyMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, false)
-            .filter((match) => sameHeadword(match, preferredMatch));
-          return {
-            term: preferredMatch.headword.term,
-            reading: preferredMatch.headword.reading,
-            wordClasses: normalizeWordClasses(preferredMatch.headword),
-            isNameMatch:
-              matchedNameDictionary || isCurrentMediaNameDictionaryEntry(preferredMatch.dictionaryEntry),
-            frequencyRank: getBestFrequencyRankForMatches(
-              exactFrequencyMatches.length > 0 ? exactFrequencyMatches : exactPrimaryMatches,
-              dictionaryPriorityByName,
-              dictionaryFrequencyModeByName
-            )
-          };
-        }
-        return null;
-      }
-`;
+async function installYomitanScanRuntime(parserWindow: BrowserWindow): Promise<void> {
+  await parserWindow.webContents.executeJavaScript(YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT, true);
+  // A fresh runtime has no candidate list; force the next scan to reinstall it.
+  yomitanScanNameCandidateKeyByWindow.delete(parserWindow);
+}
 
-function buildYomitanScanningScript(
-  text: string,
-  profileIndex: number,
-  scanLength: number,
-  includeNameMatchMetadata: boolean,
-  greedyNameScanEnabled: boolean,
-  currentCharacterDictionaryMediaId: number | null,
-  dictionaryPriorityByName: Record<string, number>,
-  dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>,
-): string {
-  return `
-    (async () => {
-      const invoke = (action, params) =>
-        new Promise((resolve, reject) => {
-          chrome.runtime.sendMessage({ action, params }, (response) => {
-            if (chrome.runtime.lastError) {
-              reject(new Error(chrome.runtime.lastError.message));
-              return;
-            }
-            if (!response || typeof response !== "object") {
-              reject(new Error("Invalid response from Yomitan backend"));
-              return;
-            }
-            if (response.error) {
-              reject(new Error(response.error.message || "Yomitan backend error"));
-              return;
-            }
-            resolve(response.result);
-          });
-        });
-${YOMITAN_SCANNING_HELPERS}
-      const includeNameMatchMetadata = ${includeNameMatchMetadata ? 'true' : 'false'};
-      const greedyNameScanEnabled = ${greedyNameScanEnabled ? 'true' : 'false'};
-      const currentCharacterDictionaryMediaId = ${
-        currentCharacterDictionaryMediaId !== null
-          ? String(currentCharacterDictionaryMediaId)
-          : 'null'
-      };
-      const dictionaryPriorityByName = ${JSON.stringify(dictionaryPriorityByName)};
-      const dictionaryFrequencyModeByName = ${JSON.stringify(dictionaryFrequencyModeByName)};
-      const text = ${JSON.stringify(text)};
-      const details = {matchType: "exact", deinflect: true};
-      const tokens = [];
-      const termsFindCache = new Map();
-      async function termsFindAt(position, windowLength) {
-        const cacheKey = position + ":" + windowLength;
-        const cached = termsFindCache.get(cacheKey);
-        if (cached) { return cached; }
-        const substring = text.substring(position, position + windowLength);
-        const result = await invoke("termsFind", { text: substring, details, optionsContext: { index: ${profileIndex} } });
-        termsFindCache.set(cacheKey, result);
-        return result;
-      }
-      function buildScanToken(position, source, preferredHeadword) {
-        const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
-        const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
-        const tokenPayload = {
-          surface: segments.map((segment) => segment.text).join("") || source,
-          reading: segments.map(getSegmentReadingContribution).join(""),
-          headword: preferredHeadword.term,
-          headwordReading: reading || undefined,
-          startPos: position,
-          endPos: position + source.length,
-          isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
-          frequencyRank:
-            typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
-              ? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
-              : undefined,
-        };
-        if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
-          tokenPayload.wordClasses = preferredHeadword.wordClasses;
-        }
-        return tokenPayload;
-      }
-      async function findTokenAt(position, windowLength) {
-        const codePoint = text.codePointAt(position);
-        const character = String.fromCodePoint(codePoint);
-        const result = await termsFindAt(position, windowLength);
-        const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
-        const originalTextLength = typeof result?.originalTextLength === "number" ? result.originalTextLength : 0;
-        if (dictionaryEntries.length === 0 || originalTextLength <= 0 || (originalTextLength === character.length && !isCodePointJapanese(codePoint))) {
-          return { token: null, matchedLength: 0 };
-        }
-        const source = text.substring(position, position + originalTextLength);
-        const preferredHeadword = getPreferredHeadword(
-          dictionaryEntries,
-          source,
-          dictionaryPriorityByName,
-          dictionaryFrequencyModeByName
-        );
-        if (!preferredHeadword || typeof preferredHeadword.term !== "string") {
-          return { token: null, matchedLength: originalTextLength };
-        }
-        return { token: buildScanToken(position, source, preferredHeadword), matchedLength: originalTextLength };
-      }
-      // Greedy name pre-pass: character-name matches claim their spans before
-      // the left-to-right walk, so a longer generic match starting earlier
-      // (e.g. とヨー → 渡洋) cannot swallow the start of a name (ヨータ).
-      const nameTokens = [];
-      if (greedyNameScanEnabled) {
-        let namePos = 0;
-        while (namePos < text.length) {
-          const codePoint = text.codePointAt(namePos);
-          if (!isCodePointJapanese(codePoint)) {
-            namePos += String.fromCodePoint(codePoint).length;
-            continue;
-          }
-          const result = await termsFindAt(namePos, ${scanLength});
-          const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
-          const textWindow = text.substring(namePos, namePos + ${scanLength});
-          const nameMatch = findLongestNameMatch(dictionaryEntries, textWindow);
-          // A name only claims its span when no strictly longer generic word
-          // starts at the same position (a character named 空 must not split
-          // 空気). Ties go to the name. Generic matches that start earlier and
-          // overlap the name are still blocked by the reservation.
-          if (
-            !nameMatch ||
-            findLongestGenericMatchLength(dictionaryEntries, textWindow) > nameMatch.sourceLength
-          ) {
-            namePos += String.fromCodePoint(codePoint).length;
-            continue;
-          }
-          const source = text.substring(namePos, namePos + nameMatch.sourceLength);
-          nameTokens.push(buildScanToken(namePos, source, {
-            term: nameMatch.headword.term,
-            reading: nameMatch.headword.reading,
-            wordClasses: normalizeWordClasses(nameMatch.headword),
-            isNameMatch: true,
-            frequencyRank: getBestFrequencyRank(
-              nameMatch.dictionaryEntry,
-              nameMatch.headwordIndex,
-              dictionaryPriorityByName,
-              dictionaryFrequencyModeByName
-            )
-          }));
-          namePos += nameMatch.sourceLength;
-        }
-      }
-      let i = 0;
-      let nameIndex = 0;
-      while (i < text.length) {
-        while (nameIndex < nameTokens.length && nameTokens[nameIndex].startPos < i) { nameIndex += 1; }
-        const nextNameToken = nameIndex < nameTokens.length ? nameTokens[nameIndex] : null;
-        if (nextNameToken && nextNameToken.startPos === i) {
-          tokens.push(nextNameToken);
-          i = nextNameToken.endPos;
-          nameIndex += 1;
-          continue;
-        }
-        // Cap the window at the next reserved name span so a generic match
-        // cannot consume into it.
-        const windowLength = nextNameToken ? Math.min(${scanLength}, nextNameToken.startPos - i) : ${scanLength};
-        let attempt = await findTokenAt(i, windowLength);
-        // Yomitan text normalization can consume characters (whitespace,
-        // punctuation) beyond the matched term, leaving no headword whose
-        // source equals the consumed text. Retry with shorter windows so a
-        // valid prefix term (e.g. a character name before a paren) still
-        // tokenizes instead of the position being skipped.
-        let retryLength = Math.min(attempt.matchedLength, windowLength) - 1;
-        while (!attempt.token && retryLength >= 1) {
-          const retry = await findTokenAt(i, retryLength);
-          if (retry.token) {
-            attempt = retry;
-            break;
-          }
-          retryLength = Math.min(retryLength - 1, retry.matchedLength - 1);
-        }
-        if (attempt.token) {
-          tokens.push(attempt.token);
-          i += attempt.matchedLength;
-          continue;
-        }
-        i += String.fromCodePoint(text.codePointAt(i)).length;
-      }
-      return tokens;
-    })();
-  `;
+// Key of the character-name candidate list currently installed in each parser
+// window, so an unchanged list costs nothing per line.
+const yomitanScanNameCandidateKeyByWindow = new WeakMap<BrowserWindow, string>();
+
+async function ensureYomitanScanNameCandidates(
+  parserWindow: BrowserWindow,
+  nameCandidates: { key: string; forms: string[] } | null,
+  logger: LoggerLike,
+): Promise<void> {
+  const installedKey = yomitanScanNameCandidateKeyByWindow.get(parserWindow);
+  const nextKey = nameCandidates?.key ?? '';
+  if (installedKey === nextKey) {
+    return;
+  }
+
+  try {
+    await parserWindow.webContents.executeJavaScript(
+      buildYomitanScanNameCandidatesScript(nameCandidates),
+      true,
+    );
+    yomitanScanNameCandidateKeyByWindow.set(parserWindow, nextKey);
+  } catch (err) {
+    // The scan falls back to checking every position when the list is absent,
+    // so a failed install costs speed, never a missed name.
+    logger.warn?.(
+      'Failed to install Yomitan character-name scan candidates:',
+      (err as Error).message,
+    );
+    yomitanScanNameCandidateKeyByWindow.delete(parserWindow);
+  }
 }
 
 export async function requestYomitanParseResults(
@@ -1635,6 +963,20 @@ export async function requestYomitanParseResults(
   }
 }
 
+// parseText fallback for when the in-window scanner cannot run (script eval
+// failure, unexpected payload). The scanner walk is the primary tokenizer and
+// emits its own filler runs, so this extra full parse only happens on errors.
+async function requestYomitanParseFallbackTokens(
+  text: string,
+  deps: YomitanParserRuntimeDeps,
+  logger: LoggerLike,
+): Promise<YomitanScanToken[] | null> {
+  const parseResults = await requestYomitanParseResults(text, deps, logger);
+  const selectedTokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
+  const parseScanTokens = selectedTokens?.map(toYomitanScanToken) ?? null;
+  return parseScanTokens && parseScanTokens.length > 0 ? parseScanTokens : null;
+}
+
 export async function requestYomitanScanTokens(
   text: string,
   deps: YomitanParserRuntimeDeps,
@@ -1642,6 +984,7 @@ export async function requestYomitanScanTokens(
   options?: {
     includeNameMatchMetadata?: boolean;
     currentCharacterDictionaryMediaId?: number | null;
+    nameCandidates?: { key: string; forms: string[] } | null;
   },
 ): Promise<YomitanScanToken[] | null> {
   const yomitanExt = deps.getYomitanExt();
@@ -1655,10 +998,6 @@ export async function requestYomitanScanTokens(
     return null;
   }
 
-  const parseResults = await requestYomitanParseResults(text, deps, logger);
-  const selectedParseTokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
-  const parseScanTokens = selectedParseTokens?.map(toYomitanScanToken) ?? null;
-
   const metadata = await requestYomitanProfileMetadata(parserWindow, logger);
   const profileIndex = metadata?.profileIndex ?? 0;
   const scanLength = metadata?.scanLength ?? DEFAULT_YOMITAN_SCAN_LENGTH;
@@ -1669,44 +1008,63 @@ export async function requestYomitanScanTokens(
       name.startsWith(CHARACTER_DICTIONARY_TITLE_PREFIX),
     );
 
+  // Candidate name forms let the in-page pre-pass skip positions where no
+  // character name can start. Installed only when it changes (per media), so
+  // the per-line call stays a single tiny script.
+  const nameCandidates = greedyNameScanEnabled ? (options?.nameCandidates ?? null) : null;
+  await ensureYomitanScanNameCandidates(parserWindow, nameCandidates, logger);
+
+  const callScript = buildYomitanScanCallScript({
+    text,
+    profileIndex,
+    scanLength,
+    includeNameMatchMetadata,
+    greedyNameScanEnabled,
+    currentCharacterDictionaryMediaId:
+      typeof options?.currentCharacterDictionaryMediaId === 'number' &&
+      Number.isFinite(options.currentCharacterDictionaryMediaId) &&
+      options.currentCharacterDictionaryMediaId > 0
+        ? Math.floor(options.currentCharacterDictionaryMediaId)
+        : null,
+    dictionaryPriorityByName: metadata?.dictionaryPriorityByName ?? {},
+    dictionaryFrequencyModeByName: metadata?.dictionaryFrequencyModeByName ?? {},
+    cacheEpoch: getYomitanScanCacheEpoch(parserWindow),
+    nameCandidateKey: nameCandidates?.key ?? null,
+  });
+
   try {
-    const rawResult = await parserWindow.webContents.executeJavaScript(
-      buildYomitanScanningScript(
-        text,
-        profileIndex,
-        scanLength,
-        includeNameMatchMetadata,
-        greedyNameScanEnabled,
-        typeof options?.currentCharacterDictionaryMediaId === 'number' &&
-          Number.isFinite(options.currentCharacterDictionaryMediaId) &&
-          options.currentCharacterDictionaryMediaId > 0
-          ? Math.floor(options.currentCharacterDictionaryMediaId)
-          : null,
-        metadata?.dictionaryPriorityByName ?? {},
-        metadata?.dictionaryFrequencyModeByName ?? {},
-      ),
-      true,
-    );
-    if (isScanTokenArray(rawResult)) {
-      if (parseScanTokens && parseScanTokens.length > 0) {
-        return mergeScannerTokensIntoParseTokens(parseScanTokens, rawResult);
+    let rawResult = await parserWindow.webContents.executeJavaScript(callScript, true);
+    if (rawResult === YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL) {
+      // First request for this window, or the page reloaded and dropped the
+      // installed runtime: install and retry once. The candidate list lives in
+      // the same page state, so it has to be reinstalled alongside it.
+      await installYomitanScanRuntime(parserWindow);
+      await ensureYomitanScanNameCandidates(parserWindow, nameCandidates, logger);
+      rawResult = await parserWindow.webContents.executeJavaScript(callScript, true);
+    }
+    // The scanner reports a line where a position ran out of shrinking-window
+    // retries: it stopped short of windows an uncapped ladder would have tried,
+    // so a real term may be sitting in an unparsed run. One parseText for the
+    // line is the bounded way to get the exhaustive answer back (this is the
+    // parse the scanner replaced, and it only runs for these rare lines).
+    if (isObject(rawResult) && rawResult.retryBudgetExhausted === true) {
+      logger.info?.('Yomitan scanner exhausted its retry budget; parsing the line as a fallback.');
+      const fallbackTokens = await requestYomitanParseFallbackTokens(text, deps, logger);
+      if (fallbackTokens) {
+        return fallbackTokens;
       }
-      return rawResult;
+      rawResult = rawResult.tokens;
     }
-    if (Array.isArray(rawResult)) {
-      const selectedTokens = selectYomitanParseTokens(rawResult, () => false, 'headword');
-      return selectedTokens?.map(toYomitanScanToken) ?? null;
+    if (isScanTokenArray(rawResult)) {
+      // Filler-only results carry no dictionary match; keep the historical
+      // contract of returning null so callers fall back to raw text.
+      return rawResult.some((token) => token.isUnparsedRun !== true) ? rawResult : null;
     }
-    if (parseScanTokens && parseScanTokens.length > 0) {
-      return parseScanTokens;
-    }
-    return null;
+    logger.error('Yomitan scanner returned an unexpected payload; using parseText fallback.');
+    return await requestYomitanParseFallbackTokens(text, deps, logger);
   } catch (err) {
-    if (parseScanTokens && parseScanTokens.length > 0) {
-      return parseScanTokens;
-    }
     logger.error('Yomitan scanner request failed:', (err as Error).message);
-    return null;
+    return await requestYomitanParseFallbackTokens(text, deps, logger);
   }
 }
 

src/core/services/tokenizer/yomitan-scan-runtime-script.ts

@@ -0,0 +1,563 @@
+// In-page Yomitan scan runtime: the scan walk that gets installed once per
+// parser window as globalThis.__subminerYomitanScan, plus the tiny per-line
+// call script. Kept separate from the host runtime module so the injected
+// script text (which is data, not executed here) does not dominate that file;
+// the helper bundle it embeds is composed in yomitan-scanning-helpers-script.ts
+// from the yomitan-*-script.ts fragments.
+import { YOMITAN_SCANNING_HELPERS } from './yomitan-scanning-helpers-script';
+
+export { CHARACTER_DICTIONARY_TITLE_PREFIX } from './yomitan-scanning-helpers-script';
+
+export type YomitanFrequencyMode = 'occurrence-based' | 'rank-based';
+
+// Bump whenever the install script below changes so already-loaded parser
+// windows re-install the new scan runtime instead of running the stale one.
+export const YOMITAN_SCAN_RUNTIME_VERSION = 12;
+export const YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL = '__subminer-yomitan-scan-runtime-missing__';
+
+export interface YomitanScanRequestParams {
+  text: string;
+  profileIndex: number;
+  scanLength: number;
+  includeNameMatchMetadata: boolean;
+  greedyNameScanEnabled: boolean;
+  currentCharacterDictionaryMediaId: number | null;
+  dictionaryPriorityByName: Record<string, number>;
+  dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>;
+  cacheEpoch: number;
+  /**
+   * Key of the character-name candidate list installed for the current media,
+   * or null to scan every Japanese position (see the pre-pass prefilter).
+   */
+  nameCandidateKey: string | null;
+}
+
+// Installed once per parser window (and re-installed after in-page reloads):
+// keeps V8 from re-parsing the helper bundle on every subtitle line, and hosts
+// the cross-line termsFind cache. Each subtitle line then only evaluates a tiny
+// call into globalThis.__subminerYomitanScan.
+export const YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT = String.raw`
+  (() => {
+    if (globalThis.__subminerYomitanScanVersion === ${YOMITAN_SCAN_RUNTIME_VERSION}) {
+      return true;
+    }
+    const invoke = (action, params) =>
+      new Promise((resolve, reject) => {
+        chrome.runtime.sendMessage({ action, params }, (response) => {
+          if (chrome.runtime.lastError) {
+            reject(new Error(chrome.runtime.lastError.message));
+            return;
+          }
+          if (!response || typeof response !== "object") {
+            reject(new Error("Invalid response from Yomitan backend"));
+            return;
+          }
+          if (response.error) {
+            reject(new Error(response.error.message || "Yomitan backend error"));
+            return;
+          }
+          resolve(response.result);
+        });
+      });
+    // Cross-line termsFind LRU keyed by profile + substring: subtitle lines
+    // repeat particles and inflections constantly, so most lookups hit here.
+    // Entries hold in-flight promises so concurrent identical lookups dedupe.
+    const termsFindCache = new Map();
+    // Two bounds. The key count keeps the map itself small; the accumulated
+    // dictionary-entry count stands in for retained bytes, because a single
+    // lookup over a common prefix can hold hundreds of entries with their full
+    // glossaries and a key-count cap alone would not bound that.
+    const TERMS_FIND_CACHE_LIMIT = 2000;
+    const TERMS_FIND_CACHE_DICTIONARY_ENTRY_LIMIT = 20000;
+    let termsFindCacheDictionaryEntries = 0;
+    let termsFindCacheEpoch = -1;
+    function dropCachedTermsFind(cacheKey, entry) {
+      if (termsFindCache.get(cacheKey) !== entry) { return; }
+      termsFindCache.delete(cacheKey);
+      termsFindCacheDictionaryEntries -= entry.dictionaryEntryCount;
+    }
+    // Runs on insert and again once a lookup resolves: an entry is only worth
+    // its estimated weight of 1 until then, so a single oversized response
+    // would otherwise sit in the cache forever, over the limit and reused.
+    function evictOverflowingTermsFindEntries() {
+      while (
+        termsFindCache.size > TERMS_FIND_CACHE_LIMIT ||
+        termsFindCacheDictionaryEntries > TERMS_FIND_CACHE_DICTIONARY_ENTRY_LIMIT
+      ) {
+        const oldest = termsFindCache.entries().next().value;
+        if (oldest === undefined) { break; }
+        dropCachedTermsFind(oldest[0], oldest[1]);
+      }
+    }
+    // Classification of a dictionary entry (which dictionaries it came from,
+    // which media ids it mentions) depends only on the entry object, so it is
+    // memoized for as long as that object lives. Entries are shared with the
+    // termsFind cache above, which is what makes this worth keeping: the same
+    // objects come back for every repeated lookup, on every line.
+    const dictionaryEntryNamesCache = new WeakMap();
+    const subMinerMediaIdsCache = new WeakMap();
+    const EMPTY_MEDIA_ID_SET = new Set();
+    // Only blind ladder steps are capped (see the retry loop): those are the
+    // ones that would otherwise degrade into O(scanLength) lookups at a single
+    // position. Steps the backend guides by reporting a shorter consumed length
+    // stay uncapped, so a valid prefix term is still found on lines where
+    // normalization eats a long tail.
+    const MAX_BLIND_SHRINKING_WINDOW_RETRIES = 4;
+    // Character-name candidate forms for the current media, installed
+    // separately from the per-line scan call so the per-line script stays tiny.
+    // Stored raw here; the normalized lookup index is built inside the scan,
+    // where the kana-normalization helper is in scope, and reused by key.
+    let rawNameCandidates = null;
+    let nameCandidateIndex = null;
+    globalThis.__subminerYomitanScanSetNameCandidates = (key, forms) => {
+      if (!key || !Array.isArray(forms) || forms.length === 0) {
+        rawNameCandidates = null;
+        nameCandidateIndex = null;
+        return false;
+      }
+      rawNameCandidates = { key, forms };
+      nameCandidateIndex = null;
+      return true;
+    };
+    globalThis.__subminerYomitanScanVersion = ${YOMITAN_SCAN_RUNTIME_VERSION};
+    globalThis.__subminerYomitanScan = async (scanParams) => {
+      const {
+        text,
+        profileIndex,
+        scanLength,
+        includeNameMatchMetadata,
+        greedyNameScanEnabled,
+        currentCharacterDictionaryMediaId,
+        dictionaryPriorityByName,
+        dictionaryFrequencyModeByName,
+        cacheEpoch,
+        nameCandidateKey
+      } = scanParams;
+      if (cacheEpoch !== termsFindCacheEpoch) {
+        termsFindCache.clear();
+        termsFindCacheDictionaryEntries = 0;
+        termsFindCacheEpoch = cacheEpoch;
+      }
+${YOMITAN_SCANNING_HELPERS}
+      const CAPTION_OPENING_BRACKETS = new Set(["(", "（", "[", "［", "{", "｛", "「", "『", "【", "〈", "《", "≪", "＜", "<"]);
+      function shouldEmitUnparsedRunAsToken(runText) {
+        if (!/[\p{L}\p{N}]/u.test(runText)) { return false; }
+        const firstChar = Array.from(runText.trim())[0];
+        return firstChar !== undefined && !CAPTION_OPENING_BRACKETS.has(firstChar);
+      }
+      function isLookupWorthyCodePoint(codePoint) {
+        if (isCodePointJapanese(codePoint)) { return true; }
+        return /[\p{L}\p{N}]/u.test(String.fromCodePoint(codePoint));
+      }
+      function isKanaOnlyRunText(runText) {
+        const chars = Array.from(runText);
+        return chars.length > 0 && chars.every((char) => isCodePointKana(char.codePointAt(0)));
+      }
+      const details = {matchType: "exact", deinflect: true};
+      const tokens = [];
+      async function termsFindAt(position, windowLength) {
+        const substring = text.substring(position, position + windowLength);
+        const cacheKey = profileIndex + "\u0000" + substring;
+        const cached = termsFindCache.get(cacheKey);
+        if (cached !== undefined) {
+          termsFindCache.delete(cacheKey);
+          termsFindCache.set(cacheKey, cached);
+          return await cached.promise;
+        }
+        // An in-flight lookup counts as one entry until it resolves; the real
+        // weight replaces that estimate once the result is known.
+        const entry = { promise: null, dictionaryEntryCount: 1 };
+        entry.promise = invoke("termsFind", { text: substring, details, optionsContext: { index: profileIndex } })
+          .then((result) => {
+            const resolvedCount =
+              1 + (Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries.length : 0);
+            const isCached = termsFindCache.get(cacheKey) === entry;
+            if (isCached) {
+              termsFindCacheDictionaryEntries += resolvedCount - entry.dictionaryEntryCount;
+            }
+            entry.dictionaryEntryCount = resolvedCount;
+            // The real weight can push the cache over its budget, and a single
+            // response can exceed it on its own, so re-check here.
+            if (isCached) { evictOverflowingTermsFindEntries(); }
+            return result;
+          });
+        termsFindCache.set(cacheKey, entry);
+        termsFindCacheDictionaryEntries += entry.dictionaryEntryCount;
+        evictOverflowingTermsFindEntries();
+        try {
+          return await entry.promise;
+        } catch (error) {
+          dropCachedTermsFind(cacheKey, entry);
+          throw error;
+        }
+      }
+      // Text the walk skips accumulates into unparsed runs, mirroring the
+      // filler chunks the parseText segmentation used to provide: runs stay
+      // hoverable (flagged isUnparsedRun) unless they are punctuation-only or
+      // caption-style asides, and kana continuations of a longer headword
+      // extend the previous token instead.
+      function flushUnparsedRun(runStart, runEnd) {
+        if (runStart === null || runEnd <= runStart) { return; }
+        const runText = text.substring(runStart, runEnd);
+        const previousToken = tokens[tokens.length - 1];
+        if (
+          previousToken &&
+          previousToken.endPos === runStart &&
+          isKanaOnlyRunText(runText) &&
+          typeof previousToken.headword === "string" &&
+          previousToken.headword.length > previousToken.surface.length &&
+          previousToken.headword.startsWith(previousToken.surface + runText)
+        ) {
+          previousToken.surface += runText;
+          // The run is kana-only, so its reading is itself: append it or the
+          // reading stops covering the surface, which disables the known-word
+          // reading fallback (isCompleteReadingForSurface) downstream.
+          previousToken.reading += runText;
+          // The run is kana-only, so its reading is itself: append it or the
+          // reading stops covering the surface, which disables the known-word
+          // reading fallback (isCompleteReadingForSurface) downstream.
+          previousToken.endPos = runEnd;
+          return;
+        }
+        if (!shouldEmitUnparsedRunAsToken(runText)) { return; }
+        tokens.push({
+          surface: runText,
+          reading: "",
+          headword: runText,
+          startPos: runStart,
+          endPos: runEnd,
+          isUnparsedRun: true
+        });
+      }
+      function buildScanToken(position, source, preferredHeadword) {
+        const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
+        const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
+        const tokenPayload = {
+          surface: segments.map((segment) => segment.text).join("") || source,
+          reading: segments.map(getSegmentReadingContribution).join(""),
+          headword: preferredHeadword.term,
+          headwordReading: reading || undefined,
+          startPos: position,
+          endPos: position + source.length,
+          isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
+          frequencyRank:
+            typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
+              ? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
+              : undefined,
+        };
+        if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
+          tokenPayload.wordClasses = preferredHeadword.wordClasses;
+        }
+        return tokenPayload;
+      }
+      // findTokenAt plus the shrinking-window ladder below it: Yomitan text
+      // normalization can consume characters (whitespace, punctuation) beyond
+      // the matched term, leaving no headword whose source equals the consumed
+      // text. Retry with shorter windows so a valid prefix term (e.g. a
+      // character name before a paren) still tokenizes instead of the position
+      // being skipped.
+      // Every window at or above the consumed length repeats the same result,
+      // so the next informative window sits just below it. A lookup that
+      // consumed its whole window reports nothing to aim at, and the step down
+      // from it is a blind guess: only those are budgeted.
+      // The window can run past the end of the line, so blindness is judged
+      // against the text the lookup actually saw.
+      // Set when a position stopped short of windows an uncapped ladder would
+      // still have tried; the line then escalates to parseText at the end.
+      let blindRetryBudgetExhausted = false;
+      async function resolveTokenAt(position, windowLength) {
+        let attempt = await findTokenAt(position, windowLength);
+        const scannedLength = Math.min(windowLength, text.length - position);
+        let retryLength = Math.min(attempt.matchedLength, scannedLength) - 1;
+        let stepIsBlind = attempt.matchedLength >= scannedLength;
+        let blindRetriesRemaining = MAX_BLIND_SHRINKING_WINDOW_RETRIES;
+        while (!attempt.token && retryLength >= 1) {
+          if (stepIsBlind) {
+            if (blindRetriesRemaining <= 0) {
+              blindRetryBudgetExhausted = true;
+              break;
+            }
+            blindRetriesRemaining -= 1;
+          }
+          const retry = await findTokenAt(position, retryLength);
+          if (retry.token) { return retry; }
+          const guidedLength = retry.matchedLength - 1;
+          stepIsBlind = guidedLength >= retryLength - 1;
+          retryLength = Math.min(retryLength - 1, guidedLength);
+        }
+        return attempt;
+      }
+      async function findTokenAt(position, windowLength) {
+        const codePoint = text.codePointAt(position);
+        const character = String.fromCodePoint(codePoint);
+        const result = await termsFindAt(position, windowLength);
+        const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
+        const originalTextLength = typeof result?.originalTextLength === "number" ? result.originalTextLength : 0;
+        if (dictionaryEntries.length === 0 || originalTextLength <= 0 || (originalTextLength === character.length && !isCodePointJapanese(codePoint))) {
+          return { token: null, matchedLength: 0 };
+        }
+        const source = text.substring(position, position + originalTextLength);
+        const preferredHeadword = getPreferredHeadword(
+          dictionaryEntries,
+          source,
+          dictionaryPriorityByName,
+          dictionaryFrequencyModeByName
+        );
+        if (!preferredHeadword || typeof preferredHeadword.term !== "string") {
+          return { token: null, matchedLength: originalTextLength };
+        }
+        return { token: buildScanToken(position, source, preferredHeadword), matchedLength: originalTextLength };
+      }
+      // Kana normalization folds halfwidth katakana one code point to one, so an
+      // unvoiced halfwidth spelling prefix-matches a candidate form like any
+      // other. What it cannot fold is a voiced pair: ｶ + ﾞ stays two characters
+      // where the candidate form carries the single が, so the comparison fails
+      // at that character. That break can sit anywhere inside the name, not
+      // just at its first character (山ｶﾞｸ starts on a kanji), so the bypass is
+      // keyed on the region a candidate could cover, not on how it starts.
+      function isHalfwidthKanaVoicedMarkCodePoint(codePoint) {
+        return codePoint === 0xff9e || codePoint === 0xff9f;
+      }
+      // Build (once per candidate list) a first-character bucket index of the
+      // normalized name forms, so the pre-pass can reject a position with a
+      // single map hit instead of a backend round trip.
+      if (rawNameCandidates && nameCandidateIndex?.key !== rawNameCandidates.key) {
+        const byFirstChar = new Map();
+        for (const form of rawNameCandidates.forms) {
+          const normalized = typeof form === "string" ? convertKatakanaToHiragana(form.trim()) : "";
+          if (!normalized) { continue; }
+          const bucket = byFirstChar.get(normalized[0]);
+          if (bucket) { bucket.push(normalized); } else { byFirstChar.set(normalized[0], [normalized]); }
+        }
+        nameCandidateIndex = byFirstChar.size > 0 ? { key: rawNameCandidates.key, byFirstChar } : null;
+      } else if (!rawNameCandidates) {
+        nameCandidateIndex = null;
+      }
+      // Only meaningful when the installed list matches the media this scan is
+      // for; otherwise fall back to scanning every position.
+      const activeNameCandidateIndex =
+        nameCandidateKey !== null && nameCandidateIndex?.key === nameCandidateKey
+          ? nameCandidateIndex
+          : null;
+      const normalizedText = activeNameCandidateIndex ? convertKatakanaToHiragana(text) : "";
+      // Yomitan collapses emphatic sequences before matching (すっっごーーい →
+      // すごい), so a stretched name still resolves to its entry. Skipping these
+      // characters keeps such spellings candidates; the filter only ever grows
+      // the probe set, so a false positive costs one lookup, never a name.
+      const EMPHATIC_SKIP_CHARS = new Set(["ぁ", "ぃ", "ぅ", "ぇ", "ぉ", "っ", "ゃ", "ゅ", "ょ", "ー"]);
+      function matchesCandidateFormAt(form, position) {
+        let textIndex = position;
+        for (let formIndex = 0; formIndex < form.length; formIndex += 1) {
+          while (
+            textIndex < normalizedText.length &&
+            normalizedText[textIndex] !== form[formIndex] &&
+            EMPHATIC_SKIP_CHARS.has(normalizedText[textIndex])
+          ) {
+            textIndex += 1;
+          }
+          if (normalizedText[textIndex] !== form[formIndex]) { return false; }
+          textIndex += 1;
+        }
+        return true;
+      }
+      // Where the folding gives up, listed once per line. Matching may skip any
+      // number of emphatic characters on its way through a form (山ーーーーーーｶﾞｸ),
+      // so there is no shorter honest bound than the window a name lookup
+      // covers: scanLength. The list is almost always empty, which is what
+      // keeps the check below free on ordinary lines.
+      const halfwidthVoicedMarkPositions = [];
+      if (activeNameCandidateIndex) {
+        for (let index = 0; index < text.length; index += 1) {
+          if (isHalfwidthKanaVoicedMarkCodePoint(text.charCodeAt(index))) {
+            halfwidthVoicedMarkPositions.push(index);
+          }
+        }
+      }
+      function hasHalfwidthVoicedMarkInScanWindow(position) {
+        const end = position + scanLength;
+        for (const markPosition of halfwidthVoicedMarkPositions) {
+          if (markPosition >= position && markPosition < end) { return true; }
+        }
+        return false;
+      }
+      // A name written ｶﾞ... folds to か + ﾞ, so its first character never leads
+      // to the が bucket the candidate form is filed under. Nothing else can
+      // find it, so such a position is always worth a probe.
+      function startsHalfwidthVoicedPair(position, codePoint) {
+        if (codePoint < 0xff66 || codePoint > 0xff9d) { return false; }
+        return isHalfwidthKanaVoicedMarkCodePoint(text.charCodeAt(position + 1));
+      }
+      function couldNameStartAt(position, codePoint) {
+        // Nothing starts with a combining voiced mark, whether or not the
+        // prefilter is active.
+        if (isHalfwidthKanaVoicedMarkCodePoint(codePoint)) { return false; }
+        if (!activeNameCandidateIndex) { return true; }
+        const bucket = activeNameCandidateIndex.byFirstChar.get(normalizedText[position]);
+        if (!bucket) {
+          // No candidate begins with this character, and the window search
+          // below would only ever say yes to positions like this one, so an
+          // unrelated ｶﾞ elsewhere in the line must not drag them in.
+          return startsHalfwidthVoicedPair(position, codePoint);
+        }
+        for (const form of bucket) {
+          if (matchesCandidateFormAt(form, position)) { return true; }
+        }
+        // A candidate does start here but did not match: an unfoldable voiced
+        // pair anywhere in the window is a reason the comparison could not see
+        // it (山ｶﾞｸ, 山ーーーーーーｶﾞｸ), so probe rather than drop the name.
+        return hasHalfwidthVoicedMarkInScanWindow(position);
+      }
+      // Greedy name pre-pass: character-name matches claim their spans before
+      // the left-to-right walk, so a longer generic match starting earlier
+      // (e.g. とヨー → 渡洋) cannot swallow the start of a name (ヨータ).
+      const nameTokens = [];
+      if (greedyNameScanEnabled) {
+        let namePos = 0;
+        while (namePos < text.length) {
+          const codePoint = text.codePointAt(namePos);
+          if (!isCodePointJapanese(codePoint) || !couldNameStartAt(namePos, codePoint)) {
+            namePos += String.fromCodePoint(codePoint).length;
+            continue;
+          }
+          const result = await termsFindAt(namePos, scanLength);
+          const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
+          const textWindow = text.substring(namePos, namePos + scanLength);
+          const nameMatch = findLongestNameMatch(dictionaryEntries, textWindow);
+          // A name only claims its span when no strictly longer generic word
+          // starts at the same position (a character named 空 must not split
+          // 空気). Ties go to the name. Generic matches that start earlier and
+          // overlap the name are still blocked by the reservation.
+          if (
+            !nameMatch ||
+            findLongestGenericMatchLength(dictionaryEntries, textWindow) > nameMatch.sourceLength
+          ) {
+            namePos += String.fromCodePoint(codePoint).length;
+            continue;
+          }
+          const source = text.substring(namePos, namePos + nameMatch.sourceLength);
+          nameTokens.push(buildScanToken(namePos, source, {
+            term: nameMatch.headword.term,
+            reading: nameMatch.headword.reading,
+            wordClasses: normalizeWordClasses(nameMatch.headword),
+            isNameMatch: true,
+            frequencyRank: getBestFrequencyRank(
+              nameMatch.dictionaryEntry,
+              nameMatch.headwordIndex,
+              dictionaryPriorityByName,
+              dictionaryFrequencyModeByName
+            )
+          }));
+          namePos += nameMatch.sourceLength;
+        }
+      }
+      // First reserved name span that a match ending at endPos would leave
+      // half-consumed. Spans the match covers entirely are not returned: those
+      // lose to the longer word instead of splitting it.
+      function findSplitNameToken(startIndex, endPos) {
+        for (let index = startIndex; index < nameTokens.length; index += 1) {
+          const nameToken = nameTokens[index];
+          if (nameToken.startPos >= endPos) { return null; }
+          if (nameToken.endPos > endPos) { return nameToken; }
+        }
+        return null;
+      }
+      let i = 0;
+      let nameIndex = 0;
+      let unparsedRunStart = null;
+      while (i < text.length) {
+        while (nameIndex < nameTokens.length && nameTokens[nameIndex].startPos < i) { nameIndex += 1; }
+        const nextNameToken = nameIndex < nameTokens.length ? nameTokens[nameIndex] : null;
+        if (nextNameToken && nextNameToken.startPos === i) {
+          flushUnparsedRun(unparsedRunStart, i);
+          unparsedRunStart = null;
+          tokens.push(nextNameToken);
+          i = nextNameToken.endPos;
+          nameIndex += 1;
+          continue;
+        }
+        const codePoint = text.codePointAt(i);
+        // Punctuation and whitespace can never start a token: skip the backend
+        // round trip entirely. Latin letters and digits stay lookup-worthy
+        // (terms like Tシャツ start on an ASCII letter).
+        if (!isLookupWorthyCodePoint(codePoint)) {
+          if (unparsedRunStart === null) { unparsedRunStart = i; }
+          i += String.fromCodePoint(codePoint).length;
+          continue;
+        }
+        // A reservation only outranks generic matches that would cut into it.
+        // Look the position up unrestricted first: a generic word that starts
+        // earlier and covers the whole name span (写真 over a character named
+        // 真) is the better reading, so the reservation yields rather than
+        // splitting the word. Only a match that ends inside a name span gets
+        // re-run against a window capped at that span.
+        let attempt = await resolveTokenAt(i, scanLength);
+        if (attempt.token) {
+          const splitNameToken = findSplitNameToken(nameIndex, attempt.token.endPos);
+          if (splitNameToken) {
+            attempt = await resolveTokenAt(i, splitNameToken.startPos - i);
+          }
+        }
+        if (attempt.token) {
+          flushUnparsedRun(unparsedRunStart, i);
+          unparsedRunStart = null;
+          tokens.push(attempt.token);
+          i += attempt.matchedLength;
+          continue;
+        }
+        if (unparsedRunStart === null) { unparsedRunStart = i; }
+        i += String.fromCodePoint(text.codePointAt(i)).length;
+      }
+      flushUnparsedRun(unparsedRunStart, text.length);
+      if (blindRetryBudgetExhausted) {
+        // A position gave up with shorter windows still worth trying. The walk
+        // is the only tokenizer now, so stopping there would leave a real term
+        // as an unparsed run; report it so the host can spend one parseText on
+        // the line instead of letting the ladder run to O(scanLength) lookups.
+        return { tokens, retryBudgetExhausted: true };
+      }
+      return tokens;
+    };
+    return true;
+  })();
+`;
+
+// Installs (or clears) the character-name candidate forms for the current
+// media. Runs only when the list changes, not per line. Passing null restores
+// the exhaustive every-position pre-pass.
+export function buildYomitanScanNameCandidatesScript(
+  nameCandidates: { key: string; forms: string[] } | null,
+): string {
+  if (!nameCandidates) {
+    return `
+    (() => {
+      if (typeof globalThis.__subminerYomitanScanSetNameCandidates !== "function") {
+        return false;
+      }
+      return globalThis.__subminerYomitanScanSetNameCandidates(null, null);
+    })();
+  `;
+  }
+
+  return `
+    (() => {
+      if (typeof globalThis.__subminerYomitanScanSetNameCandidates !== "function") {
+        return false;
+      }
+      return globalThis.__subminerYomitanScanSetNameCandidates(
+        ${JSON.stringify(nameCandidates.key)},
+        ${JSON.stringify(nameCandidates.forms)}
+      );
+    })();
+  `;
+}
+
+export function buildYomitanScanCallScript(params: YomitanScanRequestParams): string {
+  return `
+    (async () => {
+      if (typeof globalThis.__subminerYomitanScan !== "function") {
+        return ${JSON.stringify(YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL)};
+      }
+      return await globalThis.__subminerYomitanScan(${JSON.stringify(params)});
+    })();
+  `;
+}

src/core/services/tokenizer/yomitan-scan-runtime.test.ts

@@ -0,0 +1,304 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { requestYomitanScanTokens } from './yomitan-parser-runtime';
+import {
+  countTermsFindLookups,
+  createNameScanDeps,
+  NAME_SCAN_WORDS,
+} from './yomitan-scan-test-harness';
+
+// Behaviour of the in-page scan runtime around character names and kana:
+// which positions the greedy pre-pass probes, and what the walk makes of
+// halfwidth spellings. Driven end to end through requestYomitanScanTokens
+// because the runtime only exists inside the parser window.
+
+const NAME_SCAN_LINE = 'ミナトはまだ学校にいない';
+
+test('requestYomitanScanTokens skips name pre-pass lookups where no candidate name can start', async () => {
+  const exhaustiveLookups: string[] = [];
+  const exhaustive = await requestYomitanScanTokens(
+    NAME_SCAN_LINE,
+    createNameScanDeps(exhaustiveLookups),
+    { error: () => undefined },
+    { includeNameMatchMetadata: true },
+  );
+
+  const prefilteredLookups: string[] = [];
+  const prefiltered = await requestYomitanScanTokens(
+    NAME_SCAN_LINE,
+    createNameScanDeps(prefilteredLookups),
+    { error: () => undefined },
+    {
+      includeNameMatchMetadata: true,
+      currentCharacterDictionaryMediaId: 1,
+      // Terms and readings the generated dictionary exposes for this media.
+      nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] },
+    },
+  );
+
+  // Same tokenization, including the name match, with fewer round trips.
+  assert.deepEqual(prefiltered, exhaustive);
+  assert.equal(prefiltered?.[0]?.surface, 'ミナト');
+  assert.equal(prefiltered?.[0]?.isNameMatch, true);
+  assert.ok(
+    prefilteredLookups.length < exhaustiveLookups.length,
+    `expected fewer lookups with candidates (${prefilteredLookups.length} vs ${exhaustiveLookups.length})`,
+  );
+  // Mid-token positions are exactly what the pre-pass used to probe (a name can
+  // start mid-token); with candidates they cost nothing, while the main walk's
+  // own token-start lookups are unaffected.
+  assert.ok(countTermsFindLookups(exhaustiveLookups, '校に') > 0);
+  assert.equal(countTermsFindLookups(prefilteredLookups, '校に'), 0);
+});
+
+test('requestYomitanScanTokens matches a katakana name from its kana-normalized candidate form', async () => {
+  const lookups: string[] = [];
+  const result = await requestYomitanScanTokens(
+    NAME_SCAN_LINE,
+    createNameScanDeps(lookups),
+    { error: () => undefined },
+    {
+      includeNameMatchMetadata: true,
+      currentCharacterDictionaryMediaId: 1,
+      // Only the hiragana reading is listed; the katakana surface in the line
+      // must still be found through kana normalization.
+      nameCandidates: { key: 'media-1', forms: ['みなと'] },
+    },
+  );
+
+  assert.equal(result?.[0]?.surface, 'ミナト');
+  assert.equal(result?.[0]?.isNameMatch, true);
+});
+
+// Kana normalization folds halfwidth katakana, so a name written that way does
+// prefix-match a candidate form — but only if the position counts as Japanese
+// in the first place. The generic word here reaches into the name, so only a
+// pre-pass reservation can keep the name whole.
+const HALFWIDTH_NAME_SCAN_WORDS: Array<[string, string, string, boolean]> = [
+  ['ﾈｺ', 'ネコ', 'ねこ', false],
+  ['まだﾐ', 'まだミ', 'まだみ', false],
+  ['まだ', 'まだ', 'まだ', false],
+  ['ﾐﾅﾄ', 'ミナト', 'みなと', true],
+];
+
+test('requestYomitanScanTokens probes halfwidth katakana positions during the name pre-pass', async () => {
+  const lookups: string[] = [];
+  const result = await requestYomitanScanTokens(
+    'ﾈｺまだﾐﾅﾄ',
+    createNameScanDeps(lookups, HALFWIDTH_NAME_SCAN_WORDS),
+    { error: () => undefined },
+    {
+      includeNameMatchMetadata: true,
+      currentCharacterDictionaryMediaId: 1,
+      // Fullwidth forms only, as the generated dictionary stores them.
+      nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] },
+    },
+  );
+
+  assert.equal(countTermsFindLookups(lookups, 'ﾐﾅﾄ'), 1);
+  // ｺ is mid-token, so only the pre-pass would ever look it up, and it matches
+  // no candidate: folding halfwidth made those positions indexable, so they no
+  // longer cost a round trip apiece.
+  assert.equal(countTermsFindLookups(lookups, 'ｺ'), 0);
+  assert.deepEqual(
+    result?.map((token) => token.surface),
+    ['ﾈｺ', 'まだ', 'ﾐﾅﾄ'],
+  );
+  assert.equal(result?.[2]?.isNameMatch, true);
+  // The reading is written the way the fullwidth katakana path writes it
+  // (surface spelling, fullwidth): halfwidth kana is not kana to the known-word
+  // and frequency code downstream, and an empty reading there disables the
+  // reading fallback entirely.
+  assert.equal(result?.[2]?.reading, 'ミナト');
+  assert.equal(result?.[2]?.headwordReading, 'みなと');
+});
+
+test('a voiced halfwidth name still bypasses the candidate prefilter', async () => {
+  const lookups: string[] = [];
+  const result = await requestYomitanScanTokens(
+    'まだｶﾞｸ',
+    createNameScanDeps(lookups, [
+      ['まだｶ', 'まだカ', 'まだか', false],
+      ['まだ', 'まだ', 'まだ', false],
+      ['ｶﾞｸ', 'ガク', 'がく', true],
+    ]),
+    { error: () => undefined },
+    {
+      includeNameMatchMetadata: true,
+      currentCharacterDictionaryMediaId: 1,
+      nameCandidates: { key: 'media-1', forms: ['ガク', 'がく'] },
+    },
+  );
+
+  // ｶ + ﾞ folds to か + ﾞ, which cannot prefix-match が, so the prefilter would
+  // drop this position; the voiced-mark bypass is what keeps the name.
+  assert.deepEqual(
+    result?.map((token) => token.surface),
+    ['まだ', 'ｶﾞｸ'],
+  );
+  assert.equal(result?.[1]?.isNameMatch, true);
+});
+
+test('an unrelated halfwidth voiced word does not restore the exhaustive pre-pass', async () => {
+  const baseline: string[] = [];
+  await requestYomitanScanTokens(
+    NAME_SCAN_LINE,
+    createNameScanDeps(baseline),
+    { error: () => undefined },
+    {
+      includeNameMatchMetadata: true,
+      currentCharacterDictionaryMediaId: 1,
+      nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] },
+    },
+  );
+
+  const withVoicedTail: string[] = [];
+  await requestYomitanScanTokens(
+    `${NAME_SCAN_LINE}ｶﾞ`,
+    createNameScanDeps(withVoicedTail),
+    { error: () => undefined },
+    {
+      includeNameMatchMetadata: true,
+      currentCharacterDictionaryMediaId: 1,
+      nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] },
+    },
+  );
+
+  // Mid-token positions are the ones only the pre-pass would ever probe. A ｶﾞ
+  // anywhere in the line used to drag every position within scanLength of it
+  // back in; now only the voiced pair itself, which the fold cannot index, is
+  // added to what the line already looked up.
+  for (const midTokenPrefix of ['ナト', 'だ学', '校に', 'ない']) {
+    assert.equal(countTermsFindLookups(baseline, midTokenPrefix), 0, midTokenPrefix);
+    assert.equal(countTermsFindLookups(withVoicedTail, midTokenPrefix), 0, midTokenPrefix);
+  }
+  assert.ok(
+    withVoicedTail.length - baseline.length <= 3,
+    `expected the ｶﾞ tail to add only its own lookups, saw ${JSON.stringify(withVoicedTail)}`,
+  );
+});
+
+test('a mixed-width voiced name survives the candidate prefilter', async () => {
+  const lookups: string[] = [];
+  const result = await requestYomitanScanTokens(
+    'まだ山ｶﾞｸ',
+    createNameScanDeps(lookups, [
+      ['まだ山', 'まだ山', 'まだやま', false],
+      ['まだ', 'まだ', 'まだ', false],
+      ['山ｶﾞｸ', '山ガク', 'やまがく', true],
+    ]),
+    { error: () => undefined },
+    {
+      includeNameMatchMetadata: true,
+      currentCharacterDictionaryMediaId: 1,
+      nameCandidates: { key: 'media-1', forms: ['山ガク', 'やまがく'] },
+    },
+  );
+
+  // The name starts on a kanji, so the fold only breaks mid-name: 山ｶﾞｸ
+  // normalizes to 山かﾞく, which still cannot match the candidate 山がく. The
+  // bypass is keyed on the scan window rather than the first character, so the
+  // position is still probed and the generic まだ山 cannot swallow the 山.
+  assert.deepEqual(
+    result?.map((token) => token.surface),
+    ['まだ', '山ｶﾞｸ'],
+  );
+  assert.equal(result?.[1]?.isNameMatch, true);
+});
+
+test('a stretched mixed-width voiced name survives the candidate prefilter', async () => {
+  const lookups: string[] = [];
+  const result = await requestYomitanScanTokens(
+    'まだ山ーーーーーーｶﾞｸ',
+    createNameScanDeps(lookups, [
+      ['まだ山', 'まだ山', 'まだやま', false],
+      ['まだ', 'まだ', 'まだ', false],
+      ['山ーーーーーーｶﾞｸ', '山ガク', 'やまがく', true],
+    ]),
+    { error: () => undefined },
+    {
+      includeNameMatchMetadata: true,
+      currentCharacterDictionaryMediaId: 1,
+      nameCandidates: { key: 'media-1', forms: ['山ガク', 'やまがく'] },
+    },
+  );
+
+  // Matching skips any number of emphatic characters, so the voiced mark that
+  // defeats the fold can sit arbitrarily far into the name: the search for it
+  // has to cover the whole lookup window, not a multiple of the form length.
+  assert.deepEqual(
+    result?.map((token) => token.surface),
+    ['まだ', '山ーーーーーーｶﾞｸ'],
+  );
+  assert.equal(result?.[1]?.isNameMatch, true);
+});
+
+test('halfwidth voiced kana compose into the reading instead of leaving a stray mark', async () => {
+  const lookups: string[] = [];
+  const result = await requestYomitanScanTokens(
+    'ｶﾞｸ ﾊﾟﾝ',
+    createNameScanDeps(lookups, [
+      ['ｶﾞｸ', 'ガク', 'がく', false],
+      ['ﾊﾟﾝ', 'パン', 'ぱん', false],
+    ]),
+    { error: () => undefined },
+    { includeNameMatchMetadata: true },
+  );
+
+  // The name pre-pass runs over every position here (no candidate list), but a
+  // standalone voiced mark can never start a name, so it costs no lookup.
+  assert.equal(countTermsFindLookups(lookups, 'ﾞ'), 0);
+  assert.equal(countTermsFindLookups(lookups, 'ﾟ'), 0);
+  const readings = (result ?? [])
+    .filter((token) => token.isUnparsedRun !== true)
+    .map((token) => [token.surface, token.reading]);
+  assert.deepEqual(readings, [
+    ['ｶﾞｸ', 'ガク'],
+    ['ﾊﾟﾝ', 'パン'],
+  ]);
+});
+
+test('requestYomitanScanTokens falls back to the exhaustive name scan without candidates', async () => {
+  const withoutLookups: string[] = [];
+  const withoutCandidates = await requestYomitanScanTokens(
+    NAME_SCAN_LINE,
+    createNameScanDeps(withoutLookups),
+    { error: () => undefined },
+    { includeNameMatchMetadata: true, currentCharacterDictionaryMediaId: 1, nameCandidates: null },
+  );
+
+  assert.equal(withoutCandidates?.[0]?.isNameMatch, true);
+  // No candidate list means every Japanese position is probed, as before.
+  assert.ok(countTermsFindLookups(withoutLookups, '校に') > 0);
+});
+
+test('requestYomitanScanTokens reinstalls name candidates when the media changes', async () => {
+  const lookups: string[] = [];
+  const deps = createNameScanDeps(lookups);
+
+  // First media's candidates cannot match this line's name.
+  const otherMedia = await requestYomitanScanTokens(
+    NAME_SCAN_LINE,
+    deps,
+    { error: () => undefined },
+    {
+      includeNameMatchMetadata: true,
+      currentCharacterDictionaryMediaId: 2,
+      nameCandidates: { key: 'media-2', forms: ['カズマ'] },
+    },
+  );
+  assert.equal(otherMedia?.[0]?.isNameMatch, undefined);
+
+  const correctMedia = await requestYomitanScanTokens(
+    NAME_SCAN_LINE,
+    deps,
+    { error: () => undefined },
+    {
+      includeNameMatchMetadata: true,
+      currentCharacterDictionaryMediaId: 1,
+      nameCandidates: { key: 'media-1', forms: ['ミナト'] },
+    },
+  );
+  assert.equal(correctMedia?.[0]?.surface, 'ミナト');
+  assert.equal(correctMedia?.[0]?.isNameMatch, true);
+});

src/core/services/tokenizer/yomitan-scan-test-harness.ts

@@ -0,0 +1,166 @@
+// Shared harness for the Yomitan parser-runtime and scan-runtime tests: fake
+// parser-window deps whose injected scripts run in a vm context, plus the
+// backend stubs the scanner tests drive them with. Kept out of the test files
+// so the runtime tests and the in-page scanner tests can share one setup.
+import * as vm from 'node:vm';
+
+export function createDeps(
+  executeJavaScript: (script: string) => Promise<unknown>,
+  options?: {
+    createYomitanExtensionWindow?: (pageName: string) => Promise<unknown>;
+  },
+) {
+  const parserWindow = {
+    isDestroyed: () => false,
+    webContents: {
+      executeJavaScript: async (script: string) => await executeJavaScript(script),
+    },
+  };
+
+  return {
+    getYomitanExt: () => ({ id: 'ext-id' }) as never,
+    getYomitanParserWindow: () => parserWindow as never,
+    setYomitanParserWindow: () => undefined,
+    getYomitanParserReadyPromise: () => null,
+    setYomitanParserReadyPromise: () => undefined,
+    getYomitanParserInitPromise: () => null,
+    setYomitanParserInitPromise: () => undefined,
+    createYomitanExtensionWindow: options?.createYomitanExtensionWindow as never,
+  };
+}
+
+function createYomitanScriptSandbox(handler: (action: string, params: unknown) => unknown) {
+  return {
+    chrome: {
+      runtime: {
+        lastError: null,
+        sendMessage: (
+          payload: { action?: string; params?: unknown },
+          callback: (response: { result?: unknown; error?: { message?: string } }) => void,
+        ) => {
+          try {
+            callback({ result: handler(payload.action ?? '', payload.params) });
+          } catch (error) {
+            callback({ error: { message: (error as Error).message } });
+          }
+        },
+      },
+    },
+    Array,
+    Error,
+    JSON,
+    Map,
+    Math,
+    Number,
+    Object,
+    Promise,
+    RegExp,
+    Set,
+    String,
+  };
+}
+
+export async function runInjectedYomitanScript(
+  script: string,
+  handler: (action: string, params: unknown) => unknown,
+): Promise<unknown> {
+  return await vm.runInNewContext(script, createYomitanScriptSandbox(handler));
+}
+
+// Persistent page context shared across executeJavaScript calls, matching the
+// real parser window: the scan runtime is installed once via
+// globalThis.__subminerYomitanScan and per-line calls reuse it (and its
+// cross-line termsFind cache).
+function createPersistentYomitanScriptRunner(
+  handler: (action: string, params: unknown) => unknown,
+): (script: string) => Promise<unknown> {
+  const context = vm.createContext(createYomitanScriptSandbox(handler));
+  return async (script: string) => await vm.runInContext(script, context);
+}
+
+// Deps whose parser window executes every injected script (profile metadata,
+// scan runtime install, per-line scan calls, parseText fallback) inside one
+// persistent vm context, dispatching backend actions to `handler`.
+export function createScanDeps(
+  handler: (action: string, params: unknown) => unknown,
+  options?: { onScript?: (script: string) => void },
+) {
+  const runScript = createPersistentYomitanScriptRunner(handler);
+  return createDeps(async (script) => {
+    options?.onScript?.(script);
+    return await runScript(script);
+  });
+}
+
+export function countTermsFindLookups(lookups: string[], prefix: string): number {
+  return lookups.filter((lookupText) => lookupText.startsWith(prefix)).length;
+}
+
+// Backend stub for the greedy name pre-pass: one character name (ミナト) in a
+// line of ordinary words, with the SubMiner character dictionary enabled.
+export const NAME_SCAN_WORDS: Array<[string, string, string, boolean]> = [
+  ['ミナト', 'ミナト', 'みなと', true],
+  ['は', 'は', 'は', false],
+  ['まだ', 'まだ', 'まだ', false],
+  ['学校', '学校', 'がっこう', false],
+  ['に', 'に', 'に', false],
+  ['いない', 'いる', 'いる', false],
+];
+
+export function createNameScanDeps(
+  lookups: string[],
+  words: Array<[string, string, string, boolean]> = NAME_SCAN_WORDS,
+) {
+  return createScanDeps((action, params) => {
+    if (action === 'optionsGetFull') {
+      return {
+        profileCurrent: 0,
+        profiles: [
+          {
+            options: {
+              scanning: { length: 40 },
+              dictionaries: [
+                { name: 'JMdict', enabled: true, id: 0 },
+                {
+                  name: 'SubMiner Character Dictionary (AniList 1)',
+                  enabled: true,
+                  id: 1,
+                },
+              ],
+            },
+          },
+        ],
+      };
+    }
+    if (action === 'getDictionaryInfo') {
+      return [];
+    }
+    if (action !== 'termsFind') {
+      throw new Error(`unexpected action: ${action}`);
+    }
+    const text = (params as { text?: string } | undefined)?.text ?? '';
+    lookups.push(text);
+    for (const [surface, term, reading, isName] of words) {
+      if (text.startsWith(surface)) {
+        return {
+          originalTextLength: surface.length,
+          dictionaryEntries: [
+            {
+              headwords: [
+                {
+                  term,
+                  reading,
+                  sources: [{ originalText: surface, isPrimary: true, matchType: 'exact' }],
+                },
+              ],
+              definitions: [
+                { dictionary: isName ? 'SubMiner Character Dictionary (AniList 1)' : 'JMdict' },
+              ],
+            },
+          ],
+        };
+      }
+    }
+    return { originalTextLength: 0, dictionaryEntries: [] };
+  });
+}

src/core/services/tokenizer/yomitan-scanning-helpers-script.ts

@@ -0,0 +1,21 @@
+// Helper bundle for the in-page Yomitan scan runtime, composed from the
+// fragments below. Injected as text into the parser window by
+// yomitan-scan-runtime-script.ts, so it is data here, not code this process
+// runs. The fragments are concatenated into a single function body and share
+// one lexical scope: every function in them is hoisted, but the constants are
+// not, so kana stays first — the later fragments read its ranges as they run.
+import { YOMITAN_DICTIONARY_CLASSIFICATION_HELPERS } from './yomitan-dictionary-classification-script';
+import { YOMITAN_FREQUENCY_HELPERS } from './yomitan-frequency-script';
+import { YOMITAN_FURIGANA_HELPERS } from './yomitan-furigana-script';
+import { YOMITAN_KANA_HELPERS } from './yomitan-kana-script';
+import { YOMITAN_MATCH_SELECTION_HELPERS } from './yomitan-match-selection-script';
+
+export { CHARACTER_DICTIONARY_TITLE_PREFIX } from './character-dictionary-title';
+
+export const YOMITAN_SCANNING_HELPERS = [
+  YOMITAN_KANA_HELPERS,
+  YOMITAN_FURIGANA_HELPERS,
+  YOMITAN_FREQUENCY_HELPERS,
+  YOMITAN_DICTIONARY_CLASSIFICATION_HELPERS,
+  YOMITAN_MATCH_SELECTION_HELPERS,
+].join('\n');

src/core/text/han-code-points.test.ts

@@ -0,0 +1,54 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { HAN_CODE_POINT_RANGES, HAN_REGEXP_CLASS_BODY, isHanCodePoint } from './han-code-points';
+
+test('every range boundary is inside the table', () => {
+  for (const [start, end] of HAN_CODE_POINT_RANGES) {
+    for (const codePoint of [start, end]) {
+      assert.ok(isHanCodePoint(codePoint), `expected U+${codePoint.toString(16)} to be Han`);
+    }
+  }
+
+  // Extension J (Unicode 17) and the Compatibility blocks are the ones a
+  // BMP-only table used to miss.
+  assert.ok(isHanCodePoint(0x323b0));
+  assert.ok(isHanCodePoint(0x33479));
+  assert.ok(isHanCodePoint(0xf900));
+  assert.ok(isHanCodePoint(0x2f800));
+});
+
+test('no unified ideograph the runtime knows about falls outside the table', () => {
+  // One direction only: a runtime with older Unicode data simply checks fewer
+  // code points, where asserting the reverse would fail on Extension J.
+  const unifiedIdeograph = /\p{Unified_Ideograph}/u;
+
+  for (let codePoint = 0x3000; codePoint <= 0x40000; codePoint += 1) {
+    if (unifiedIdeograph.test(String.fromCodePoint(codePoint))) {
+      assert.ok(
+        isHanCodePoint(codePoint),
+        `expected unified ideograph U+${codePoint.toString(16)} to be in the table`,
+      );
+    }
+  }
+});
+
+test('code points just outside the table are rejected', () => {
+  for (const codePoint of [0x33ff, 0x4dc0, 0xa000, 0x1f000, 0x3347a]) {
+    assert.equal(
+      isHanCodePoint(codePoint),
+      false,
+      `expected U+${codePoint.toString(16)} not to be Han`,
+    );
+  }
+});
+
+test('the regexp class body matches the same code points as the predicate', () => {
+  const classRegExp = new RegExp(`^[${HAN_REGEXP_CLASS_BODY}]$`, 'u');
+
+  for (const codePoint of [0x3400, 0x4e00, 0x9fff, 0xf900, 0x20000, 0x323b0, 0x33479]) {
+    assert.match(String.fromCodePoint(codePoint), classRegExp);
+  }
+  for (const codePoint of [0x3040, 0x30ff, 0x33fa, 0x3347a]) {
+    assert.doesNotMatch(String.fromCodePoint(codePoint), classRegExp);
+  }
+});

src/core/text/han-code-points.ts

@@ -0,0 +1,31 @@
+// Single source of truth for "this code point is a Han character", shared by
+// the main-process character dictionary and the in-page Yomitan scan runtime.
+// The two used to carry separate range lists, and they drifted: a name written
+// with a supplementary-plane kanji could enter the generated dictionary while
+// the scanner's greedy name pre-pass refused to probe the position.
+//
+// Ranges rather than \p{Script=Han}: the scan walk tests one code point per
+// character of every subtitle line, where an integer compare beats building a
+// string for a regex, and the script is injected as text into a page where a
+// shared helper cannot be imported.
+export const HAN_CODE_POINT_RANGES: ReadonlyArray<readonly [number, number]> = [
+  [0x3400, 0x4dbf], // Extension A
+  [0x4e00, 0x9fff], // CJK Unified Ideographs
+  [0xf900, 0xfaff], // Compatibility Ideographs
+  [0x20000, 0x2a6df], // Extension B
+  [0x2a700, 0x2ebef], // Extensions C-F
+  [0x2ebf0, 0x2ee5f], // Extension I
+  [0x2f800, 0x2fa1f], // Compatibility Ideographs Supplement
+  [0x30000, 0x3134f], // Extension G
+  [0x31350, 0x323af], // Extension H
+  [0x323b0, 0x33479], // Extension J (Unicode 17)
+];
+
+export function isHanCodePoint(codePoint: number): boolean {
+  return HAN_CODE_POINT_RANGES.some(([start, end]) => codePoint >= start && codePoint <= end);
+}
+
+/** The same ranges as a regular expression character class body (needs the `u` flag). */
+export const HAN_REGEXP_CLASS_BODY = HAN_CODE_POINT_RANGES.map(
+  ([start, end]) => `\\u{${start.toString(16)}}-\\u{${end.toString(16)}}`,
+).join('');