perf(tokenizer): single-pass Yomitan scan with install-once runtime and cross-line cache

- drop the duplicate parseText full parse per line; the termsFind scanner walk
  is now authoritative and emits its own unparsed filler runs (parseText kept
  only as error fallback)
- install scan helpers once per parser window (__subminerYomitanScan) instead
  of re-shipping ~500 lines of script per subtitle line
- persist termsFind results across lines in a window-scoped LRU keyed by
  substring, invalidated via a cache epoch on dictionary/settings changes
- skip lookups at punctuation/whitespace positions and cap the shrinking-window
  retry ladder at 4 lookups per position
- build tokenizer runtime deps once (JLPT lookup cache never hit before; mecab
  availability check ran per line)
- stop restarting the prefetch run on every subtitle change; resume prefetch
  only after the tokenized payload lands, not on provisional raw emits
- add per-stage debug timings (scanMs/mecabMs/frequencyMs/annotateMs)

src/core/services/tokenizer.test.ts

@@ -2934,44 +2934,12 @@ test('tokenizeSubtitle preserves Yomitan compound token when MeCab components ar
                 return [];
               }
 
-              if (script.includes('parseText')) {
-                return [
-                  {
-                    source: 'scanning-parser',
-                    index: 0,
-                    content: [
-                      [
-                        {
-                          text: '取り組んで',
-                          reading: 'とりくんで',
-                          headwords: [[{ term: '取り組む' }]],
-                        },
-                      ],
-                      [
-                        {
-                          text: 'もらいます',
-                          reading: 'もらいます',
-                          headwords: [[{ term: 'もらう' }]],
-                        },
-                      ],
-                    ],
-                  },
-                ];
-              }
-
               return [
                 {
-                  surface: '取り',
-                  reading: 'とり',
-                  headword: '取る',
+                  surface: '取り組んで',
+                  reading: 'とりくんで',
+                  headword: '取り組む',
                   startPos: 0,
-                  endPos: 2,
-                },
-                {
-                  surface: '組んで',
-                  reading: 'くんで',
-                  headword: '組む',
-                  startPos: 2,
                   endPos: 5,
                 },
                 {

src/core/services/tokenizer.ts

@@ -716,15 +716,29 @@ function getAnnotationOptions(deps: TokenizerServiceDeps): TokenizerAnnotationOp
   };
 }
 
+// Per-line stage durations for the pipeline debug log; every field is filled in
+// by the stage that awaits the corresponding work.
+interface TokenizationStageTimings {
+  scanMs?: number;
+  mecabMs?: number;
+  frequencyMs?: number;
+  annotateMs?: number;
+}
+
 async function parseWithYomitanInternalParser(
   text: string,
   deps: TokenizerServiceDeps,
   options: TokenizerAnnotationOptions,
+  stageTimings?: TokenizationStageTimings,
 ): Promise<MergedToken[] | null> {
+  const scanStartedAtMs = Date.now();
   const selectedTokens = await requestYomitanScanTokens(text, deps, logger, {
     includeNameMatchMetadata: options.nameMatchEnabled,
     currentCharacterDictionaryMediaId: deps.getCurrentCharacterDictionaryMediaId?.() ?? null,
   });
+  if (stageTimings) {
+    stageTimings.scanMs = Date.now() - scanStartedAtMs;
+  }
   if (!selectedTokens || selectedTokens.length === 0) {
     return null;
   }
@@ -757,6 +771,7 @@ async function parseWithYomitanInternalParser(
 
   const frequencyRankPromise: Promise<YomitanFrequencyIndex> = options.frequencyEnabled
     ? (async () => {
+        const frequencyStartedAtMs = Date.now();
         const frequencyMatchMode = options.frequencyMatchMode;
         const termReadingList = buildYomitanFrequencyTermReadingList(
           normalizedSelectedTokens,
@@ -767,12 +782,17 @@ async function parseWithYomitanInternalParser(
           deps,
           logger,
         );
-        return buildYomitanFrequencyIndex(yomitanFrequencies);
+        const frequencyIndex = buildYomitanFrequencyIndex(yomitanFrequencies);
+        if (stageTimings) {
+          stageTimings.frequencyMs = Date.now() - frequencyStartedAtMs;
+        }
+        return frequencyIndex;
       })()
     : Promise.resolve({ byPair: new Map(), byTerm: new Map() });
 
   const mecabEnrichmentPromise: Promise<MergedToken[]> = needsMecabPosEnrichment(options)
     ? (async () => {
+        const mecabStartedAtMs = Date.now();
         try {
           const mecabTokens = await deps.tokenizeWithMecab(text);
           const enrichTokensWithMecab = deps.enrichTokensWithMecab ?? enrichTokensWithMecabAsync;
@@ -786,6 +806,10 @@ async function parseWithYomitanInternalParser(
             `textLength=${text.length}`,
           );
           return normalizedSelectedTokens;
+        } finally {
+          if (stageTimings) {
+            stageTimings.mecabMs = Date.now() - mecabStartedAtMs;
+          }
         }
       })()
     : Promise.resolve(normalizedSelectedTokens);
@@ -876,15 +900,35 @@ export async function tokenizeSubtitle(
   const annotationOptions = getAnnotationOptions(deps);
   annotationOptions.sourceText = tokenizeText;
 
-  const yomitanTokens = await parseWithYomitanInternalParser(tokenizeText, deps, annotationOptions);
+  const stageTimings: TokenizationStageTimings = {};
+  const startedAtMs = Date.now();
+  const logStageTimings = (tokenCount: number): void => {
+    logger.debug(
+      `Subtitle tokenization stages; textLength=${tokenizeText.length}, tokenCount=${tokenCount}, ` +
+        `scanMs=${stageTimings.scanMs ?? '-'}, mecabMs=${stageTimings.mecabMs ?? '-'}, ` +
+        `frequencyMs=${stageTimings.frequencyMs ?? '-'}, annotateMs=${stageTimings.annotateMs ?? '-'}, ` +
+        `totalMs=${Date.now() - startedAtMs}`,
+    );
+  };
+
+  const yomitanTokens = await parseWithYomitanInternalParser(
+    tokenizeText,
+    deps,
+    annotationOptions,
+    stageTimings,
+  );
   if (yomitanTokens && yomitanTokens.length > 0) {
+    const annotateStartedAtMs = Date.now();
     const annotatedTokens = await applyAnnotationStage(yomitanTokens, deps, annotationOptions);
     const renderedTokens = applyCharacterNameImages(annotatedTokens, deps, annotationOptions);
+    stageTimings.annotateMs = Date.now() - annotateStartedAtMs;
+    logStageTimings(renderedTokens.length);
     return {
       text: displayText,
       tokens: renderedTokens.length > 0 ? renderedTokens : null,
     };
   }
 
+  logStageTimings(0);
   return { text: displayText, tokens: null };
 }

src/core/services/tokenizer/golden-corpus-harness.ts

@@ -366,8 +366,11 @@ export function createReplayMessageStore(messages: GoldenRecordedMessage[]): Rep
   };
 }
 
-async function runInjectedScriptInVm(script: string, store: ReplayMessageStore): Promise<unknown> {
-  return await vm.runInNewContext(script, {
+// One persistent context per fixture, matching the real parser window: the
+// scan runtime installs itself once into globalThis and later per-line call
+// scripts reuse it.
+function createInjectedScriptVm(store: ReplayMessageStore): (script: string) => Promise<unknown> {
+  const context = vm.createContext({
     chrome: {
       runtime: {
         lastError: null,
@@ -393,6 +396,7 @@ async function runInjectedScriptInVm(script: string, store: ReplayMessageStore):
     Set,
     String,
   });
+  return async (script: string) => await vm.runInContext(script, context);
 }
 
 export function createReplayTokenizerDeps(fixture: GoldenFixture): TokenizerServiceDeps {
@@ -400,13 +404,14 @@ export function createReplayTokenizerDeps(fixture: GoldenFixture): TokenizerServ
   const scriptResults = new Map(
     fixture.recording.scripts.map((entry) => [entry.sha256, entry] as const),
   );
+  const runInjectedScriptInVm = createInjectedScriptVm(store);
 
   const parserWindow = {
     isDestroyed: () => false,
     webContents: {
       executeJavaScript: async (script: string) => {
         try {
-          return await runInjectedScriptInVm(script, store);
+          return await runInjectedScriptInVm(script);
         } catch (vmError) {
           const recorded = scriptResults.get(hashInjectedScript(script));
           if (recorded) {

src/core/services/tokenizer/yomitan-parser-runtime.ts

@@ -81,6 +81,13 @@ const yomitanFrequencyCacheByWindow = new WeakMap<
   BrowserWindow,
   Map<string, YomitanTermFrequency[]>
 >();
+// Epoch passed with every scan request; the in-window termsFind cache clears
+// itself when the epoch changes (dictionary imports, settings changes).
+const yomitanScanCacheEpochByWindow = new WeakMap<BrowserWindow, number>();
+
+function getYomitanScanCacheEpoch(window: BrowserWindow): number {
+  return yomitanScanCacheEpochByWindow.get(window) ?? 0;
+}
 
 function isObject(value: unknown): value is Record<string, unknown> {
   return Boolean(value && typeof value === 'object');
@@ -99,6 +106,7 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
         typeof entry.startPos === 'number' &&
         typeof entry.endPos === 'number' &&
         (entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean') &&
+        (entry.isUnparsedRun === undefined || typeof entry.isUnparsedRun === 'boolean') &&
         (entry.frequencyRank === undefined || typeof entry.frequencyRank === 'number') &&
         (entry.wordClasses === undefined ||
           (Array.isArray(entry.wordClasses) &&
@@ -107,13 +115,9 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
   );
 }
 
-function scanTokenSpanKey(token: YomitanScanToken): string {
-  return `${token.startPos}:${token.endPos}:${token.surface}`;
-}
-
 // Maps a parse-selected token to the scanner-token shape carried out of the
-// parser runtime. Shared by both selectYomitanParseTokens fallback paths so the
-// projected fields stay in sync as the shape changes.
+// parser runtime, used by the parseText fallback path when the in-window
+// scanner is unavailable.
 function toYomitanScanToken(token: {
   surface: string;
   reading: string;
@@ -132,66 +136,6 @@ function toYomitanScanToken(token: {
   };
 }
 
-// parseText segmentation is authoritative (it emits filler chunks for text the
-// termsFind scanner skips), but only the termsFind scanner carries annotation
-// metadata (isNameMatch, frequencyRank, headwordReading, wordClasses). Graft
-// scanner tokens onto the parseText segmentation per matching span so one
-// unmatched chunk degrades only itself instead of dropping the whole line's
-// metadata.
-//
-// Exception: character-name tokens. The greedy name scan can re-segment text
-// around a name (e.g. とヨータ → と + ヨータ instead of とヨー + タ), so
-// parseText segmentation cannot be authoritative there. Each name span is
-// expanded until it aligns with token boundaries in both segmentations, then
-// the parse tokens inside are replaced with the scanner tokens.
-function mergeScannerTokensIntoParseTokens(
-  parseScanTokens: YomitanScanToken[],
-  scannerTokens: YomitanScanToken[],
-): YomitanScanToken[] {
-  const scannerTokensBySpan = new Map<string, YomitanScanToken>();
-  for (const token of scannerTokens) {
-    scannerTokensBySpan.set(scanTokenSpanKey(token), token);
-  }
-  const graftedTokens = parseScanTokens.map(
-    (token) => scannerTokensBySpan.get(scanTokenSpanKey(token)) ?? token,
-  );
-
-  const nameTokens = scannerTokens.filter((token) => token.isNameMatch === true);
-  if (nameTokens.length === 0) {
-    return graftedTokens;
-  }
-
-  const regions = nameTokens.map((token) => ({ start: token.startPos, end: token.endPos }));
-  const allTokens = [...parseScanTokens, ...scannerTokens];
-  let expanded = true;
-  while (expanded) {
-    expanded = false;
-    for (const region of regions) {
-      for (const token of allTokens) {
-        const overlaps = token.startPos < region.end && token.endPos > region.start;
-        const extendsBeyond = token.startPos < region.start || token.endPos > region.end;
-        if (overlaps && extendsBeyond) {
-          region.start = Math.min(region.start, token.startPos);
-          region.end = Math.max(region.end, token.endPos);
-          expanded = true;
-        }
-      }
-    }
-  }
-
-  const isInsideNameRegion = (token: YomitanScanToken): boolean =>
-    regions.some((region) => token.startPos >= region.start && token.endPos <= region.end);
-
-  const merged = graftedTokens.filter((token) => !isInsideNameRegion(token));
-  for (const token of scannerTokens) {
-    if (isInsideNameRegion(token)) {
-      merged.push(token);
-    }
-  }
-  merged.sort((a, b) => a.startPos - b.startPos || a.endPos - b.endPos);
-  return merged;
-}
-
 function makeTermReadingCacheKey(term: string, reading: string | null): string {
   return `${term}\u0000${reading ?? ''}`;
 }
@@ -208,6 +152,7 @@ function getWindowFrequencyCache(window: BrowserWindow): Map<string, YomitanTerm
 function clearWindowCaches(window: BrowserWindow): void {
   yomitanProfileMetadataByWindow.delete(window);
   yomitanFrequencyCacheByWindow.delete(window);
+  yomitanScanCacheEpochByWindow.set(window, getYomitanScanCacheEpoch(window) + 1);
 }
 export function clearYomitanParserCachesForWindow(window: BrowserWindow): void {
   clearWindowCaches(window);
@@ -704,6 +649,10 @@ async function ensureYomitanParserWindow(
       if (readyPromise) {
         await readyPromise;
       }
+      // Eagerly install the scan runtime so the first subtitle line does not
+      // pay the install round trip; failures fall back to the per-request
+      // install-and-retry path.
+      await installYomitanScanRuntime(parserWindow).catch(() => {});
 
       return true;
     } catch (err) {
@@ -1362,58 +1311,144 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
       }
 `;
 
-function buildYomitanScanningScript(
-  text: string,
-  profileIndex: number,
-  scanLength: number,
-  includeNameMatchMetadata: boolean,
-  greedyNameScanEnabled: boolean,
-  currentCharacterDictionaryMediaId: number | null,
-  dictionaryPriorityByName: Record<string, number>,
-  dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>,
-): string {
-  return `
-    (async () => {
-      const invoke = (action, params) =>
-        new Promise((resolve, reject) => {
-          chrome.runtime.sendMessage({ action, params }, (response) => {
-            if (chrome.runtime.lastError) {
-              reject(new Error(chrome.runtime.lastError.message));
-              return;
-            }
-            if (!response || typeof response !== "object") {
-              reject(new Error("Invalid response from Yomitan backend"));
-              return;
-            }
-            if (response.error) {
-              reject(new Error(response.error.message || "Yomitan backend error"));
-              return;
-            }
-            resolve(response.result);
-          });
+// Bump whenever the install script below changes so already-loaded parser
+// windows re-install the new scan runtime instead of running the stale one.
+const YOMITAN_SCAN_RUNTIME_VERSION = 1;
+const YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL = '__subminer-yomitan-scan-runtime-missing__';
+
+interface YomitanScanRequestParams {
+  text: string;
+  profileIndex: number;
+  scanLength: number;
+  includeNameMatchMetadata: boolean;
+  greedyNameScanEnabled: boolean;
+  currentCharacterDictionaryMediaId: number | null;
+  dictionaryPriorityByName: Record<string, number>;
+  dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>;
+  cacheEpoch: number;
+}
+
+// Installed once per parser window (and re-installed after in-page reloads):
+// keeps V8 from re-parsing the helper bundle on every subtitle line, and hosts
+// the cross-line termsFind cache. Each subtitle line then only evaluates a tiny
+// call into globalThis.__subminerYomitanScan.
+const YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT = String.raw`
+  (() => {
+    if (globalThis.__subminerYomitanScanVersion === ${YOMITAN_SCAN_RUNTIME_VERSION}) {
+      return true;
+    }
+    const invoke = (action, params) =>
+      new Promise((resolve, reject) => {
+        chrome.runtime.sendMessage({ action, params }, (response) => {
+          if (chrome.runtime.lastError) {
+            reject(new Error(chrome.runtime.lastError.message));
+            return;
+          }
+          if (!response || typeof response !== "object") {
+            reject(new Error("Invalid response from Yomitan backend"));
+            return;
+          }
+          if (response.error) {
+            reject(new Error(response.error.message || "Yomitan backend error"));
+            return;
+          }
+          resolve(response.result);
         });
+      });
+    // Cross-line termsFind LRU keyed by profile + substring: subtitle lines
+    // repeat particles and inflections constantly, so most lookups hit here.
+    // Entries hold in-flight promises so concurrent identical lookups dedupe.
+    const termsFindCache = new Map();
+    const TERMS_FIND_CACHE_LIMIT = 2000;
+    let termsFindCacheEpoch = -1;
+    const MAX_SHRINKING_WINDOW_RETRY_LOOKUPS = 4;
+    globalThis.__subminerYomitanScanVersion = ${YOMITAN_SCAN_RUNTIME_VERSION};
+    globalThis.__subminerYomitanScan = async (scanParams) => {
+      const {
+        text,
+        profileIndex,
+        scanLength,
+        includeNameMatchMetadata,
+        greedyNameScanEnabled,
+        currentCharacterDictionaryMediaId,
+        dictionaryPriorityByName,
+        dictionaryFrequencyModeByName,
+        cacheEpoch
+      } = scanParams;
+      if (cacheEpoch !== termsFindCacheEpoch) {
+        termsFindCache.clear();
+        termsFindCacheEpoch = cacheEpoch;
+      }
 ${YOMITAN_SCANNING_HELPERS}
-      const includeNameMatchMetadata = ${includeNameMatchMetadata ? 'true' : 'false'};
-      const greedyNameScanEnabled = ${greedyNameScanEnabled ? 'true' : 'false'};
-      const currentCharacterDictionaryMediaId = ${
-        currentCharacterDictionaryMediaId !== null
-          ? String(currentCharacterDictionaryMediaId)
-          : 'null'
-      };
-      const dictionaryPriorityByName = ${JSON.stringify(dictionaryPriorityByName)};
-      const dictionaryFrequencyModeByName = ${JSON.stringify(dictionaryFrequencyModeByName)};
-      const text = ${JSON.stringify(text)};
+      const CAPTION_OPENING_BRACKETS = new Set(["(", "（", "[", "［", "{", "｛", "「", "『", "【", "〈", "《", "≪", "＜", "<"]);
+      function shouldEmitUnparsedRunAsToken(runText) {
+        if (!/[\p{L}\p{N}]/u.test(runText)) { return false; }
+        const firstChar = Array.from(runText.trim())[0];
+        return firstChar !== undefined && !CAPTION_OPENING_BRACKETS.has(firstChar);
+      }
+      function isLookupWorthyCodePoint(codePoint) {
+        if (isCodePointJapanese(codePoint)) { return true; }
+        return /[\p{L}\p{N}]/u.test(String.fromCodePoint(codePoint));
+      }
+      function isKanaOnlyRunText(runText) {
+        const chars = Array.from(runText);
+        return chars.length > 0 && chars.every((char) => isCodePointKana(char.codePointAt(0)));
+      }
       const details = {matchType: "exact", deinflect: true};
       const tokens = [];
-      const termsFindCache = new Map();
       async function termsFindAt(position, windowLength) {
-        const cacheKey = position + ":" + windowLength;
-        const cached = termsFindCache.get(cacheKey);
-        if (cached) { return cached; }
         const substring = text.substring(position, position + windowLength);
-        const result = await invoke("termsFind", { text: substring, details, optionsContext: { index: ${profileIndex} } });
-        termsFindCache.set(cacheKey, result);
-        return result;
+        const cacheKey = profileIndex + "" + substring;
+        const cached = termsFindCache.get(cacheKey);
+        if (cached !== undefined) {
+          termsFindCache.delete(cacheKey);
+          termsFindCache.set(cacheKey, cached);
+          return await cached;
+        }
+        const pending = invoke("termsFind", { text: substring, details, optionsContext: { index: profileIndex } });
+        termsFindCache.set(cacheKey, pending);
+        while (termsFindCache.size > TERMS_FIND_CACHE_LIMIT) {
+          const oldestKey = termsFindCache.keys().next().value;
+          if (oldestKey === undefined) { break; }
+          termsFindCache.delete(oldestKey);
+        }
+        try {
+          return await pending;
+        } catch (error) {
+          termsFindCache.delete(cacheKey);
+          throw error;
+        }
+      }
+      // Text the walk skips accumulates into unparsed runs, mirroring the
+      // filler chunks the parseText segmentation used to provide: runs stay
+      // hoverable (flagged isUnparsedRun) unless they are punctuation-only or
+      // caption-style asides, and kana continuations of a longer headword
+      // extend the previous token instead.
+      function flushUnparsedRun(runStart, runEnd) {
+        if (runStart === null || runEnd <= runStart) { return; }
+        const runText = text.substring(runStart, runEnd);
+        const previousToken = tokens[tokens.length - 1];
+        if (
+          previousToken &&
+          previousToken.endPos === runStart &&
+          isKanaOnlyRunText(runText) &&
+          typeof previousToken.headword === "string" &&
+          previousToken.headword.length > previousToken.surface.length &&
+          previousToken.headword.startsWith(previousToken.surface + runText)
+        ) {
+          previousToken.surface += runText;
+          previousToken.endPos = runEnd;
+          return;
+        }
+        if (!shouldEmitUnparsedRunAsToken(runText)) { return; }
+        tokens.push({
+          surface: runText,
+          reading: "",
+          headword: runText,
+          startPos: runStart,
+          endPos: runEnd,
+          isUnparsedRun: true
+        });
       }
       function buildScanToken(position, source, preferredHeadword) {
         const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
@@ -1469,9 +1504,9 @@ ${YOMITAN_SCANNING_HELPERS}
             namePos += String.fromCodePoint(codePoint).length;
             continue;
           }
-          const result = await termsFindAt(namePos, ${scanLength});
+          const result = await termsFindAt(namePos, scanLength);
           const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
-          const textWindow = text.substring(namePos, namePos + ${scanLength});
+          const textWindow = text.substring(namePos, namePos + scanLength);
           const nameMatch = findLongestNameMatch(dictionaryEntries, textWindow);
           // A name only claims its span when no strictly longer generic word
           // starts at the same position (a character named 空 must not split
@@ -1502,26 +1537,42 @@ ${YOMITAN_SCANNING_HELPERS}
       }
       let i = 0;
       let nameIndex = 0;
+      let unparsedRunStart = null;
       while (i < text.length) {
         while (nameIndex < nameTokens.length && nameTokens[nameIndex].startPos < i) { nameIndex += 1; }
         const nextNameToken = nameIndex < nameTokens.length ? nameTokens[nameIndex] : null;
         if (nextNameToken && nextNameToken.startPos === i) {
+          flushUnparsedRun(unparsedRunStart, i);
+          unparsedRunStart = null;
           tokens.push(nextNameToken);
           i = nextNameToken.endPos;
           nameIndex += 1;
           continue;
         }
+        const codePoint = text.codePointAt(i);
+        // Punctuation and whitespace can never start a token: skip the backend
+        // round trip entirely. Latin letters and digits stay lookup-worthy
+        // (terms like Tシャツ start on an ASCII letter).
+        if (!isLookupWorthyCodePoint(codePoint)) {
+          if (unparsedRunStart === null) { unparsedRunStart = i; }
+          i += String.fromCodePoint(codePoint).length;
+          continue;
+        }
         // Cap the window at the next reserved name span so a generic match
         // cannot consume into it.
-        const windowLength = nextNameToken ? Math.min(${scanLength}, nextNameToken.startPos - i) : ${scanLength};
+        const windowLength = nextNameToken ? Math.min(scanLength, nextNameToken.startPos - i) : scanLength;
         let attempt = await findTokenAt(i, windowLength);
         // Yomitan text normalization can consume characters (whitespace,
         // punctuation) beyond the matched term, leaving no headword whose
         // source equals the consumed text. Retry with shorter windows so a
         // valid prefix term (e.g. a character name before a paren) still
-        // tokenizes instead of the position being skipped.
+        // tokenizes instead of the position being skipped. The ladder is
+        // capped: without a cap it degrades to O(scanLength) lookups at a
+        // single position.
         let retryLength = Math.min(attempt.matchedLength, windowLength) - 1;
-        while (!attempt.token && retryLength >= 1) {
+        let retryLookupsRemaining = MAX_SHRINKING_WINDOW_RETRY_LOOKUPS;
+        while (!attempt.token && retryLength >= 1 && retryLookupsRemaining > 0) {
+          retryLookupsRemaining -= 1;
           const retry = await findTokenAt(i, retryLength);
           if (retry.token) {
             attempt = retry;
@@ -1530,17 +1581,37 @@ ${YOMITAN_SCANNING_HELPERS}
           retryLength = Math.min(retryLength - 1, retry.matchedLength - 1);
         }
         if (attempt.token) {
+          flushUnparsedRun(unparsedRunStart, i);
+          unparsedRunStart = null;
           tokens.push(attempt.token);
           i += attempt.matchedLength;
           continue;
         }
+        if (unparsedRunStart === null) { unparsedRunStart = i; }
         i += String.fromCodePoint(text.codePointAt(i)).length;
       }
+      flushUnparsedRun(unparsedRunStart, text.length);
       return tokens;
+    };
+    return true;
+  })();
+`;
+
+function buildYomitanScanCallScript(params: YomitanScanRequestParams): string {
+  return `
+    (async () => {
+      if (typeof globalThis.__subminerYomitanScan !== "function") {
+        return ${JSON.stringify(YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL)};
+      }
+      return await globalThis.__subminerYomitanScan(${JSON.stringify(params)});
     })();
   `;
 }
 
+async function installYomitanScanRuntime(parserWindow: BrowserWindow): Promise<void> {
+  await parserWindow.webContents.executeJavaScript(YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT, true);
+}
+
 export async function requestYomitanParseResults(
   text: string,
   deps: YomitanParserRuntimeDeps,
@@ -1635,6 +1706,20 @@ export async function requestYomitanParseResults(
   }
 }
 
+// parseText fallback for when the in-window scanner cannot run (script eval
+// failure, unexpected payload). The scanner walk is the primary tokenizer and
+// emits its own filler runs, so this extra full parse only happens on errors.
+async function requestYomitanParseFallbackTokens(
+  text: string,
+  deps: YomitanParserRuntimeDeps,
+  logger: LoggerLike,
+): Promise<YomitanScanToken[] | null> {
+  const parseResults = await requestYomitanParseResults(text, deps, logger);
+  const selectedTokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
+  const parseScanTokens = selectedTokens?.map(toYomitanScanToken) ?? null;
+  return parseScanTokens && parseScanTokens.length > 0 ? parseScanTokens : null;
+}
+
 export async function requestYomitanScanTokens(
   text: string,
   deps: YomitanParserRuntimeDeps,
@@ -1655,10 +1740,6 @@ export async function requestYomitanScanTokens(
     return null;
   }
 
-  const parseResults = await requestYomitanParseResults(text, deps, logger);
-  const selectedParseTokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
-  const parseScanTokens = selectedParseTokens?.map(toYomitanScanToken) ?? null;
-
   const metadata = await requestYomitanProfileMetadata(parserWindow, logger);
   const profileIndex = metadata?.profileIndex ?? 0;
   const scanLength = metadata?.scanLength ?? DEFAULT_YOMITAN_SCAN_LENGTH;
@@ -1669,44 +1750,41 @@ export async function requestYomitanScanTokens(
       name.startsWith(CHARACTER_DICTIONARY_TITLE_PREFIX),
     );
 
+  const callScript = buildYomitanScanCallScript({
+    text,
+    profileIndex,
+    scanLength,
+    includeNameMatchMetadata,
+    greedyNameScanEnabled,
+    currentCharacterDictionaryMediaId:
+      typeof options?.currentCharacterDictionaryMediaId === 'number' &&
+      Number.isFinite(options.currentCharacterDictionaryMediaId) &&
+      options.currentCharacterDictionaryMediaId > 0
+        ? Math.floor(options.currentCharacterDictionaryMediaId)
+        : null,
+    dictionaryPriorityByName: metadata?.dictionaryPriorityByName ?? {},
+    dictionaryFrequencyModeByName: metadata?.dictionaryFrequencyModeByName ?? {},
+    cacheEpoch: getYomitanScanCacheEpoch(parserWindow),
+  });
+
   try {
-    const rawResult = await parserWindow.webContents.executeJavaScript(
-      buildYomitanScanningScript(
-        text,
-        profileIndex,
-        scanLength,
-        includeNameMatchMetadata,
-        greedyNameScanEnabled,
-        typeof options?.currentCharacterDictionaryMediaId === 'number' &&
-          Number.isFinite(options.currentCharacterDictionaryMediaId) &&
-          options.currentCharacterDictionaryMediaId > 0
-          ? Math.floor(options.currentCharacterDictionaryMediaId)
-          : null,
-        metadata?.dictionaryPriorityByName ?? {},
-        metadata?.dictionaryFrequencyModeByName ?? {},
-      ),
-      true,
-    );
+    let rawResult = await parserWindow.webContents.executeJavaScript(callScript, true);
+    if (rawResult === YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL) {
+      // First request for this window, or the page reloaded and dropped the
+      // installed runtime: install and retry once.
+      await installYomitanScanRuntime(parserWindow);
+      rawResult = await parserWindow.webContents.executeJavaScript(callScript, true);
+    }
     if (isScanTokenArray(rawResult)) {
-      if (parseScanTokens && parseScanTokens.length > 0) {
-        return mergeScannerTokensIntoParseTokens(parseScanTokens, rawResult);
-      }
-      return rawResult;
+      // Filler-only results carry no dictionary match; keep the historical
+      // contract of returning null so callers fall back to raw text.
+      return rawResult.some((token) => token.isUnparsedRun !== true) ? rawResult : null;
     }
-    if (Array.isArray(rawResult)) {
-      const selectedTokens = selectYomitanParseTokens(rawResult, () => false, 'headword');
-      return selectedTokens?.map(toYomitanScanToken) ?? null;
-    }
-    if (parseScanTokens && parseScanTokens.length > 0) {
-      return parseScanTokens;
-    }
-    return null;
+    logger.error('Yomitan scanner returned an unexpected payload; using parseText fallback.');
+    return await requestYomitanParseFallbackTokens(text, deps, logger);
   } catch (err) {
-    if (parseScanTokens && parseScanTokens.length > 0) {
-      return parseScanTokens;
-    }
     logger.error('Yomitan scanner request failed:', (err as Error).message);
-    return null;
+    return await requestYomitanParseFallbackTokens(text, deps, logger);
   }
 }
 

src/main.ts

@@ -1816,7 +1816,7 @@ function withCurrentSubtitleTiming(payload: SubtitleData): SubtitleData {
     endTime: appState.mpvClient?.currentSubEnd ?? null,
   };
 }
-function emitSubtitlePayload(payload: SubtitleData): void {
+function emitSubtitlePayload(payload: SubtitleData, options?: { resumePrefetch?: boolean }): void {
   const timedPayload = withCurrentSubtitleTiming(payload);
   const currentSubtitleData = appState.currentSubtitleData;
   const isAnnotationUpgrade = isSubtitleAnnotationUpgrade(currentSubtitleData, timedPayload);
@@ -1833,7 +1833,12 @@ function emitSubtitlePayload(payload: SubtitleData): void {
   }
   annotationSubtitleWsService.broadcast(timedPayload, frequencyOptions);
   autoplayReadyGate.maybeSignalPluginAutoplayReady(timedPayload, { forceWhilePaused: true });
-  subtitlePrefetchService?.resume();
+  // resumePrefetch: false marks a provisional pre-tokenization emit; prefetch
+  // stays paused until the tokenized payload for the line lands so it does not
+  // compete with the on-screen line for the single Yomitan parser window.
+  if (options?.resumePrefetch !== false) {
+    subtitlePrefetchService?.resume();
+  }
 }
 function getCurrentAutoplaySubtitlePayload(): SubtitleData | null {
   const payload = appState.currentSubtitleData;
@@ -1926,7 +1931,7 @@ const autoplaySubtitlePrimingRuntime = createAutoplaySubtitlePrimingRuntime({
     appState.activeParsedSubtitleMediaPath = mediaPath;
   },
   subtitleProcessingController,
-  emitSubtitlePayload: (payload) => emitSubtitlePayload(payload),
+  emitSubtitlePayload: (payload, options) => emitSubtitlePayload(payload, options),
   getSubtitlePrefetchService: () => subtitlePrefetchService,
   getLastObservedTimePos: () => lastObservedTimePos,
   getVisibleOverlayVisible: () => overlayManager.getVisibleOverlayVisible(),
@@ -4372,8 +4377,9 @@ const {
       emitSubtitlePayload(payload);
     },
     onSubtitleChange: (text) => {
+      // Pause only; restarting the prefetch run here would discard in-flight
+      // tokenization work on every line. Real seeks restart via onTimePosUpdate.
       subtitlePrefetchService?.pause();
-      subtitlePrefetchService?.onSeek(lastObservedTimePos);
       subtitleProcessingController.onSubtitleChange(text);
     },
     refreshDiscordPresence: () => {

src/main/main-wiring.test.ts

@@ -223,7 +223,7 @@ test('update overlay notification action triggers install flow', () => {
   assert.match(runtimeSource, /fallbackClient\.openNoteInBrowser\(noteId\)/);
 });
 
-test('subtitle change re-prioritizes prefetch around live playback before tokenizing current line', () => {
+test('subtitle change pauses prefetch without restarting its run before tokenizing current line', () => {
   const source = readMainSource();
   const actionBlock = source.match(
     /onSubtitleChange:\s*\(text\)\s*=>\s*\{(?<body>[\s\S]*?)\n    \},\n    refreshDiscordPresence:/,
@@ -231,14 +231,12 @@ test('subtitle change re-prioritizes prefetch around live playback before tokeni
 
   assert.ok(actionBlock);
   assert.match(actionBlock, /subtitlePrefetchService\?\.pause\(\);/);
-  assert.match(actionBlock, /subtitlePrefetchService\?\.onSeek\(lastObservedTimePos\);/);
+  // Restarting the run per line (onSeek) discards in-flight prefetch work;
+  // only real seeks restart via onTimePosUpdate.
+  assert.doesNotMatch(actionBlock, /subtitlePrefetchService\?\.onSeek\(/);
   assert.match(actionBlock, /subtitleProcessingController\.onSubtitleChange\(text\);/);
   assert.ok(
     actionBlock.indexOf('subtitlePrefetchService?.pause();') <
-      actionBlock.indexOf('subtitlePrefetchService?.onSeek(lastObservedTimePos);'),
-  );
-  assert.ok(
-    actionBlock.indexOf('subtitlePrefetchService?.onSeek(lastObservedTimePos);') <
       actionBlock.indexOf('subtitleProcessingController.onSubtitleChange(text);'),
   );
 });
@@ -593,7 +591,7 @@ test('YouTube media cache lifecycle routes through configured status notificatio
 test('subtitle broadcasts share one frequency options snapshot per emitted payload', () => {
   const source = readMainSource();
   const emitBlock = source.match(
-    /function emitSubtitlePayload\(payload: SubtitleData\): void \{(?<body>[\s\S]*?)\n\}/,
+    /function emitSubtitlePayload\([\s\S]*?\): void \{(?<body>[\s\S]*?)\n\}/,
   )?.groups?.body;
   const frequencyOptionsSnapshot = emitBlock?.match(
     /const frequencyDictionary = configService\.getConfig\(\)\.subtitleStyle\.frequencyDictionary;(?<body>[\s\S]*?)\n  \};/,

src/main/runtime/autoplay-subtitle-priming-runtime.test.ts

@@ -96,10 +96,10 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su
       onSubtitleChange: (text) => calls.push(`change:${text}`),
       refreshCurrentSubtitle: (text) => calls.push(`refresh:${text ?? ''}`),
     },
-    emitSubtitlePayload: (payload) => calls.push(`emit:${payload.text}`),
+    emitSubtitlePayload: (payload, options) =>
+      calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`),
     getSubtitlePrefetchService: () => ({
       pause: () => calls.push('prefetch:pause'),
-      onSeek: (timePos) => calls.push(`prefetch:seek:${timePos}`),
     }),
     getLastObservedTimePos: () => 12,
     getVisibleOverlayVisible: () => true,
@@ -120,7 +120,7 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su
     'request:time-pos',
     'set:起動字幕',
     'prefetch:pause',
-    'emit:起動字幕',
+    'emit:起動字幕:resume=false',
     'change:起動字幕',
   ]);
 });
@@ -154,10 +154,10 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before
       onSubtitleChange: (text) => calls.push(`change:${text}`),
       refreshCurrentSubtitle: (text) => calls.push(`refresh:${text ?? ''}`),
     },
-    emitSubtitlePayload: (payload) => calls.push(`emit:${payload.text}`),
+    emitSubtitlePayload: (payload, options) =>
+      calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`),
     getSubtitlePrefetchService: () => ({
       pause: () => calls.push('prefetch:pause'),
-      onSeek: (timePos) => calls.push(`prefetch:seek:${timePos}`),
     }),
     getLastObservedTimePos: () => 12,
     getVisibleOverlayVisible: () => true,
@@ -175,7 +175,7 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before
     'request:sub-text',
     'set:起動字幕',
     'prefetch:pause',
-    'emit:起動字幕',
+    'emit:起動字幕:resume=false',
     'change:起動字幕',
   ]);
 });

src/main/runtime/autoplay-subtitle-priming-runtime.ts

@@ -17,7 +17,6 @@ type AutoplaySubtitlePrimingMpvClient = {
 
 type AutoplaySubtitlePrimingPrefetchService = {
   pause: () => void;
-  onSeek: (timePos: number) => void;
 };
 
 export interface AutoplaySubtitlePrimingRuntimeDeps {
@@ -33,7 +32,7 @@ export interface AutoplaySubtitlePrimingRuntimeDeps {
     onSubtitleChange: (text: string) => void;
     refreshCurrentSubtitle: (text: string) => void;
   };
-  emitSubtitlePayload: (payload: SubtitleData) => void;
+  emitSubtitlePayload: (payload: SubtitleData, options?: { resumePrefetch?: boolean }) => void;
   getSubtitlePrefetchService: () => AutoplaySubtitlePrimingPrefetchService | null;
   getLastObservedTimePos: () => number;
   getVisibleOverlayVisible: () => boolean;
@@ -108,7 +107,9 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
       return true;
     }
 
-    emitSubtitlePayload({ text, tokens: null });
+    // Provisional raw emit: keep prefetch paused until the tokenized payload
+    // for this line is delivered by the processing controller.
+    emitSubtitlePayload({ text, tokens: null }, { resumePrefetch: false });
     subtitleProcessingController.onSubtitleChange(text);
     return true;
   }
@@ -154,12 +155,10 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
       consumeCachedSubtitle: (text) => subtitleProcessingController.consumeCachedSubtitle(text),
       onSubtitleChange: (text) => {
         deps.getSubtitlePrefetchService()?.pause();
-        deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
         subtitleProcessingController.onSubtitleChange(text);
       },
       refreshCurrentSubtitle: (text) => {
         deps.getSubtitlePrefetchService()?.pause();
-        deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
         subtitleProcessingController.refreshCurrentSubtitle(text);
       },
       deferUncachedRefresh: true,
@@ -205,7 +204,6 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
         return;
       }
       deps.getSubtitlePrefetchService()?.pause();
-      deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
       subtitleProcessingController.refreshCurrentSubtitle(text);
     }, VISIBLE_OVERLAY_SUBTITLE_REFRESH_AFTER_FIRST_PAINT_DELAY_MS);
     visibleOverlaySubtitleRefreshAfterFirstPaintTimer.unref?.();

src/main/runtime/composers/mpv-runtime-composer.ts

@@ -225,24 +225,35 @@ export function composeMpvRuntimeHandlers<
     }
     return tokenizationWarmupInFlight;
   };
+  // Built once and reused for every tokenization: per-call rebuilds create
+  // fresh closures, which defeats identity-keyed caches downstream (the JLPT
+  // lookup cache keys on the getJlptLevel function, and the mecab availability
+  // WeakSet keys on the runtime deps instance).
+  let cachedTokenizerRuntimeDeps: TTokenizerRuntimeDeps | null = null;
+  const getTokenizerRuntimeDeps = (): TTokenizerRuntimeDeps => {
+    if (cachedTokenizerRuntimeDeps) {
+      return cachedTokenizerRuntimeDeps;
+    }
+    const tokenizerMainDeps = buildTokenizerDepsHandler();
+    const baseOnTokenizationReady = tokenizerMainDeps.onTokenizationReady;
+    tokenizerMainDeps.onTokenizationReady = (tokenizedText: string): void => {
+      if (!shouldWarmupAnnotationDictionaries()) {
+        baseOnTokenizationReady?.(tokenizedText);
+        return;
+      }
+      markTokenizationPlaybackReady();
+      baseOnTokenizationReady?.(tokenizedText);
+      if (!tokenizationWarmupCompleted) {
+        void prewarmSubtitleDictionaries({ showLoadingOsd: true }).catch(() => {});
+      }
+    };
+    cachedTokenizerRuntimeDeps = options.tokenizer.createTokenizerRuntimeDeps(tokenizerMainDeps);
+    return cachedTokenizerRuntimeDeps;
+  };
   const tokenizeSubtitle = async (text: string): Promise<TTokenizedSubtitle> => {
     if (!tokenizationWarmupCompleted) void startTokenizationWarmups();
     await ensureTokenizationPrerequisites();
-    const tokenizerMainDeps = buildTokenizerDepsHandler();
-    if (shouldWarmupAnnotationDictionaries()) {
-      const onTokenizationReady = tokenizerMainDeps.onTokenizationReady;
-      tokenizerMainDeps.onTokenizationReady = (tokenizedText: string): void => {
-        markTokenizationPlaybackReady();
-        onTokenizationReady?.(tokenizedText);
-        if (!tokenizationWarmupCompleted) {
-          void prewarmSubtitleDictionaries({ showLoadingOsd: true }).catch(() => {});
-        }
-      };
-    }
-    return options.tokenizer.tokenizeSubtitle(
-      text,
-      options.tokenizer.createTokenizerRuntimeDeps(tokenizerMainDeps),
-    );
+    return options.tokenizer.tokenizeSubtitle(text, getTokenizerRuntimeDeps());
   };
 
   const launchBackgroundWarmupTask = createLaunchBackgroundWarmupTaskFromStartup(