perf(tokenizer): single-pass Yomitan scan with install-once runtime and cross-line cache

- drop the duplicate parseText full parse per line; the termsFind scanner walk
  is now authoritative and emits its own unparsed filler runs (parseText kept
  only as error fallback)
- install scan helpers once per parser window (__subminerYomitanScan) instead
  of re-shipping ~500 lines of script per subtitle line
- persist termsFind results across lines in a window-scoped LRU keyed by
  substring, invalidated via a cache epoch on dictionary/settings changes
- skip lookups at punctuation/whitespace positions and cap the shrinking-window
  retry ladder at 4 lookups per position
- build tokenizer runtime deps once (JLPT lookup cache never hit before; mecab
  availability check ran per line)
- stop restarting the prefetch run on every subtitle change; resume prefetch
  only after the tokenized payload lands, not on provisional raw emits
- add per-stage debug timings (scanMs/mecabMs/frequencyMs/annotateMs)

changes/subtitle-tokenization-performance.md

@@ -0,0 +1,10 @@
+type: changed
+area: subtitles
+
+- Subtitle tokenization no longer runs a duplicate full `parseText` pass per line: the termsFind scanner walk is now the only tokenizer and emits its own hoverable filler runs for unmatched text (parseText is kept only as an error fallback). This roughly halves the dictionary work per line.
+- The Yomitan scanning helpers are now installed once per parser window (`__subminerYomitanScan`) instead of re-shipping and re-parsing a ~500-line script for every subtitle line; each line only evaluates a tiny call.
+- termsFind lookups are cached across subtitle lines in a window-persistent LRU keyed by substring, so repeated particles and verb forms stop costing backend round trips. The cache invalidates on dictionary/settings changes and window reloads.
+- The scanner walk now skips lookups at punctuation and whitespace positions (latin letters and digits still look up, e.g. Tシャツ) and caps the shrinking-window retry ladder at four extra lookups per position.
+- Tokenizer runtime dependencies are built once instead of per line, fixing a JLPT lookup cache that never hit (it was keyed on a per-call closure identity and leaked a Map per line) and a `which mecab` availability check that re-ran synchronously on every line when MeCab is absent.
+- Subtitle changes no longer restart the prefetch run per line (which discarded in-flight tokenization work); prefetch now only pauses for the live line and restarts on real seeks, cache invalidation, or option changes. Prefetch also stays paused across a provisional raw-subtitle emit and resumes only after the tokenized payload lands, so it never competes with the on-screen line for the parser window.
+- Added per-stage debug timings (`scanMs`, `mecabMs`, `frequencyMs`, `annotateMs`) to the subtitle tokenization pipeline log.

src/core/services/tokenizer.test.ts

@@ -2934,44 +2934,12 @@ test('tokenizeSubtitle preserves Yomitan compound token when MeCab components ar
                 return [];
               }
 
-              if (script.includes('parseText')) {
-                return [
-                  {
-                    source: 'scanning-parser',
-                    index: 0,
-                    content: [
-                      [
-                        {
-                          text: '取り組んで',
-                          reading: 'とりくんで',
-                          headwords: [[{ term: '取り組む' }]],
-                        },
-                      ],
-                      [
-                        {
-                          text: 'もらいます',
-                          reading: 'もらいます',
-                          headwords: [[{ term: 'もらう' }]],
-                        },
-                      ],
-                    ],
-                  },
-                ];
-              }
-
               return [
                 {
-                  surface: '取り',
+                  surface: '取り組んで',
-                  reading: 'とり',
+                  reading: 'とりくんで',
-                  headword: '取る',
+                  headword: '取り組む',
                   startPos: 0,
-                  endPos: 2,
-                },
-                {
-                  surface: '組んで',
-                  reading: 'くんで',
-                  headword: '組む',
-                  startPos: 2,
                   endPos: 5,
                 },
                 {

src/core/services/tokenizer.ts

@@ -716,15 +716,29 @@ function getAnnotationOptions(deps: TokenizerServiceDeps): TokenizerAnnotationOp
   };
 }
 
+// Per-line stage durations for the pipeline debug log; every field is filled in
+// by the stage that awaits the corresponding work.
+interface TokenizationStageTimings {
+  scanMs?: number;
+  mecabMs?: number;
+  frequencyMs?: number;
+  annotateMs?: number;
+}
+
 async function parseWithYomitanInternalParser(
   text: string,
   deps: TokenizerServiceDeps,
   options: TokenizerAnnotationOptions,
+  stageTimings?: TokenizationStageTimings,
 ): Promise<MergedToken[] | null> {
+  const scanStartedAtMs = Date.now();
   const selectedTokens = await requestYomitanScanTokens(text, deps, logger, {
     includeNameMatchMetadata: options.nameMatchEnabled,
     currentCharacterDictionaryMediaId: deps.getCurrentCharacterDictionaryMediaId?.() ?? null,
   });
+  if (stageTimings) {
+    stageTimings.scanMs = Date.now() - scanStartedAtMs;
+  }
   if (!selectedTokens || selectedTokens.length === 0) {
     return null;
   }
@@ -757,6 +771,7 @@ async function parseWithYomitanInternalParser(
 
   const frequencyRankPromise: Promise<YomitanFrequencyIndex> = options.frequencyEnabled
     ? (async () => {
+        const frequencyStartedAtMs = Date.now();
         const frequencyMatchMode = options.frequencyMatchMode;
         const termReadingList = buildYomitanFrequencyTermReadingList(
           normalizedSelectedTokens,
@@ -767,12 +782,17 @@ async function parseWithYomitanInternalParser(
           deps,
           logger,
         );
-        return buildYomitanFrequencyIndex(yomitanFrequencies);
+        const frequencyIndex = buildYomitanFrequencyIndex(yomitanFrequencies);
+        if (stageTimings) {
+          stageTimings.frequencyMs = Date.now() - frequencyStartedAtMs;
+        }
+        return frequencyIndex;
       })()
     : Promise.resolve({ byPair: new Map(), byTerm: new Map() });
 
   const mecabEnrichmentPromise: Promise<MergedToken[]> = needsMecabPosEnrichment(options)
     ? (async () => {
+        const mecabStartedAtMs = Date.now();
         try {
           const mecabTokens = await deps.tokenizeWithMecab(text);
           const enrichTokensWithMecab = deps.enrichTokensWithMecab ?? enrichTokensWithMecabAsync;
@@ -786,6 +806,10 @@ async function parseWithYomitanInternalParser(
             `textLength=${text.length}`,
           );
           return normalizedSelectedTokens;
+        } finally {
+          if (stageTimings) {
+            stageTimings.mecabMs = Date.now() - mecabStartedAtMs;
+          }
         }
       })()
     : Promise.resolve(normalizedSelectedTokens);
@@ -876,15 +900,35 @@ export async function tokenizeSubtitle(
   const annotationOptions = getAnnotationOptions(deps);
   annotationOptions.sourceText = tokenizeText;
 
-  const yomitanTokens = await parseWithYomitanInternalParser(tokenizeText, deps, annotationOptions);
+  const stageTimings: TokenizationStageTimings = {};
+  const startedAtMs = Date.now();
+  const logStageTimings = (tokenCount: number): void => {
+    logger.debug(
+      `Subtitle tokenization stages; textLength=${tokenizeText.length}, tokenCount=${tokenCount}, ` +
+        `scanMs=${stageTimings.scanMs ?? '-'}, mecabMs=${stageTimings.mecabMs ?? '-'}, ` +
+        `frequencyMs=${stageTimings.frequencyMs ?? '-'}, annotateMs=${stageTimings.annotateMs ?? '-'}, ` +
+        `totalMs=${Date.now() - startedAtMs}`,
+    );
+  };
+
+  const yomitanTokens = await parseWithYomitanInternalParser(
+    tokenizeText,
+    deps,
+    annotationOptions,
+    stageTimings,
+  );
   if (yomitanTokens && yomitanTokens.length > 0) {
+    const annotateStartedAtMs = Date.now();
     const annotatedTokens = await applyAnnotationStage(yomitanTokens, deps, annotationOptions);
     const renderedTokens = applyCharacterNameImages(annotatedTokens, deps, annotationOptions);
+    stageTimings.annotateMs = Date.now() - annotateStartedAtMs;
+    logStageTimings(renderedTokens.length);
     return {
       text: displayText,
       tokens: renderedTokens.length > 0 ? renderedTokens : null,
     };
   }
 
+  logStageTimings(0);
   return { text: displayText, tokens: null };
 }

src/core/services/tokenizer/golden-corpus-harness.ts

@@ -366,8 +366,11 @@ export function createReplayMessageStore(messages: GoldenRecordedMessage[]): Rep
   };
 }
 
-async function runInjectedScriptInVm(script: string, store: ReplayMessageStore): Promise<unknown> {
+// One persistent context per fixture, matching the real parser window: the
-  return await vm.runInNewContext(script, {
+// scan runtime installs itself once into globalThis and later per-line call
+// scripts reuse it.
+function createInjectedScriptVm(store: ReplayMessageStore): (script: string) => Promise<unknown> {
+  const context = vm.createContext({
     chrome: {
       runtime: {
         lastError: null,
@@ -393,6 +396,7 @@ async function runInjectedScriptInVm(script: string, store: ReplayMessageStore):
     Set,
     String,
   });
+  return async (script: string) => await vm.runInContext(script, context);
 }
 
 export function createReplayTokenizerDeps(fixture: GoldenFixture): TokenizerServiceDeps {
@@ -400,13 +404,14 @@ export function createReplayTokenizerDeps(fixture: GoldenFixture): TokenizerServ
   const scriptResults = new Map(
     fixture.recording.scripts.map((entry) => [entry.sha256, entry] as const),
   );
+  const runInjectedScriptInVm = createInjectedScriptVm(store);
 
   const parserWindow = {
     isDestroyed: () => false,
     webContents: {
       executeJavaScript: async (script: string) => {
         try {
-          return await runInjectedScriptInVm(script, store);
+          return await runInjectedScriptInVm(script);
         } catch (vmError) {
           const recorded = scriptResults.get(hashInjectedScript(script));
           if (recorded) {

src/core/services/tokenizer/yomitan-parser-runtime.ts

@@ -81,6 +81,13 @@ const yomitanFrequencyCacheByWindow = new WeakMap<
   BrowserWindow,
   Map<string, YomitanTermFrequency[]>
 >();
+// Epoch passed with every scan request; the in-window termsFind cache clears
+// itself when the epoch changes (dictionary imports, settings changes).
+const yomitanScanCacheEpochByWindow = new WeakMap<BrowserWindow, number>();
+
+function getYomitanScanCacheEpoch(window: BrowserWindow): number {
+  return yomitanScanCacheEpochByWindow.get(window) ?? 0;
+}
 
 function isObject(value: unknown): value is Record<string, unknown> {
   return Boolean(value && typeof value === 'object');
@@ -99,6 +106,7 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
         typeof entry.startPos === 'number' &&
         typeof entry.endPos === 'number' &&
         (entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean') &&
+        (entry.isUnparsedRun === undefined || typeof entry.isUnparsedRun === 'boolean') &&
         (entry.frequencyRank === undefined || typeof entry.frequencyRank === 'number') &&
         (entry.wordClasses === undefined ||
           (Array.isArray(entry.wordClasses) &&
@@ -107,13 +115,9 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
   );
 }
 
-function scanTokenSpanKey(token: YomitanScanToken): string {
-  return `${token.startPos}:${token.endPos}:${token.surface}`;
-}
-
 // Maps a parse-selected token to the scanner-token shape carried out of the
-// parser runtime. Shared by both selectYomitanParseTokens fallback paths so the
+// parser runtime, used by the parseText fallback path when the in-window
-// projected fields stay in sync as the shape changes.
+// scanner is unavailable.
 function toYomitanScanToken(token: {
   surface: string;
   reading: string;
@@ -132,66 +136,6 @@ function toYomitanScanToken(token: {
   };
 }
 
-// parseText segmentation is authoritative (it emits filler chunks for text the
-// termsFind scanner skips), but only the termsFind scanner carries annotation
-// metadata (isNameMatch, frequencyRank, headwordReading, wordClasses). Graft
-// scanner tokens onto the parseText segmentation per matching span so one
-// unmatched chunk degrades only itself instead of dropping the whole line's
-// metadata.
-//
-// Exception: character-name tokens. The greedy name scan can re-segment text
-// around a name (e.g. とヨータ → と + ヨータ instead of とヨー + タ), so
-// parseText segmentation cannot be authoritative there. Each name span is
-// expanded until it aligns with token boundaries in both segmentations, then
-// the parse tokens inside are replaced with the scanner tokens.
-function mergeScannerTokensIntoParseTokens(
-  parseScanTokens: YomitanScanToken[],
-  scannerTokens: YomitanScanToken[],
-): YomitanScanToken[] {
-  const scannerTokensBySpan = new Map<string, YomitanScanToken>();
-  for (const token of scannerTokens) {
-    scannerTokensBySpan.set(scanTokenSpanKey(token), token);
-  }
-  const graftedTokens = parseScanTokens.map(
-    (token) => scannerTokensBySpan.get(scanTokenSpanKey(token)) ?? token,
-  );
-
-  const nameTokens = scannerTokens.filter((token) => token.isNameMatch === true);
-  if (nameTokens.length === 0) {
-    return graftedTokens;
-  }
-
-  const regions = nameTokens.map((token) => ({ start: token.startPos, end: token.endPos }));
-  const allTokens = [...parseScanTokens, ...scannerTokens];
-  let expanded = true;
-  while (expanded) {
-    expanded = false;
-    for (const region of regions) {
-      for (const token of allTokens) {
-        const overlaps = token.startPos < region.end && token.endPos > region.start;
-        const extendsBeyond = token.startPos < region.start || token.endPos > region.end;
-        if (overlaps && extendsBeyond) {
-          region.start = Math.min(region.start, token.startPos);
-          region.end = Math.max(region.end, token.endPos);
-          expanded = true;
-        }
-      }
-    }
-  }
-
-  const isInsideNameRegion = (token: YomitanScanToken): boolean =>
-    regions.some((region) => token.startPos >= region.start && token.endPos <= region.end);
-
-  const merged = graftedTokens.filter((token) => !isInsideNameRegion(token));
-  for (const token of scannerTokens) {
-    if (isInsideNameRegion(token)) {
-      merged.push(token);
-    }
-  }
-  merged.sort((a, b) => a.startPos - b.startPos || a.endPos - b.endPos);
-  return merged;
-}
-
 function makeTermReadingCacheKey(term: string, reading: string | null): string {
   return `${term}\u0000${reading ?? ''}`;
 }
@@ -208,6 +152,7 @@ function getWindowFrequencyCache(window: BrowserWindow): Map<string, YomitanTerm
 function clearWindowCaches(window: BrowserWindow): void {
   yomitanProfileMetadataByWindow.delete(window);
   yomitanFrequencyCacheByWindow.delete(window);
+  yomitanScanCacheEpochByWindow.set(window, getYomitanScanCacheEpoch(window) + 1);
 }
 export function clearYomitanParserCachesForWindow(window: BrowserWindow): void {
   clearWindowCaches(window);
@@ -704,6 +649,10 @@ async function ensureYomitanParserWindow(
       if (readyPromise) {
         await readyPromise;
       }
+      // Eagerly install the scan runtime so the first subtitle line does not
+      // pay the install round trip; failures fall back to the per-request
+      // install-and-retry path.
+      await installYomitanScanRuntime(parserWindow).catch(() => {});
 
       return true;
     } catch (err) {
@@ -1362,58 +1311,144 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
       }
 `;
 
-function buildYomitanScanningScript(
+// Bump whenever the install script below changes so already-loaded parser
-  text: string,
+// windows re-install the new scan runtime instead of running the stale one.
-  profileIndex: number,
+const YOMITAN_SCAN_RUNTIME_VERSION = 1;
-  scanLength: number,
+const YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL = '__subminer-yomitan-scan-runtime-missing__';
-  includeNameMatchMetadata: boolean,
+
-  greedyNameScanEnabled: boolean,
+interface YomitanScanRequestParams {
-  currentCharacterDictionaryMediaId: number | null,
+  text: string;
-  dictionaryPriorityByName: Record<string, number>,
+  profileIndex: number;
-  dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>,
+  scanLength: number;
-): string {
+  includeNameMatchMetadata: boolean;
-  return `
+  greedyNameScanEnabled: boolean;
-    (async () => {
+  currentCharacterDictionaryMediaId: number | null;
-      const invoke = (action, params) =>
+  dictionaryPriorityByName: Record<string, number>;
-        new Promise((resolve, reject) => {
+  dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>;
-          chrome.runtime.sendMessage({ action, params }, (response) => {
+  cacheEpoch: number;
-            if (chrome.runtime.lastError) {
+}
-              reject(new Error(chrome.runtime.lastError.message));
+
-              return;
+// Installed once per parser window (and re-installed after in-page reloads):
-            }
+// keeps V8 from re-parsing the helper bundle on every subtitle line, and hosts
-            if (!response || typeof response !== "object") {
+// the cross-line termsFind cache. Each subtitle line then only evaluates a tiny
-              reject(new Error("Invalid response from Yomitan backend"));
+// call into globalThis.__subminerYomitanScan.
-              return;
+const YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT = String.raw`
-            }
+  (() => {
-            if (response.error) {
+    if (globalThis.__subminerYomitanScanVersion === ${YOMITAN_SCAN_RUNTIME_VERSION}) {
-              reject(new Error(response.error.message || "Yomitan backend error"));
+      return true;
-              return;
+    }
-            }
+    const invoke = (action, params) =>
-            resolve(response.result);
+      new Promise((resolve, reject) => {
-          });
+        chrome.runtime.sendMessage({ action, params }, (response) => {
+          if (chrome.runtime.lastError) {
+            reject(new Error(chrome.runtime.lastError.message));
+            return;
+          }
+          if (!response || typeof response !== "object") {
+            reject(new Error("Invalid response from Yomitan backend"));
+            return;
+          }
+          if (response.error) {
+            reject(new Error(response.error.message || "Yomitan backend error"));
+            return;
+          }
+          resolve(response.result);
         });
+      });
+    // Cross-line termsFind LRU keyed by profile + substring: subtitle lines
+    // repeat particles and inflections constantly, so most lookups hit here.
+    // Entries hold in-flight promises so concurrent identical lookups dedupe.
+    const termsFindCache = new Map();
+    const TERMS_FIND_CACHE_LIMIT = 2000;
+    let termsFindCacheEpoch = -1;
+    const MAX_SHRINKING_WINDOW_RETRY_LOOKUPS = 4;
+    globalThis.__subminerYomitanScanVersion = ${YOMITAN_SCAN_RUNTIME_VERSION};
+    globalThis.__subminerYomitanScan = async (scanParams) => {
+      const {
+        text,
+        profileIndex,
+        scanLength,
+        includeNameMatchMetadata,
+        greedyNameScanEnabled,
+        currentCharacterDictionaryMediaId,
+        dictionaryPriorityByName,
+        dictionaryFrequencyModeByName,
+        cacheEpoch
+      } = scanParams;
+      if (cacheEpoch !== termsFindCacheEpoch) {
+        termsFindCache.clear();
+        termsFindCacheEpoch = cacheEpoch;
+      }
 ${YOMITAN_SCANNING_HELPERS}
-      const includeNameMatchMetadata = ${includeNameMatchMetadata ? 'true' : 'false'};
+      const CAPTION_OPENING_BRACKETS = new Set(["(", "（", "[", "［", "{", "｛", "「", "『", "【", "〈", "《", "≪", "＜", "<"]);
-      const greedyNameScanEnabled = ${greedyNameScanEnabled ? 'true' : 'false'};
+      function shouldEmitUnparsedRunAsToken(runText) {
-      const currentCharacterDictionaryMediaId = ${
+        if (!/[\p{L}\p{N}]/u.test(runText)) { return false; }
-        currentCharacterDictionaryMediaId !== null
+        const firstChar = Array.from(runText.trim())[0];
-          ? String(currentCharacterDictionaryMediaId)
+        return firstChar !== undefined && !CAPTION_OPENING_BRACKETS.has(firstChar);
-          : 'null'
+      }
-      };
+      function isLookupWorthyCodePoint(codePoint) {
-      const dictionaryPriorityByName = ${JSON.stringify(dictionaryPriorityByName)};
+        if (isCodePointJapanese(codePoint)) { return true; }
-      const dictionaryFrequencyModeByName = ${JSON.stringify(dictionaryFrequencyModeByName)};
+        return /[\p{L}\p{N}]/u.test(String.fromCodePoint(codePoint));
-      const text = ${JSON.stringify(text)};
+      }
+      function isKanaOnlyRunText(runText) {
+        const chars = Array.from(runText);
+        return chars.length > 0 && chars.every((char) => isCodePointKana(char.codePointAt(0)));
+      }
       const details = {matchType: "exact", deinflect: true};
       const tokens = [];
-      const termsFindCache = new Map();
       async function termsFindAt(position, windowLength) {
-        const cacheKey = position + ":" + windowLength;
-        const cached = termsFindCache.get(cacheKey);
-        if (cached) { return cached; }
         const substring = text.substring(position, position + windowLength);
-        const result = await invoke("termsFind", { text: substring, details, optionsContext: { index: ${profileIndex} } });
+        const cacheKey = profileIndex + "" + substring;
-        termsFindCache.set(cacheKey, result);
+        const cached = termsFindCache.get(cacheKey);
-        return result;
+        if (cached !== undefined) {
+          termsFindCache.delete(cacheKey);
+          termsFindCache.set(cacheKey, cached);
+          return await cached;
+        }
+        const pending = invoke("termsFind", { text: substring, details, optionsContext: { index: profileIndex } });
+        termsFindCache.set(cacheKey, pending);
+        while (termsFindCache.size > TERMS_FIND_CACHE_LIMIT) {
+          const oldestKey = termsFindCache.keys().next().value;
+          if (oldestKey === undefined) { break; }
+          termsFindCache.delete(oldestKey);
+        }
+        try {
+          return await pending;
+        } catch (error) {
+          termsFindCache.delete(cacheKey);
+          throw error;
+        }
+      }
+      // Text the walk skips accumulates into unparsed runs, mirroring the
+      // filler chunks the parseText segmentation used to provide: runs stay
+      // hoverable (flagged isUnparsedRun) unless they are punctuation-only or
+      // caption-style asides, and kana continuations of a longer headword
+      // extend the previous token instead.
+      function flushUnparsedRun(runStart, runEnd) {
+        if (runStart === null || runEnd <= runStart) { return; }
+        const runText = text.substring(runStart, runEnd);
+        const previousToken = tokens[tokens.length - 1];
+        if (
+          previousToken &&
+          previousToken.endPos === runStart &&
+          isKanaOnlyRunText(runText) &&
+          typeof previousToken.headword === "string" &&
+          previousToken.headword.length > previousToken.surface.length &&
+          previousToken.headword.startsWith(previousToken.surface + runText)
+        ) {
+          previousToken.surface += runText;
+          previousToken.endPos = runEnd;
+          return;
+        }
+        if (!shouldEmitUnparsedRunAsToken(runText)) { return; }
+        tokens.push({
+          surface: runText,
+          reading: "",
+          headword: runText,
+          startPos: runStart,
+          endPos: runEnd,
+          isUnparsedRun: true
+        });
       }
       function buildScanToken(position, source, preferredHeadword) {
         const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
@@ -1469,9 +1504,9 @@ ${YOMITAN_SCANNING_HELPERS}
             namePos += String.fromCodePoint(codePoint).length;
             continue;
           }
-          const result = await termsFindAt(namePos, ${scanLength});
+          const result = await termsFindAt(namePos, scanLength);
           const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
-          const textWindow = text.substring(namePos, namePos + ${scanLength});
+          const textWindow = text.substring(namePos, namePos + scanLength);
           const nameMatch = findLongestNameMatch(dictionaryEntries, textWindow);
           // A name only claims its span when no strictly longer generic word
           // starts at the same position (a character named 空 must not split
@@ -1502,26 +1537,42 @@ ${YOMITAN_SCANNING_HELPERS}
       }
       let i = 0;
       let nameIndex = 0;
+      let unparsedRunStart = null;
       while (i < text.length) {
         while (nameIndex < nameTokens.length && nameTokens[nameIndex].startPos < i) { nameIndex += 1; }
         const nextNameToken = nameIndex < nameTokens.length ? nameTokens[nameIndex] : null;
         if (nextNameToken && nextNameToken.startPos === i) {
+          flushUnparsedRun(unparsedRunStart, i);
+          unparsedRunStart = null;
           tokens.push(nextNameToken);
           i = nextNameToken.endPos;
           nameIndex += 1;
           continue;
         }
+        const codePoint = text.codePointAt(i);
+        // Punctuation and whitespace can never start a token: skip the backend
+        // round trip entirely. Latin letters and digits stay lookup-worthy
+        // (terms like Tシャツ start on an ASCII letter).
+        if (!isLookupWorthyCodePoint(codePoint)) {
+          if (unparsedRunStart === null) { unparsedRunStart = i; }
+          i += String.fromCodePoint(codePoint).length;
+          continue;
+        }
         // Cap the window at the next reserved name span so a generic match
         // cannot consume into it.
-        const windowLength = nextNameToken ? Math.min(${scanLength}, nextNameToken.startPos - i) : ${scanLength};
+        const windowLength = nextNameToken ? Math.min(scanLength, nextNameToken.startPos - i) : scanLength;
         let attempt = await findTokenAt(i, windowLength);
         // Yomitan text normalization can consume characters (whitespace,
         // punctuation) beyond the matched term, leaving no headword whose
         // source equals the consumed text. Retry with shorter windows so a
         // valid prefix term (e.g. a character name before a paren) still
-        // tokenizes instead of the position being skipped.
+        // tokenizes instead of the position being skipped. The ladder is
+        // capped: without a cap it degrades to O(scanLength) lookups at a
+        // single position.
         let retryLength = Math.min(attempt.matchedLength, windowLength) - 1;
-        while (!attempt.token && retryLength >= 1) {
+        let retryLookupsRemaining = MAX_SHRINKING_WINDOW_RETRY_LOOKUPS;
+        while (!attempt.token && retryLength >= 1 && retryLookupsRemaining > 0) {
+          retryLookupsRemaining -= 1;
           const retry = await findTokenAt(i, retryLength);
           if (retry.token) {
             attempt = retry;
@@ -1530,17 +1581,37 @@ ${YOMITAN_SCANNING_HELPERS}
           retryLength = Math.min(retryLength - 1, retry.matchedLength - 1);
         }
         if (attempt.token) {
+          flushUnparsedRun(unparsedRunStart, i);
+          unparsedRunStart = null;
           tokens.push(attempt.token);
           i += attempt.matchedLength;
           continue;
         }
+        if (unparsedRunStart === null) { unparsedRunStart = i; }
         i += String.fromCodePoint(text.codePointAt(i)).length;
       }
+      flushUnparsedRun(unparsedRunStart, text.length);
       return tokens;
+    };
+    return true;
+  })();
+`;
+
+function buildYomitanScanCallScript(params: YomitanScanRequestParams): string {
+  return `
+    (async () => {
+      if (typeof globalThis.__subminerYomitanScan !== "function") {
+        return ${JSON.stringify(YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL)};
+      }
+      return await globalThis.__subminerYomitanScan(${JSON.stringify(params)});
     })();
   `;
 }
 
+async function installYomitanScanRuntime(parserWindow: BrowserWindow): Promise<void> {
+  await parserWindow.webContents.executeJavaScript(YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT, true);
+}
+
 export async function requestYomitanParseResults(
   text: string,
   deps: YomitanParserRuntimeDeps,
@@ -1635,6 +1706,20 @@ export async function requestYomitanParseResults(
   }
 }
 
+// parseText fallback for when the in-window scanner cannot run (script eval
+// failure, unexpected payload). The scanner walk is the primary tokenizer and
+// emits its own filler runs, so this extra full parse only happens on errors.
+async function requestYomitanParseFallbackTokens(
+  text: string,
+  deps: YomitanParserRuntimeDeps,
+  logger: LoggerLike,
+): Promise<YomitanScanToken[] | null> {
+  const parseResults = await requestYomitanParseResults(text, deps, logger);
+  const selectedTokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
+  const parseScanTokens = selectedTokens?.map(toYomitanScanToken) ?? null;
+  return parseScanTokens && parseScanTokens.length > 0 ? parseScanTokens : null;
+}
+
 export async function requestYomitanScanTokens(
   text: string,
   deps: YomitanParserRuntimeDeps,
@@ -1655,10 +1740,6 @@ export async function requestYomitanScanTokens(
     return null;
   }
 
-  const parseResults = await requestYomitanParseResults(text, deps, logger);
-  const selectedParseTokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
-  const parseScanTokens = selectedParseTokens?.map(toYomitanScanToken) ?? null;
-
   const metadata = await requestYomitanProfileMetadata(parserWindow, logger);
   const profileIndex = metadata?.profileIndex ?? 0;
   const scanLength = metadata?.scanLength ?? DEFAULT_YOMITAN_SCAN_LENGTH;
@@ -1669,44 +1750,41 @@ export async function requestYomitanScanTokens(
       name.startsWith(CHARACTER_DICTIONARY_TITLE_PREFIX),
     );
 
+  const callScript = buildYomitanScanCallScript({
+    text,
+    profileIndex,
+    scanLength,
+    includeNameMatchMetadata,
+    greedyNameScanEnabled,
+    currentCharacterDictionaryMediaId:
+      typeof options?.currentCharacterDictionaryMediaId === 'number' &&
+      Number.isFinite(options.currentCharacterDictionaryMediaId) &&
+      options.currentCharacterDictionaryMediaId > 0
+        ? Math.floor(options.currentCharacterDictionaryMediaId)
+        : null,
+    dictionaryPriorityByName: metadata?.dictionaryPriorityByName ?? {},
+    dictionaryFrequencyModeByName: metadata?.dictionaryFrequencyModeByName ?? {},
+    cacheEpoch: getYomitanScanCacheEpoch(parserWindow),
+  });
+
   try {
-    const rawResult = await parserWindow.webContents.executeJavaScript(
+    let rawResult = await parserWindow.webContents.executeJavaScript(callScript, true);
-      buildYomitanScanningScript(
+    if (rawResult === YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL) {
-        text,
+      // First request for this window, or the page reloaded and dropped the
-        profileIndex,
+      // installed runtime: install and retry once.
-        scanLength,
+      await installYomitanScanRuntime(parserWindow);
-        includeNameMatchMetadata,
+      rawResult = await parserWindow.webContents.executeJavaScript(callScript, true);
-        greedyNameScanEnabled,
+    }
-        typeof options?.currentCharacterDictionaryMediaId === 'number' &&
-          Number.isFinite(options.currentCharacterDictionaryMediaId) &&
-          options.currentCharacterDictionaryMediaId > 0
-          ? Math.floor(options.currentCharacterDictionaryMediaId)
-          : null,
-        metadata?.dictionaryPriorityByName ?? {},
-        metadata?.dictionaryFrequencyModeByName ?? {},
-      ),
-      true,
-    );
     if (isScanTokenArray(rawResult)) {
-      if (parseScanTokens && parseScanTokens.length > 0) {
+      // Filler-only results carry no dictionary match; keep the historical
-        return mergeScannerTokensIntoParseTokens(parseScanTokens, rawResult);
+      // contract of returning null so callers fall back to raw text.
-      }
+      return rawResult.some((token) => token.isUnparsedRun !== true) ? rawResult : null;
-      return rawResult;
     }
-    if (Array.isArray(rawResult)) {
+    logger.error('Yomitan scanner returned an unexpected payload; using parseText fallback.');
-      const selectedTokens = selectYomitanParseTokens(rawResult, () => false, 'headword');
+    return await requestYomitanParseFallbackTokens(text, deps, logger);
-      return selectedTokens?.map(toYomitanScanToken) ?? null;
-    }
-    if (parseScanTokens && parseScanTokens.length > 0) {
-      return parseScanTokens;
-    }
-    return null;
   } catch (err) {
-    if (parseScanTokens && parseScanTokens.length > 0) {
-      return parseScanTokens;
-    }
     logger.error('Yomitan scanner request failed:', (err as Error).message);
-    return null;
+    return await requestYomitanParseFallbackTokens(text, deps, logger);
   }
 }
 

src/main.ts

@@ -1816,7 +1816,7 @@ function withCurrentSubtitleTiming(payload: SubtitleData): SubtitleData {
     endTime: appState.mpvClient?.currentSubEnd ?? null,
   };
 }
-function emitSubtitlePayload(payload: SubtitleData): void {
+function emitSubtitlePayload(payload: SubtitleData, options?: { resumePrefetch?: boolean }): void {
   const timedPayload = withCurrentSubtitleTiming(payload);
   const currentSubtitleData = appState.currentSubtitleData;
   const isAnnotationUpgrade = isSubtitleAnnotationUpgrade(currentSubtitleData, timedPayload);
@@ -1833,7 +1833,12 @@ function emitSubtitlePayload(payload: SubtitleData): void {
   }
   annotationSubtitleWsService.broadcast(timedPayload, frequencyOptions);
   autoplayReadyGate.maybeSignalPluginAutoplayReady(timedPayload, { forceWhilePaused: true });
-  subtitlePrefetchService?.resume();
+  // resumePrefetch: false marks a provisional pre-tokenization emit; prefetch
+  // stays paused until the tokenized payload for the line lands so it does not
+  // compete with the on-screen line for the single Yomitan parser window.
+  if (options?.resumePrefetch !== false) {
+    subtitlePrefetchService?.resume();
+  }
 }
 function getCurrentAutoplaySubtitlePayload(): SubtitleData | null {
   const payload = appState.currentSubtitleData;
@@ -1926,7 +1931,7 @@ const autoplaySubtitlePrimingRuntime = createAutoplaySubtitlePrimingRuntime({
     appState.activeParsedSubtitleMediaPath = mediaPath;
   },
   subtitleProcessingController,
-  emitSubtitlePayload: (payload) => emitSubtitlePayload(payload),
+  emitSubtitlePayload: (payload, options) => emitSubtitlePayload(payload, options),
   getSubtitlePrefetchService: () => subtitlePrefetchService,
   getLastObservedTimePos: () => lastObservedTimePos,
   getVisibleOverlayVisible: () => overlayManager.getVisibleOverlayVisible(),
@@ -4372,8 +4377,9 @@ const {
       emitSubtitlePayload(payload);
     },
     onSubtitleChange: (text) => {
+      // Pause only; restarting the prefetch run here would discard in-flight
+      // tokenization work on every line. Real seeks restart via onTimePosUpdate.
       subtitlePrefetchService?.pause();
-      subtitlePrefetchService?.onSeek(lastObservedTimePos);
       subtitleProcessingController.onSubtitleChange(text);
     },
     refreshDiscordPresence: () => {

src/main/main-wiring.test.ts

@@ -223,7 +223,7 @@ test('update overlay notification action triggers install flow', () => {
   assert.match(runtimeSource, /fallbackClient\.openNoteInBrowser\(noteId\)/);
 });
 
-test('subtitle change re-prioritizes prefetch around live playback before tokenizing current line', () => {
+test('subtitle change pauses prefetch without restarting its run before tokenizing current line', () => {
   const source = readMainSource();
   const actionBlock = source.match(
     /onSubtitleChange:\s*\(text\)\s*=>\s*\{(?<body>[\s\S]*?)\n    \},\n    refreshDiscordPresence:/,
@@ -231,14 +231,12 @@ test('subtitle change re-prioritizes prefetch around live playback before tokeni
 
   assert.ok(actionBlock);
   assert.match(actionBlock, /subtitlePrefetchService\?\.pause\(\);/);
-  assert.match(actionBlock, /subtitlePrefetchService\?\.onSeek\(lastObservedTimePos\);/);
+  // Restarting the run per line (onSeek) discards in-flight prefetch work;
+  // only real seeks restart via onTimePosUpdate.
+  assert.doesNotMatch(actionBlock, /subtitlePrefetchService\?\.onSeek\(/);
   assert.match(actionBlock, /subtitleProcessingController\.onSubtitleChange\(text\);/);
   assert.ok(
     actionBlock.indexOf('subtitlePrefetchService?.pause();') <
-      actionBlock.indexOf('subtitlePrefetchService?.onSeek(lastObservedTimePos);'),
-  );
-  assert.ok(
-    actionBlock.indexOf('subtitlePrefetchService?.onSeek(lastObservedTimePos);') <
       actionBlock.indexOf('subtitleProcessingController.onSubtitleChange(text);'),
   );
 });
@@ -593,7 +591,7 @@ test('YouTube media cache lifecycle routes through configured status notificatio
 test('subtitle broadcasts share one frequency options snapshot per emitted payload', () => {
   const source = readMainSource();
   const emitBlock = source.match(
-    /function emitSubtitlePayload\(payload: SubtitleData\): void \{(?<body>[\s\S]*?)\n\}/,
+    /function emitSubtitlePayload\([\s\S]*?\): void \{(?<body>[\s\S]*?)\n\}/,
   )?.groups?.body;
   const frequencyOptionsSnapshot = emitBlock?.match(
     /const frequencyDictionary = configService\.getConfig\(\)\.subtitleStyle\.frequencyDictionary;(?<body>[\s\S]*?)\n  \};/,

src/main/runtime/autoplay-subtitle-priming-runtime.test.ts

@@ -96,10 +96,10 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su
       onSubtitleChange: (text) => calls.push(`change:${text}`),
       refreshCurrentSubtitle: (text) => calls.push(`refresh:${text ?? ''}`),
     },
-    emitSubtitlePayload: (payload) => calls.push(`emit:${payload.text}`),
+    emitSubtitlePayload: (payload, options) =>
+      calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`),
     getSubtitlePrefetchService: () => ({
       pause: () => calls.push('prefetch:pause'),
-      onSeek: (timePos) => calls.push(`prefetch:seek:${timePos}`),
     }),
     getLastObservedTimePos: () => 12,
     getVisibleOverlayVisible: () => true,
@@ -120,7 +120,7 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su
     'request:time-pos',
     'set:起動字幕',
     'prefetch:pause',
-    'emit:起動字幕',
+    'emit:起動字幕:resume=false',
     'change:起動字幕',
   ]);
 });
@@ -154,10 +154,10 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before
       onSubtitleChange: (text) => calls.push(`change:${text}`),
       refreshCurrentSubtitle: (text) => calls.push(`refresh:${text ?? ''}`),
     },
-    emitSubtitlePayload: (payload) => calls.push(`emit:${payload.text}`),
+    emitSubtitlePayload: (payload, options) =>
+      calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`),
     getSubtitlePrefetchService: () => ({
       pause: () => calls.push('prefetch:pause'),
-      onSeek: (timePos) => calls.push(`prefetch:seek:${timePos}`),
     }),
     getLastObservedTimePos: () => 12,
     getVisibleOverlayVisible: () => true,
@@ -175,7 +175,7 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before
     'request:sub-text',
     'set:起動字幕',
     'prefetch:pause',
-    'emit:起動字幕',
+    'emit:起動字幕:resume=false',
     'change:起動字幕',
   ]);
 });

src/main/runtime/autoplay-subtitle-priming-runtime.ts

@@ -17,7 +17,6 @@ type AutoplaySubtitlePrimingMpvClient = {
 
 type AutoplaySubtitlePrimingPrefetchService = {
   pause: () => void;
-  onSeek: (timePos: number) => void;
 };
 
 export interface AutoplaySubtitlePrimingRuntimeDeps {
@@ -33,7 +32,7 @@ export interface AutoplaySubtitlePrimingRuntimeDeps {
     onSubtitleChange: (text: string) => void;
     refreshCurrentSubtitle: (text: string) => void;
   };
-  emitSubtitlePayload: (payload: SubtitleData) => void;
+  emitSubtitlePayload: (payload: SubtitleData, options?: { resumePrefetch?: boolean }) => void;
   getSubtitlePrefetchService: () => AutoplaySubtitlePrimingPrefetchService | null;
   getLastObservedTimePos: () => number;
   getVisibleOverlayVisible: () => boolean;
@@ -108,7 +107,9 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
       return true;
     }
 
-    emitSubtitlePayload({ text, tokens: null });
+    // Provisional raw emit: keep prefetch paused until the tokenized payload
+    // for this line is delivered by the processing controller.
+    emitSubtitlePayload({ text, tokens: null }, { resumePrefetch: false });
     subtitleProcessingController.onSubtitleChange(text);
     return true;
   }
@@ -154,12 +155,10 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
       consumeCachedSubtitle: (text) => subtitleProcessingController.consumeCachedSubtitle(text),
       onSubtitleChange: (text) => {
         deps.getSubtitlePrefetchService()?.pause();
-        deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
         subtitleProcessingController.onSubtitleChange(text);
       },
       refreshCurrentSubtitle: (text) => {
         deps.getSubtitlePrefetchService()?.pause();
-        deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
         subtitleProcessingController.refreshCurrentSubtitle(text);
       },
       deferUncachedRefresh: true,
@@ -205,7 +204,6 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
         return;
       }
       deps.getSubtitlePrefetchService()?.pause();
-      deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
       subtitleProcessingController.refreshCurrentSubtitle(text);
     }, VISIBLE_OVERLAY_SUBTITLE_REFRESH_AFTER_FIRST_PAINT_DELAY_MS);
     visibleOverlaySubtitleRefreshAfterFirstPaintTimer.unref?.();

src/main/runtime/composers/mpv-runtime-composer.ts

@@ -225,24 +225,35 @@ export function composeMpvRuntimeHandlers<
     }
     return tokenizationWarmupInFlight;
   };
+  // Built once and reused for every tokenization: per-call rebuilds create
+  // fresh closures, which defeats identity-keyed caches downstream (the JLPT
+  // lookup cache keys on the getJlptLevel function, and the mecab availability
+  // WeakSet keys on the runtime deps instance).
+  let cachedTokenizerRuntimeDeps: TTokenizerRuntimeDeps | null = null;
+  const getTokenizerRuntimeDeps = (): TTokenizerRuntimeDeps => {
+    if (cachedTokenizerRuntimeDeps) {
+      return cachedTokenizerRuntimeDeps;
+    }
+    const tokenizerMainDeps = buildTokenizerDepsHandler();
+    const baseOnTokenizationReady = tokenizerMainDeps.onTokenizationReady;
+    tokenizerMainDeps.onTokenizationReady = (tokenizedText: string): void => {
+      if (!shouldWarmupAnnotationDictionaries()) {
+        baseOnTokenizationReady?.(tokenizedText);
+        return;
+      }
+      markTokenizationPlaybackReady();
+      baseOnTokenizationReady?.(tokenizedText);
+      if (!tokenizationWarmupCompleted) {
+        void prewarmSubtitleDictionaries({ showLoadingOsd: true }).catch(() => {});
+      }
+    };
+    cachedTokenizerRuntimeDeps = options.tokenizer.createTokenizerRuntimeDeps(tokenizerMainDeps);
+    return cachedTokenizerRuntimeDeps;
+  };
   const tokenizeSubtitle = async (text: string): Promise<TTokenizedSubtitle> => {
     if (!tokenizationWarmupCompleted) void startTokenizationWarmups();
     await ensureTokenizationPrerequisites();
-    const tokenizerMainDeps = buildTokenizerDepsHandler();
+    return options.tokenizer.tokenizeSubtitle(text, getTokenizerRuntimeDeps());
-    if (shouldWarmupAnnotationDictionaries()) {
-      const onTokenizationReady = tokenizerMainDeps.onTokenizationReady;
-      tokenizerMainDeps.onTokenizationReady = (tokenizedText: string): void => {
-        markTokenizationPlaybackReady();
-        onTokenizationReady?.(tokenizedText);
-        if (!tokenizationWarmupCompleted) {
-          void prewarmSubtitleDictionaries({ showLoadingOsd: true }).catch(() => {});
-        }
-      };
-    }
-    return options.tokenizer.tokenizeSubtitle(
-      text,
-      options.tokenizer.createTokenizerRuntimeDeps(tokenizerMainDeps),
-    );
   };
 
   const launchBackgroundWarmupTask = createLaunchBackgroundWarmupTaskFromStartup(