perf(tokenizer): single-pass Yomitan scan with install-once runtime and cross-line cache

- drop the duplicate parseText full parse per line; the termsFind scanner walk
  is now authoritative and emits its own unparsed filler runs (parseText kept
  only as error fallback)
- install scan helpers once per parser window (__subminerYomitanScan) instead
  of re-shipping ~500 lines of script per subtitle line
- persist termsFind results across lines in a window-scoped LRU keyed by
  substring, invalidated via a cache epoch on dictionary/settings changes
- skip lookups at punctuation/whitespace positions and cap the shrinking-window
  retry ladder at 4 lookups per position
- build tokenizer runtime deps once (JLPT lookup cache never hit before; mecab
  availability check ran per line)
- stop restarting the prefetch run on every subtitle change; resume prefetch
  only after the tokenized payload lands, not on provisional raw emits
- add per-stage debug timings (scanMs/mecabMs/frequencyMs/annotateMs)

src/core/services/tokenizer.ts

@@ -716,15 +716,29 @@ function getAnnotationOptions(deps: TokenizerServiceDeps): TokenizerAnnotationOp
   };
 }
 
+// Per-line stage durations for the pipeline debug log; every field is filled in
+// by the stage that awaits the corresponding work.
+interface TokenizationStageTimings {
+  scanMs?: number;
+  mecabMs?: number;
+  frequencyMs?: number;
+  annotateMs?: number;
+}
+
 async function parseWithYomitanInternalParser(
   text: string,
   deps: TokenizerServiceDeps,
   options: TokenizerAnnotationOptions,
+  stageTimings?: TokenizationStageTimings,
 ): Promise<MergedToken[] | null> {
+  const scanStartedAtMs = Date.now();
   const selectedTokens = await requestYomitanScanTokens(text, deps, logger, {
     includeNameMatchMetadata: options.nameMatchEnabled,
     currentCharacterDictionaryMediaId: deps.getCurrentCharacterDictionaryMediaId?.() ?? null,
   });
+  if (stageTimings) {
+    stageTimings.scanMs = Date.now() - scanStartedAtMs;
+  }
   if (!selectedTokens || selectedTokens.length === 0) {
     return null;
   }
@@ -757,6 +771,7 @@ async function parseWithYomitanInternalParser(
 
   const frequencyRankPromise: Promise<YomitanFrequencyIndex> = options.frequencyEnabled
     ? (async () => {
+        const frequencyStartedAtMs = Date.now();
         const frequencyMatchMode = options.frequencyMatchMode;
         const termReadingList = buildYomitanFrequencyTermReadingList(
           normalizedSelectedTokens,
@@ -767,12 +782,17 @@ async function parseWithYomitanInternalParser(
           deps,
           logger,
         );
-        return buildYomitanFrequencyIndex(yomitanFrequencies);
+        const frequencyIndex = buildYomitanFrequencyIndex(yomitanFrequencies);
+        if (stageTimings) {
+          stageTimings.frequencyMs = Date.now() - frequencyStartedAtMs;
+        }
+        return frequencyIndex;
       })()
     : Promise.resolve({ byPair: new Map(), byTerm: new Map() });
 
   const mecabEnrichmentPromise: Promise<MergedToken[]> = needsMecabPosEnrichment(options)
     ? (async () => {
+        const mecabStartedAtMs = Date.now();
         try {
           const mecabTokens = await deps.tokenizeWithMecab(text);
           const enrichTokensWithMecab = deps.enrichTokensWithMecab ?? enrichTokensWithMecabAsync;
@@ -786,6 +806,10 @@ async function parseWithYomitanInternalParser(
             `textLength=${text.length}`,
           );
           return normalizedSelectedTokens;
+        } finally {
+          if (stageTimings) {
+            stageTimings.mecabMs = Date.now() - mecabStartedAtMs;
+          }
         }
       })()
     : Promise.resolve(normalizedSelectedTokens);
@@ -876,15 +900,35 @@ export async function tokenizeSubtitle(
   const annotationOptions = getAnnotationOptions(deps);
   annotationOptions.sourceText = tokenizeText;
 
-  const yomitanTokens = await parseWithYomitanInternalParser(tokenizeText, deps, annotationOptions);
+  const stageTimings: TokenizationStageTimings = {};
+  const startedAtMs = Date.now();
+  const logStageTimings = (tokenCount: number): void => {
+    logger.debug(
+      `Subtitle tokenization stages; textLength=${tokenizeText.length}, tokenCount=${tokenCount}, ` +
+        `scanMs=${stageTimings.scanMs ?? '-'}, mecabMs=${stageTimings.mecabMs ?? '-'}, ` +
+        `frequencyMs=${stageTimings.frequencyMs ?? '-'}, annotateMs=${stageTimings.annotateMs ?? '-'}, ` +
+        `totalMs=${Date.now() - startedAtMs}`,
+    );
+  };
+
+  const yomitanTokens = await parseWithYomitanInternalParser(
+    tokenizeText,
+    deps,
+    annotationOptions,
+    stageTimings,
+  );
   if (yomitanTokens && yomitanTokens.length > 0) {
+    const annotateStartedAtMs = Date.now();
     const annotatedTokens = await applyAnnotationStage(yomitanTokens, deps, annotationOptions);
     const renderedTokens = applyCharacterNameImages(annotatedTokens, deps, annotationOptions);
+    stageTimings.annotateMs = Date.now() - annotateStartedAtMs;
+    logStageTimings(renderedTokens.length);
     return {
       text: displayText,
       tokens: renderedTokens.length > 0 ? renderedTokens : null,
     };
   }
 
+  logStageTimings(0);
   return { text: displayText, tokens: null };
 }