perf(tokenizer): single-pass Yomitan scan with install-once runtime and cross-line cache

- drop the duplicate parseText full parse per line; the termsFind scanner walk
  is now authoritative and emits its own unparsed filler runs (parseText kept
  only as error fallback)
- install scan helpers once per parser window (__subminerYomitanScan) instead
  of re-shipping ~500 lines of script per subtitle line
- persist termsFind results across lines in a window-scoped LRU keyed by
  substring, invalidated via a cache epoch on dictionary/settings changes
- skip lookups at punctuation/whitespace positions and cap the shrinking-window
  retry ladder at 4 lookups per position
- build tokenizer runtime deps once (JLPT lookup cache never hit before; mecab
  availability check ran per line)
- stop restarting the prefetch run on every subtitle change; resume prefetch
  only after the tokenized payload lands, not on provisional raw emits
- add per-stage debug timings (scanMs/mecabMs/frequencyMs/annotateMs)
This commit is contained in:
2026-08-03 22:33:01 -07:00
parent fe4dacc1e7
commit e7cef039f3
11 changed files with 959 additions and 1034 deletions
@@ -17,7 +17,6 @@ type AutoplaySubtitlePrimingMpvClient = {
type AutoplaySubtitlePrimingPrefetchService = {
pause: () => void;
onSeek: (timePos: number) => void;
};
export interface AutoplaySubtitlePrimingRuntimeDeps {
@@ -33,7 +32,7 @@ export interface AutoplaySubtitlePrimingRuntimeDeps {
onSubtitleChange: (text: string) => void;
refreshCurrentSubtitle: (text: string) => void;
};
emitSubtitlePayload: (payload: SubtitleData) => void;
emitSubtitlePayload: (payload: SubtitleData, options?: { resumePrefetch?: boolean }) => void;
getSubtitlePrefetchService: () => AutoplaySubtitlePrimingPrefetchService | null;
getLastObservedTimePos: () => number;
getVisibleOverlayVisible: () => boolean;
@@ -108,7 +107,9 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
return true;
}
emitSubtitlePayload({ text, tokens: null });
// Provisional raw emit: keep prefetch paused until the tokenized payload
// for this line is delivered by the processing controller.
emitSubtitlePayload({ text, tokens: null }, { resumePrefetch: false });
subtitleProcessingController.onSubtitleChange(text);
return true;
}
@@ -154,12 +155,10 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
consumeCachedSubtitle: (text) => subtitleProcessingController.consumeCachedSubtitle(text),
onSubtitleChange: (text) => {
deps.getSubtitlePrefetchService()?.pause();
deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
subtitleProcessingController.onSubtitleChange(text);
},
refreshCurrentSubtitle: (text) => {
deps.getSubtitlePrefetchService()?.pause();
deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
subtitleProcessingController.refreshCurrentSubtitle(text);
},
deferUncachedRefresh: true,
@@ -205,7 +204,6 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
return;
}
deps.getSubtitlePrefetchService()?.pause();
deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
subtitleProcessingController.refreshCurrentSubtitle(text);
}, VISIBLE_OVERLAY_SUBTITLE_REFRESH_AFTER_FIRST_PAINT_DELAY_MS);
visibleOverlaySubtitleRefreshAfterFirstPaintTimer.unref?.();