perf(tokenizer): single-pass Yomitan scan with install-once runtime and cross-line cache

- drop the duplicate parseText full parse per line; the termsFind scanner walk
  is now authoritative and emits its own unparsed filler runs (parseText kept
  only as error fallback)
- install scan helpers once per parser window (__subminerYomitanScan) instead
  of re-shipping ~500 lines of script per subtitle line
- persist termsFind results across lines in a window-scoped LRU keyed by
  substring, invalidated via a cache epoch on dictionary/settings changes
- skip lookups at punctuation/whitespace positions and cap the shrinking-window
  retry ladder at 4 lookups per position
- build tokenizer runtime deps once (JLPT lookup cache never hit before; mecab
  availability check ran per line)
- stop restarting the prefetch run on every subtitle change; resume prefetch
  only after the tokenized payload lands, not on provisional raw emits
- add per-stage debug timings (scanMs/mecabMs/frequencyMs/annotateMs)
This commit is contained in:
2026-08-03 22:33:01 -07:00
parent fe4dacc1e7
commit e7cef039f3
11 changed files with 959 additions and 1034 deletions
@@ -96,10 +96,10 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su
onSubtitleChange: (text) => calls.push(`change:${text}`),
refreshCurrentSubtitle: (text) => calls.push(`refresh:${text ?? ''}`),
},
emitSubtitlePayload: (payload) => calls.push(`emit:${payload.text}`),
emitSubtitlePayload: (payload, options) =>
calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`),
getSubtitlePrefetchService: () => ({
pause: () => calls.push('prefetch:pause'),
onSeek: (timePos) => calls.push(`prefetch:seek:${timePos}`),
}),
getLastObservedTimePos: () => 12,
getVisibleOverlayVisible: () => true,
@@ -120,7 +120,7 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su
'request:time-pos',
'set:起動字幕',
'prefetch:pause',
'emit:起動字幕',
'emit:起動字幕:resume=false',
'change:起動字幕',
]);
});
@@ -154,10 +154,10 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before
onSubtitleChange: (text) => calls.push(`change:${text}`),
refreshCurrentSubtitle: (text) => calls.push(`refresh:${text ?? ''}`),
},
emitSubtitlePayload: (payload) => calls.push(`emit:${payload.text}`),
emitSubtitlePayload: (payload, options) =>
calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`),
getSubtitlePrefetchService: () => ({
pause: () => calls.push('prefetch:pause'),
onSeek: (timePos) => calls.push(`prefetch:seek:${timePos}`),
}),
getLastObservedTimePos: () => 12,
getVisibleOverlayVisible: () => true,
@@ -175,7 +175,7 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before
'request:sub-text',
'set:起動字幕',
'prefetch:pause',
'emit:起動字幕',
'emit:起動字幕:resume=false',
'change:起動字幕',
]);
});