sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-08-05 19:21:35 -07:00
Code Issues Packages Projects Releases Wiki Activity

perf(tokenizer): single-pass Yomitan scan with install-once runtime and cross-line cache

Browse Source 
- drop the duplicate parseText full parse per line; the termsFind scanner walk
  is now authoritative and emits its own unparsed filler runs (parseText kept
  only as error fallback)
- install scan helpers once per parser window (__subminerYomitanScan) instead
  of re-shipping ~500 lines of script per subtitle line
- persist termsFind results across lines in a window-scoped LRU keyed by
  substring, invalidated via a cache epoch on dictionary/settings changes
- skip lookups at punctuation/whitespace positions and cap the shrinking-window
  retry ladder at 4 lookups per position
- build tokenizer runtime deps once (JLPT lookup cache never hit before; mecab
  availability check ran per line)
- stop restarting the prefetch run on every subtitle change; resume prefetch
  only after the tokenized payload lands, not on provisional raw emits
- add per-stage debug timings (scanMs/mecabMs/frequencyMs/annotateMs)
This commit is contained in:
Kyle Yasuda
2026-08-03 22:33:01 -07:00
parent fe4dacc1e7
commit e7cef039f3
11 changed files with 959 additions and 1034 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/main-wiring.test.ts
+5 -7
View File
@@ -223,7 +223,7 @@ test('update overlay notification action triggers install flow', () => {
assert.match(runtimeSource, /fallbackClient\.openNoteInBrowser\(noteId\)/);
});
test('subtitle change re-prioritizes prefetch around live playback before tokenizing current line', () => {
test('subtitle change pauses prefetch without restarting its run before tokenizing current line', () => {
const source = readMainSource();
const actionBlock = source.match(
/onSubtitleChange:\s*\(text\)\s*=>\s*\{(?<body>[\s\S]*?)\n \},\n refreshDiscordPresence:/,
@@ -231,14 +231,12 @@ test('subtitle change re-prioritizes prefetch around live playback before tokeni
assert.ok(actionBlock);
assert.match(actionBlock, /subtitlePrefetchService\?\.pause\(\);/);
assert.match(actionBlock, /subtitlePrefetchService\?\.onSeek\(lastObservedTimePos\);/);
// Restarting the run per line (onSeek) discards in-flight prefetch work;
// only real seeks restart via onTimePosUpdate.
assert.doesNotMatch(actionBlock, /subtitlePrefetchService\?\.onSeek\(/);
assert.match(actionBlock, /subtitleProcessingController\.onSubtitleChange\(text\);/);
assert.ok(
actionBlock.indexOf('subtitlePrefetchService?.pause();') <
actionBlock.indexOf('subtitlePrefetchService?.onSeek(lastObservedTimePos);'),
);
assert.ok(
actionBlock.indexOf('subtitlePrefetchService?.onSeek(lastObservedTimePos);') <
actionBlock.indexOf('subtitleProcessingController.onSubtitleChange(text);'),
);
});
@@ -593,7 +591,7 @@ test('YouTube media cache lifecycle routes through configured status notificatio
test('subtitle broadcasts share one frequency options snapshot per emitted payload', () => {
const source = readMainSource();
const emitBlock = source.match(
/function emitSubtitlePayload\(payload: SubtitleData\): void \{(?<body>[\s\S]*?)\n\}/,
/function emitSubtitlePayload\([\s\S]*?\): void \{(?<body>[\s\S]*?)\n\}/,
)?.groups?.body;
const frequencyOptionsSnapshot = emitBlock?.match(
/const frequencyDictionary = configService\.getConfig\(\)\.subtitleStyle\.frequencyDictionary;(?<body>[\s\S]*?)\n \};/,
src/main/runtime/autoplay-subtitle-priming-runtime.test.ts
+6 -6
View File
@@ -96,10 +96,10 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su
onSubtitleChange: (text) => calls.push(`change:${text}`),
refreshCurrentSubtitle: (text) => calls.push(`refresh:${text ?? ''}`),
},
emitSubtitlePayload: (payload) => calls.push(`emit:${payload.text}`),
emitSubtitlePayload: (payload, options) =>
calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`),
getSubtitlePrefetchService: () => ({
pause: () => calls.push('prefetch:pause'),
onSeek: (timePos) => calls.push(`prefetch:seek:${timePos}`),
}),
getLastObservedTimePos: () => 12,
getVisibleOverlayVisible: () => true,
@@ -120,7 +120,7 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su
'request:time-pos',
'set:起動字幕',
'prefetch:pause',
'emit:起動字幕',
'emit:起動字幕:resume=false',
'change:起動字幕',
]);
});
@@ -154,10 +154,10 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before
onSubtitleChange: (text) => calls.push(`change:${text}`),
refreshCurrentSubtitle: (text) => calls.push(`refresh:${text ?? ''}`),
},
emitSubtitlePayload: (payload) => calls.push(`emit:${payload.text}`),
emitSubtitlePayload: (payload, options) =>
calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`),
getSubtitlePrefetchService: () => ({
pause: () => calls.push('prefetch:pause'),
onSeek: (timePos) => calls.push(`prefetch:seek:${timePos}`),
}),
getLastObservedTimePos: () => 12,
getVisibleOverlayVisible: () => true,
@@ -175,7 +175,7 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before
'request:sub-text',
'set:起動字幕',
'prefetch:pause',
'emit:起動字幕',
'emit:起動字幕:resume=false',
'change:起動字幕',
]);
});
src/main/runtime/autoplay-subtitle-priming-runtime.ts
+4 -6
View File
@@ -17,7 +17,6 @@ type AutoplaySubtitlePrimingMpvClient = {
type AutoplaySubtitlePrimingPrefetchService = {
pause: () => void;
onSeek: (timePos: number) => void;
};
export interface AutoplaySubtitlePrimingRuntimeDeps {
@@ -33,7 +32,7 @@ export interface AutoplaySubtitlePrimingRuntimeDeps {
onSubtitleChange: (text: string) => void;
refreshCurrentSubtitle: (text: string) => void;
};
emitSubtitlePayload: (payload: SubtitleData) => void;
emitSubtitlePayload: (payload: SubtitleData, options?: { resumePrefetch?: boolean }) => void;
getSubtitlePrefetchService: () => AutoplaySubtitlePrimingPrefetchService | null;
getLastObservedTimePos: () => number;
getVisibleOverlayVisible: () => boolean;
@@ -108,7 +107,9 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
return true;
}
emitSubtitlePayload({ text, tokens: null });
// Provisional raw emit: keep prefetch paused until the tokenized payload
// for this line is delivered by the processing controller.
emitSubtitlePayload({ text, tokens: null }, { resumePrefetch: false });
subtitleProcessingController.onSubtitleChange(text);
return true;
}
@@ -154,12 +155,10 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
consumeCachedSubtitle: (text) => subtitleProcessingController.consumeCachedSubtitle(text),
onSubtitleChange: (text) => {
deps.getSubtitlePrefetchService()?.pause();
deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
subtitleProcessingController.onSubtitleChange(text);
},
refreshCurrentSubtitle: (text) => {
deps.getSubtitlePrefetchService()?.pause();
deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
subtitleProcessingController.refreshCurrentSubtitle(text);
},
deferUncachedRefresh: true,
@@ -205,7 +204,6 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
return;
}
deps.getSubtitlePrefetchService()?.pause();
deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
subtitleProcessingController.refreshCurrentSubtitle(text);
}, VISIBLE_OVERLAY_SUBTITLE_REFRESH_AFTER_FIRST_PAINT_DELAY_MS);
visibleOverlaySubtitleRefreshAfterFirstPaintTimer.unref?.();
src/main/runtime/composers/mpv-runtime-composer.ts
+26 -15
View File
@@ -225,24 +225,35 @@ export function composeMpvRuntimeHandlers<
}
return tokenizationWarmupInFlight;
};
// Built once and reused for every tokenization: per-call rebuilds create
// fresh closures, which defeats identity-keyed caches downstream (the JLPT
// lookup cache keys on the getJlptLevel function, and the mecab availability
// WeakSet keys on the runtime deps instance).
let cachedTokenizerRuntimeDeps: TTokenizerRuntimeDeps | null = null;
const getTokenizerRuntimeDeps = (): TTokenizerRuntimeDeps => {
if (cachedTokenizerRuntimeDeps) {
return cachedTokenizerRuntimeDeps;
}
const tokenizerMainDeps = buildTokenizerDepsHandler();
const baseOnTokenizationReady = tokenizerMainDeps.onTokenizationReady;
tokenizerMainDeps.onTokenizationReady = (tokenizedText: string): void => {
if (!shouldWarmupAnnotationDictionaries()) {
baseOnTokenizationReady?.(tokenizedText);
return;
}
markTokenizationPlaybackReady();
baseOnTokenizationReady?.(tokenizedText);
if (!tokenizationWarmupCompleted) {
void prewarmSubtitleDictionaries({ showLoadingOsd: true }).catch(() => {});
}
};
cachedTokenizerRuntimeDeps = options.tokenizer.createTokenizerRuntimeDeps(tokenizerMainDeps);
return cachedTokenizerRuntimeDeps;
};
const tokenizeSubtitle = async (text: string): Promise<TTokenizedSubtitle> => {
if (!tokenizationWarmupCompleted) void startTokenizationWarmups();
await ensureTokenizationPrerequisites();
const tokenizerMainDeps = buildTokenizerDepsHandler();
if (shouldWarmupAnnotationDictionaries()) {
const onTokenizationReady = tokenizerMainDeps.onTokenizationReady;
tokenizerMainDeps.onTokenizationReady = (tokenizedText: string): void => {
markTokenizationPlaybackReady();
onTokenizationReady?.(tokenizedText);
if (!tokenizationWarmupCompleted) {
void prewarmSubtitleDictionaries({ showLoadingOsd: true }).catch(() => {});
}
};
}
return options.tokenizer.tokenizeSubtitle(
text,
options.tokenizer.createTokenizerRuntimeDeps(tokenizerMainDeps),
);
return options.tokenizer.tokenizeSubtitle(text, getTokenizerRuntimeDeps());
};
const launchBackgroundWarmupTask = createLaunchBackgroundWarmupTaskFromStartup(