sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-03-02 18:22:42 -08:00
Code Issues Packages Projects Releases Wiki Activity

perf(tokenizer): cut annotation latency with persistent mecab

Browse Source
This commit is contained in:
Kyle Yasuda
2026-03-02 01:15:21 -08:00
parent 11e9c721c6
commit 9a91951656
7 changed files with 650 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/runtime/composers/mpv-runtime-composer.ts
View File

@@ -133,12 +133,23 @@ export function composeMpvRuntimeHandlers<
const prewarmSubtitleDictionaries = createPrewarmSubtitleDictionariesMainHandler(
options.tokenizer.prewarmSubtitleDictionariesMainDeps,
);
const shouldInitializeMecabForAnnotations = (): boolean => {
const nPlusOneEnabled =
options.tokenizer.buildTokenizerDepsMainDeps.getNPlusOneEnabled?.() !== false;
const jlptEnabled = options.tokenizer.buildTokenizerDepsMainDeps.getJlptEnabled() !== false;
const frequencyEnabled =
options.tokenizer.buildTokenizerDepsMainDeps.getFrequencyDictionaryEnabled() !== false;
return nPlusOneEnabled || jlptEnabled || frequencyEnabled;
};
let tokenizationWarmupInFlight: Promise<void> | null = null;
const startTokenizationWarmups = (): Promise<void> => {
if (!tokenizationWarmupInFlight) {
tokenizationWarmupInFlight = (async () => {
await options.warmups.startBackgroundWarmupsMainDeps.ensureYomitanExtensionLoaded();
if (!options.tokenizer.createMecabTokenizerAndCheckMainDeps.getMecabTokenizer()) {
if (
shouldInitializeMecabForAnnotations() &&
!options.tokenizer.createMecabTokenizerAndCheckMainDeps.getMecabTokenizer()
) {
await createMecabTokenizerAndCheck().catch(() => {});
}
await prewarmSubtitleDictionaries({ showLoadingOsd: true });