Files
SubMiner/src/main/runtime/subtitle-tokenization-main-deps.ts
T
sudacode b0a2ce6e8a perf(tokenizer): skip character-name lookups where no name can start
The greedy name pre-pass asked the Yomitan backend at every Japanese
position, because a character name can begin mid-token. With the character
dictionary enabled that roughly doubled the round trips per line (measured
10 -> 21 on a 23-char line).

SubMiner generates the character dictionary, so the cached snapshots already
list every form a character entry can be matched by (term and reading). Those
forms are installed into the scan runtime once per media and the pre-pass now
probes only positions where one of them starts, compared after kana
normalization so a katakana name still matches a hiragana reading form. The
overhead drops to zero (21 -> 10, the same as with the dictionary disabled).

Fail-safe: with no candidate list (no media id, no cached snapshot, failed
install) the pre-pass keeps its exhaustive behavior, so stale character data
costs speed rather than a missing name. Halfwidth katakana positions bypass
the filter since kana normalization does not fold them.

The candidate lookup is consulted per subtitle line, so it caches its snapshot
directory signature for 5s; dictionary writes still call invalidate().
2026-08-04 01:56:44 -07:00

216 lines
7.5 KiB
TypeScript

import type { TokenizerDepsRuntimeOptions } from '../../core/services/tokenizer';
type TokenizerMainDeps = TokenizerDepsRuntimeOptions & {
getJlptEnabled: NonNullable<TokenizerDepsRuntimeOptions['getJlptEnabled']>;
getCharacterDictionaryEnabled?: () => boolean;
getNameMatchEnabled?: NonNullable<TokenizerDepsRuntimeOptions['getNameMatchEnabled']>;
getNameMatchImagesEnabled?: NonNullable<TokenizerDepsRuntimeOptions['getNameMatchImagesEnabled']>;
getCharacterNameImage?: NonNullable<TokenizerDepsRuntimeOptions['getCharacterNameImage']>;
getCurrentCharacterDictionaryMediaId?: NonNullable<
TokenizerDepsRuntimeOptions['getCurrentCharacterDictionaryMediaId']
>;
getCharacterNameCandidates?: NonNullable<
TokenizerDepsRuntimeOptions['getCharacterNameCandidates']
>;
getFrequencyDictionaryEnabled: NonNullable<
TokenizerDepsRuntimeOptions['getFrequencyDictionaryEnabled']
>;
getFrequencyDictionaryMatchMode: NonNullable<
TokenizerDepsRuntimeOptions['getFrequencyDictionaryMatchMode']
>;
getFrequencyRank: NonNullable<TokenizerDepsRuntimeOptions['getFrequencyRank']>;
getMinSentenceWordsForNPlusOne: NonNullable<
TokenizerDepsRuntimeOptions['getMinSentenceWordsForNPlusOne']
>;
getYomitanGroupDebugEnabled: NonNullable<
TokenizerDepsRuntimeOptions['getYomitanGroupDebugEnabled']
>;
recordLookup: (hit: boolean) => void;
};
export function createBuildTokenizerDepsMainHandler(deps: TokenizerMainDeps) {
return (): TokenizerDepsRuntimeOptions => ({
getYomitanExt: () => deps.getYomitanExt(),
getYomitanSession: () => deps.getYomitanSession?.() ?? null,
getYomitanParserWindow: () => deps.getYomitanParserWindow(),
setYomitanParserWindow: (window) => deps.setYomitanParserWindow(window),
getYomitanParserReadyPromise: () => deps.getYomitanParserReadyPromise(),
setYomitanParserReadyPromise: (promise: Promise<void> | null) =>
deps.setYomitanParserReadyPromise(promise),
getYomitanParserInitPromise: () => deps.getYomitanParserInitPromise(),
setYomitanParserInitPromise: (promise: Promise<boolean> | null) =>
deps.setYomitanParserInitPromise(promise),
isKnownWord: (text, reading, options) => {
const hit = deps.isKnownWord(text, reading, options);
deps.recordLookup(hit);
return hit;
},
...(deps.getKnownWordTier
? {
getKnownWordTier: (text, reading, options) =>
deps.getKnownWordTier!(text, reading, options),
}
: {}),
getKnownWordMatchMode: () => deps.getKnownWordMatchMode(),
...(deps.getKnownWordsEnabled
? {
getKnownWordsEnabled: () => deps.getKnownWordsEnabled!(),
}
: {}),
...(deps.getNPlusOneEnabled
? {
getNPlusOneEnabled: () => deps.getNPlusOneEnabled!(),
}
: {}),
getMinSentenceWordsForNPlusOne: () => deps.getMinSentenceWordsForNPlusOne(),
getJlptLevel: (text: string) => deps.getJlptLevel(text),
getJlptEnabled: () => deps.getJlptEnabled(),
...(deps.getNameMatchEnabled
? {
getNameMatchEnabled: () =>
deps.getCharacterDictionaryEnabled?.() !== false && deps.getNameMatchEnabled!(),
}
: {}),
...(deps.getNameMatchImagesEnabled
? {
getNameMatchImagesEnabled: () =>
deps.getCharacterDictionaryEnabled?.() !== false && deps.getNameMatchImagesEnabled!(),
}
: {}),
...(deps.getCharacterNameImage
? {
getCharacterNameImage: (term: string) => deps.getCharacterNameImage!(term),
}
: {}),
...(deps.getCurrentCharacterDictionaryMediaId
? {
getCurrentCharacterDictionaryMediaId: () => deps.getCurrentCharacterDictionaryMediaId!(),
}
: {}),
...(deps.getCharacterNameCandidates
? {
getCharacterNameCandidates: () => deps.getCharacterNameCandidates!(),
}
: {}),
getFrequencyDictionaryEnabled: () => deps.getFrequencyDictionaryEnabled(),
getFrequencyDictionaryMatchMode: () => deps.getFrequencyDictionaryMatchMode(),
getFrequencyRank: (text: string) => deps.getFrequencyRank(text),
getYomitanGroupDebugEnabled: () => deps.getYomitanGroupDebugEnabled(),
getMecabTokenizer: () => deps.getMecabTokenizer(),
onTokenizationReady: (text: string) => deps.onTokenizationReady?.(text),
});
}
export function createCreateMecabTokenizerAndCheckMainHandler<TMecab>(deps: {
getMecabTokenizer: () => TMecab | null;
setMecabTokenizer: (tokenizer: TMecab) => void;
createMecabTokenizer: () => TMecab;
checkAvailability: (tokenizer: TMecab) => Promise<unknown>;
}) {
return async (): Promise<void> => {
let tokenizer = deps.getMecabTokenizer();
if (!tokenizer) {
tokenizer = deps.createMecabTokenizer();
deps.setMecabTokenizer(tokenizer);
}
await deps.checkAvailability(tokenizer);
};
}
export function createPrewarmSubtitleDictionariesMainHandler(deps: {
ensureJlptDictionaryLookup: () => Promise<void>;
ensureFrequencyDictionaryLookup: () => Promise<void>;
showMpvOsd?: (message: string) => void;
showLoadingOsd?: (message: string) => void;
showLoadedOsd?: (message: string) => void;
shouldShowOsdNotification?: () => boolean;
setInterval?: (callback: () => void, delayMs: number) => unknown;
clearInterval?: (timer: unknown) => void;
}) {
let prewarmed = false;
let prewarmPromise: Promise<void> | null = null;
let loadingOsdDepth = 0;
let loadingOsdFrame = 0;
let loadingOsdTimer: unknown = null;
const showMpvOsd = deps.showMpvOsd;
const showLoadingOsd = deps.showLoadingOsd ?? showMpvOsd;
const showLoadedOsd = deps.showLoadedOsd ?? showMpvOsd;
const setIntervalHandler =
deps.setInterval ??
((callback: () => void, delayMs: number): unknown => setInterval(callback, delayMs));
const clearIntervalHandler =
deps.clearInterval ??
((timer: unknown): void => clearInterval(timer as ReturnType<typeof setInterval>));
const spinnerFrames = ['|', '/', '-', '\\'];
const beginLoadingOsd = (): boolean => {
if (!showLoadingOsd) {
return false;
}
loadingOsdDepth += 1;
if (loadingOsdDepth > 1) {
return true;
}
loadingOsdFrame = 0;
showLoadingOsd(`Loading subtitle annotations ${spinnerFrames[loadingOsdFrame]}`);
loadingOsdFrame += 1;
loadingOsdTimer = setIntervalHandler(() => {
if (!showLoadingOsd) {
return;
}
showLoadingOsd(
`Loading subtitle annotations ${spinnerFrames[loadingOsdFrame % spinnerFrames.length]}`,
);
loadingOsdFrame += 1;
}, 180);
return true;
};
const endLoadingOsd = (): void => {
if (!showLoadedOsd) {
return;
}
loadingOsdDepth = Math.max(0, loadingOsdDepth - 1);
if (loadingOsdDepth > 0) {
return;
}
if (loadingOsdTimer) {
clearIntervalHandler(loadingOsdTimer);
loadingOsdTimer = null;
}
showLoadedOsd('Subtitle annotations loaded');
};
return async (options?: { showLoadingOsd?: boolean }): Promise<void> => {
if (prewarmed) {
return;
}
const shouldTrackLoadingOsd = options?.showLoadingOsd === true;
const loadingOsdStarted = shouldTrackLoadingOsd ? beginLoadingOsd() : false;
if (!prewarmPromise) {
prewarmPromise = (async () => {
try {
await Promise.all([
deps.ensureJlptDictionaryLookup(),
deps.ensureFrequencyDictionaryLookup(),
]);
prewarmed = true;
} finally {
prewarmPromise = null;
}
})();
}
try {
await prewarmPromise;
} finally {
if (loadingOsdStarted) {
endLoadingOsd();
}
}
};
}