mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-05 19:21:35 -07:00
b0a2ce6e8a
The greedy name pre-pass asked the Yomitan backend at every Japanese position, because a character name can begin mid-token. With the character dictionary enabled that roughly doubled the round trips per line (measured 10 -> 21 on a 23-char line). SubMiner generates the character dictionary, so the cached snapshots already list every form a character entry can be matched by (term and reading). Those forms are installed into the scan runtime once per media and the pre-pass now probes only positions where one of them starts, compared after kana normalization so a katakana name still matches a hiragana reading form. The overhead drops to zero (21 -> 10, the same as with the dictionary disabled). Fail-safe: with no candidate list (no media id, no cached snapshot, failed install) the pre-pass keeps its exhaustive behavior, so stale character data costs speed rather than a missing name. Halfwidth katakana positions bypass the filter since kana normalization does not fold them. The candidate lookup is consulted per subtitle line, so it caches its snapshot directory signature for 5s; dictionary writes still call invalidate().
216 lines
7.5 KiB
TypeScript
216 lines
7.5 KiB
TypeScript
import type { TokenizerDepsRuntimeOptions } from '../../core/services/tokenizer';
|
|
|
|
type TokenizerMainDeps = TokenizerDepsRuntimeOptions & {
|
|
getJlptEnabled: NonNullable<TokenizerDepsRuntimeOptions['getJlptEnabled']>;
|
|
getCharacterDictionaryEnabled?: () => boolean;
|
|
getNameMatchEnabled?: NonNullable<TokenizerDepsRuntimeOptions['getNameMatchEnabled']>;
|
|
getNameMatchImagesEnabled?: NonNullable<TokenizerDepsRuntimeOptions['getNameMatchImagesEnabled']>;
|
|
getCharacterNameImage?: NonNullable<TokenizerDepsRuntimeOptions['getCharacterNameImage']>;
|
|
getCurrentCharacterDictionaryMediaId?: NonNullable<
|
|
TokenizerDepsRuntimeOptions['getCurrentCharacterDictionaryMediaId']
|
|
>;
|
|
getCharacterNameCandidates?: NonNullable<
|
|
TokenizerDepsRuntimeOptions['getCharacterNameCandidates']
|
|
>;
|
|
getFrequencyDictionaryEnabled: NonNullable<
|
|
TokenizerDepsRuntimeOptions['getFrequencyDictionaryEnabled']
|
|
>;
|
|
getFrequencyDictionaryMatchMode: NonNullable<
|
|
TokenizerDepsRuntimeOptions['getFrequencyDictionaryMatchMode']
|
|
>;
|
|
getFrequencyRank: NonNullable<TokenizerDepsRuntimeOptions['getFrequencyRank']>;
|
|
getMinSentenceWordsForNPlusOne: NonNullable<
|
|
TokenizerDepsRuntimeOptions['getMinSentenceWordsForNPlusOne']
|
|
>;
|
|
getYomitanGroupDebugEnabled: NonNullable<
|
|
TokenizerDepsRuntimeOptions['getYomitanGroupDebugEnabled']
|
|
>;
|
|
recordLookup: (hit: boolean) => void;
|
|
};
|
|
|
|
export function createBuildTokenizerDepsMainHandler(deps: TokenizerMainDeps) {
|
|
return (): TokenizerDepsRuntimeOptions => ({
|
|
getYomitanExt: () => deps.getYomitanExt(),
|
|
getYomitanSession: () => deps.getYomitanSession?.() ?? null,
|
|
getYomitanParserWindow: () => deps.getYomitanParserWindow(),
|
|
setYomitanParserWindow: (window) => deps.setYomitanParserWindow(window),
|
|
getYomitanParserReadyPromise: () => deps.getYomitanParserReadyPromise(),
|
|
setYomitanParserReadyPromise: (promise: Promise<void> | null) =>
|
|
deps.setYomitanParserReadyPromise(promise),
|
|
getYomitanParserInitPromise: () => deps.getYomitanParserInitPromise(),
|
|
setYomitanParserInitPromise: (promise: Promise<boolean> | null) =>
|
|
deps.setYomitanParserInitPromise(promise),
|
|
isKnownWord: (text, reading, options) => {
|
|
const hit = deps.isKnownWord(text, reading, options);
|
|
deps.recordLookup(hit);
|
|
return hit;
|
|
},
|
|
...(deps.getKnownWordTier
|
|
? {
|
|
getKnownWordTier: (text, reading, options) =>
|
|
deps.getKnownWordTier!(text, reading, options),
|
|
}
|
|
: {}),
|
|
getKnownWordMatchMode: () => deps.getKnownWordMatchMode(),
|
|
...(deps.getKnownWordsEnabled
|
|
? {
|
|
getKnownWordsEnabled: () => deps.getKnownWordsEnabled!(),
|
|
}
|
|
: {}),
|
|
...(deps.getNPlusOneEnabled
|
|
? {
|
|
getNPlusOneEnabled: () => deps.getNPlusOneEnabled!(),
|
|
}
|
|
: {}),
|
|
getMinSentenceWordsForNPlusOne: () => deps.getMinSentenceWordsForNPlusOne(),
|
|
getJlptLevel: (text: string) => deps.getJlptLevel(text),
|
|
getJlptEnabled: () => deps.getJlptEnabled(),
|
|
...(deps.getNameMatchEnabled
|
|
? {
|
|
getNameMatchEnabled: () =>
|
|
deps.getCharacterDictionaryEnabled?.() !== false && deps.getNameMatchEnabled!(),
|
|
}
|
|
: {}),
|
|
...(deps.getNameMatchImagesEnabled
|
|
? {
|
|
getNameMatchImagesEnabled: () =>
|
|
deps.getCharacterDictionaryEnabled?.() !== false && deps.getNameMatchImagesEnabled!(),
|
|
}
|
|
: {}),
|
|
...(deps.getCharacterNameImage
|
|
? {
|
|
getCharacterNameImage: (term: string) => deps.getCharacterNameImage!(term),
|
|
}
|
|
: {}),
|
|
...(deps.getCurrentCharacterDictionaryMediaId
|
|
? {
|
|
getCurrentCharacterDictionaryMediaId: () => deps.getCurrentCharacterDictionaryMediaId!(),
|
|
}
|
|
: {}),
|
|
...(deps.getCharacterNameCandidates
|
|
? {
|
|
getCharacterNameCandidates: () => deps.getCharacterNameCandidates!(),
|
|
}
|
|
: {}),
|
|
getFrequencyDictionaryEnabled: () => deps.getFrequencyDictionaryEnabled(),
|
|
getFrequencyDictionaryMatchMode: () => deps.getFrequencyDictionaryMatchMode(),
|
|
getFrequencyRank: (text: string) => deps.getFrequencyRank(text),
|
|
getYomitanGroupDebugEnabled: () => deps.getYomitanGroupDebugEnabled(),
|
|
getMecabTokenizer: () => deps.getMecabTokenizer(),
|
|
onTokenizationReady: (text: string) => deps.onTokenizationReady?.(text),
|
|
});
|
|
}
|
|
|
|
export function createCreateMecabTokenizerAndCheckMainHandler<TMecab>(deps: {
|
|
getMecabTokenizer: () => TMecab | null;
|
|
setMecabTokenizer: (tokenizer: TMecab) => void;
|
|
createMecabTokenizer: () => TMecab;
|
|
checkAvailability: (tokenizer: TMecab) => Promise<unknown>;
|
|
}) {
|
|
return async (): Promise<void> => {
|
|
let tokenizer = deps.getMecabTokenizer();
|
|
if (!tokenizer) {
|
|
tokenizer = deps.createMecabTokenizer();
|
|
deps.setMecabTokenizer(tokenizer);
|
|
}
|
|
await deps.checkAvailability(tokenizer);
|
|
};
|
|
}
|
|
|
|
export function createPrewarmSubtitleDictionariesMainHandler(deps: {
|
|
ensureJlptDictionaryLookup: () => Promise<void>;
|
|
ensureFrequencyDictionaryLookup: () => Promise<void>;
|
|
showMpvOsd?: (message: string) => void;
|
|
showLoadingOsd?: (message: string) => void;
|
|
showLoadedOsd?: (message: string) => void;
|
|
shouldShowOsdNotification?: () => boolean;
|
|
setInterval?: (callback: () => void, delayMs: number) => unknown;
|
|
clearInterval?: (timer: unknown) => void;
|
|
}) {
|
|
let prewarmed = false;
|
|
let prewarmPromise: Promise<void> | null = null;
|
|
let loadingOsdDepth = 0;
|
|
let loadingOsdFrame = 0;
|
|
let loadingOsdTimer: unknown = null;
|
|
const showMpvOsd = deps.showMpvOsd;
|
|
const showLoadingOsd = deps.showLoadingOsd ?? showMpvOsd;
|
|
const showLoadedOsd = deps.showLoadedOsd ?? showMpvOsd;
|
|
const setIntervalHandler =
|
|
deps.setInterval ??
|
|
((callback: () => void, delayMs: number): unknown => setInterval(callback, delayMs));
|
|
const clearIntervalHandler =
|
|
deps.clearInterval ??
|
|
((timer: unknown): void => clearInterval(timer as ReturnType<typeof setInterval>));
|
|
const spinnerFrames = ['|', '/', '-', '\\'];
|
|
|
|
const beginLoadingOsd = (): boolean => {
|
|
if (!showLoadingOsd) {
|
|
return false;
|
|
}
|
|
loadingOsdDepth += 1;
|
|
if (loadingOsdDepth > 1) {
|
|
return true;
|
|
}
|
|
|
|
loadingOsdFrame = 0;
|
|
showLoadingOsd(`Loading subtitle annotations ${spinnerFrames[loadingOsdFrame]}`);
|
|
loadingOsdFrame += 1;
|
|
loadingOsdTimer = setIntervalHandler(() => {
|
|
if (!showLoadingOsd) {
|
|
return;
|
|
}
|
|
showLoadingOsd(
|
|
`Loading subtitle annotations ${spinnerFrames[loadingOsdFrame % spinnerFrames.length]}`,
|
|
);
|
|
loadingOsdFrame += 1;
|
|
}, 180);
|
|
return true;
|
|
};
|
|
|
|
const endLoadingOsd = (): void => {
|
|
if (!showLoadedOsd) {
|
|
return;
|
|
}
|
|
|
|
loadingOsdDepth = Math.max(0, loadingOsdDepth - 1);
|
|
if (loadingOsdDepth > 0) {
|
|
return;
|
|
}
|
|
|
|
if (loadingOsdTimer) {
|
|
clearIntervalHandler(loadingOsdTimer);
|
|
loadingOsdTimer = null;
|
|
}
|
|
showLoadedOsd('Subtitle annotations loaded');
|
|
};
|
|
|
|
return async (options?: { showLoadingOsd?: boolean }): Promise<void> => {
|
|
if (prewarmed) {
|
|
return;
|
|
}
|
|
const shouldTrackLoadingOsd = options?.showLoadingOsd === true;
|
|
const loadingOsdStarted = shouldTrackLoadingOsd ? beginLoadingOsd() : false;
|
|
|
|
if (!prewarmPromise) {
|
|
prewarmPromise = (async () => {
|
|
try {
|
|
await Promise.all([
|
|
deps.ensureJlptDictionaryLookup(),
|
|
deps.ensureFrequencyDictionaryLookup(),
|
|
]);
|
|
prewarmed = true;
|
|
} finally {
|
|
prewarmPromise = null;
|
|
}
|
|
})();
|
|
}
|
|
try {
|
|
await prewarmPromise;
|
|
} finally {
|
|
if (loadingOsdStarted) {
|
|
endLoadingOsd();
|
|
}
|
|
}
|
|
};
|
|
}
|