sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-03-06 19:57:26 -08:00
Code Issues Packages Projects Releases Wiki Activity

fix: unblock autoplay on tokenization-ready and defer annotation loading

Browse Source
This commit is contained in:
Kyle Yasuda
2026-03-02 02:43:09 -08:00
parent 5167e3a494
commit e744fab067
9 changed files with 440 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/core/services/tokenizer.ts
View File

@@ -51,6 +51,7 @@ export interface TokenizerServiceDeps {
getYomitanGroupDebugEnabled?: () => boolean;
tokenizeWithMecab: (text: string) => Promise<MergedToken[] | null>;
enrichTokensWithMecab?: MecabTokenEnrichmentFn;
onTokenizationReady?: (text: string) => void;
}
interface MecabTokenizerLike {
@@ -78,6 +79,7 @@ export interface TokenizerDepsRuntimeOptions {
getMinSentenceWordsForNPlusOne?: () => number;
getYomitanGroupDebugEnabled?: () => boolean;
getMecabTokenizer: () => MecabTokenizerLike | null;
onTokenizationReady?: (text: string) => void;
}
interface TokenizerAnnotationOptions {
@@ -215,6 +217,7 @@ export function createTokenizerDepsRuntime(
},
enrichTokensWithMecab: async (tokens, mecabTokens) =>
enrichTokensWithMecabAsync(tokens, mecabTokens),
onTokenizationReady: options.onTokenizationReady,
};
}
@@ -477,6 +480,7 @@ async function parseWithYomitanInternalParser(
if (deps.getYomitanGroupDebugEnabled?.() === true) {
logSelectedYomitanGroups(text, normalizedSelectedTokens);
}
deps.onTokenizationReady?.(text);
const frequencyRankPromise: Promise<Map<string, number>> = options.frequencyEnabled
? (async () => {