perf(tokenizer): single-pass Yomitan scan with cross-line caching and prefetch fixes (#185)

This commit is contained in:
2026-08-06 21:44:09 -07:00
committed by GitHub
parent 441ecf3c04
commit dbdf578c68
38 changed files with 4073 additions and 1720 deletions
@@ -9,6 +9,9 @@ type TokenizerMainDeps = TokenizerDepsRuntimeOptions & {
getCurrentCharacterDictionaryMediaId?: NonNullable<
TokenizerDepsRuntimeOptions['getCurrentCharacterDictionaryMediaId']
>;
getCharacterNameCandidates?: NonNullable<
TokenizerDepsRuntimeOptions['getCharacterNameCandidates']
>;
getFrequencyDictionaryEnabled: NonNullable<
TokenizerDepsRuntimeOptions['getFrequencyDictionaryEnabled']
>;
@@ -84,6 +87,11 @@ export function createBuildTokenizerDepsMainHandler(deps: TokenizerMainDeps) {
getCurrentCharacterDictionaryMediaId: () => deps.getCurrentCharacterDictionaryMediaId!(),
}
: {}),
...(deps.getCharacterNameCandidates
? {
getCharacterNameCandidates: () => deps.getCharacterNameCandidates!(),
}
: {}),
getFrequencyDictionaryEnabled: () => deps.getFrequencyDictionaryEnabled(),
getFrequencyDictionaryMatchMode: () => deps.getFrequencyDictionaryMatchMode(),
getFrequencyRank: (text: string) => deps.getFrequencyRank(text),