mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-02-27 18:22:41 -08:00
feat(tokenizer): refine Yomitan grouping and parser tooling
- map segmented Yomitan lines into single logical tokens and improve candidate selection heuristics - limit frequency lookup to selected token text with POS-based exclusions and add debug logging hook - add standalone Yomitan parser test script, deterministic utility-script shutdown, and docs/backlog updates
This commit is contained in:
@@ -911,6 +911,7 @@ async function tokenizeSubtitle(text: string): Promise<SubtitleData> {
|
||||
getFrequencyDictionaryEnabled: () =>
|
||||
getResolvedConfig().subtitleStyle.frequencyDictionary.enabled,
|
||||
getFrequencyRank: (text) => appState.frequencyRankLookup(text),
|
||||
getYomitanGroupDebugEnabled: () => appState.overlayDebugVisualizationEnabled,
|
||||
getMecabTokenizer: () => appState.mecabTokenizer,
|
||||
}),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user