sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-06-13 15:13:32 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat(config): add configuration window (#70)

Browse Source
This commit is contained in:
Kyle Yasuda
2026-05-21 04:16:21 -07:00
committed by GitHub
parent a54f03f0cd
commit dc52bc2fba
287 changed files with 14507 additions and 8134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/services/tokenizer.ts
+22 -4
View File
@@ -43,6 +43,7 @@ export interface TokenizerServiceDeps {
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
isKnownWord: (text: string) => boolean;
getKnownWordMatchMode: () => NPlusOneMatchMode;
getKnownWordsEnabled?: () => boolean;
getJlptLevel: (text: string) => JlptLevel | null;
getNPlusOneEnabled?: () => boolean;
getJlptEnabled?: () => boolean;
@@ -74,6 +75,7 @@ export interface TokenizerDepsRuntimeOptions {
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
isKnownWord: (text: string) => boolean;
getKnownWordMatchMode: () => NPlusOneMatchMode;
getKnownWordsEnabled?: () => boolean;
getJlptLevel: (text: string) => JlptLevel | null;
getNPlusOneEnabled?: () => boolean;
getJlptEnabled?: () => boolean;
@@ -88,6 +90,7 @@ export interface TokenizerDepsRuntimeOptions {
}
interface TokenizerAnnotationOptions {
knownWordsEnabled: boolean;
nPlusOneEnabled: boolean;
jlptEnabled: boolean;
nameMatchEnabled: boolean;
@@ -119,18 +122,28 @@ function getKnownWordLookup(
deps: TokenizerServiceDeps,
options: TokenizerAnnotationOptions,
): (text: string) => boolean {
if (!options.nPlusOneEnabled) {
if (!options.knownWordsEnabled && !options.nPlusOneEnabled) {
return () => false;
}
return deps.isKnownWord;
}
function needsMecabPosEnrichment(options: TokenizerAnnotationOptions): boolean {
return options.nPlusOneEnabled || options.jlptEnabled || options.frequencyEnabled;
return (
options.knownWordsEnabled ||
options.nPlusOneEnabled ||
options.jlptEnabled ||
options.frequencyEnabled
);
}
function hasAnyAnnotationEnabled(options: TokenizerAnnotationOptions): boolean {
return options.nPlusOneEnabled || options.jlptEnabled || options.frequencyEnabled;
return (
options.knownWordsEnabled ||
options.nPlusOneEnabled ||
options.jlptEnabled ||
options.frequencyEnabled
);
}
async function enrichTokensWithMecabAsync(
@@ -211,6 +224,7 @@ export function createTokenizerDepsRuntime(
setYomitanParserInitPromise: options.setYomitanParserInitPromise,
isKnownWord: options.isKnownWord,
getKnownWordMatchMode: options.getKnownWordMatchMode,
getKnownWordsEnabled: options.getKnownWordsEnabled,
getJlptLevel: options.getJlptLevel,
getNPlusOneEnabled: options.getNPlusOneEnabled,
getJlptEnabled: options.getJlptEnabled,
@@ -662,8 +676,12 @@ function applyFrequencyRanks(
}
function getAnnotationOptions(deps: TokenizerServiceDeps): TokenizerAnnotationOptions {
const nPlusOneEnabled = deps.getNPlusOneEnabled?.() !== false;
return {
nPlusOneEnabled: deps.getNPlusOneEnabled?.() !== false,
knownWordsEnabled: deps.getKnownWordsEnabled
? deps.getKnownWordsEnabled() !== false
: nPlusOneEnabled,
nPlusOneEnabled,
jlptEnabled: deps.getJlptEnabled?.() !== false,
nameMatchEnabled: deps.getNameMatchEnabled?.() !== false,
frequencyEnabled: deps.getFrequencyDictionaryEnabled?.() !== false,