sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-03-06 19:57:26 -08:00
Code Issues Packages Projects Releases Wiki Activity

perf(tokenizer): skip known-word lookup in MeCab POS enrichment

Browse Source
This commit is contained in:
Kyle Yasuda
2026-03-02 01:38:37 -08:00
parent cde231b1ff
commit 83f13df627
3 changed files with 52 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/core/services/tokenizer.ts
View File

@@ -211,8 +211,7 @@ export function createTokenizerDepsRuntime(
return null;
}
const isKnownWordLookup = options.getNPlusOneEnabled?.() === false ? () => false : options.isKnownWord;
return mergeTokens(rawTokens, isKnownWordLookup, options.getKnownWordMatchMode());
return mergeTokens(rawTokens, options.isKnownWord, options.getKnownWordMatchMode(), false);
},
enrichTokensWithMecab: async (tokens, mecabTokens) =>
enrichTokensWithMecabAsync(tokens, mecabTokens),