mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-28 04:49:49 -07:00
fix(tokenizer): block kanji reading collisions and name/generic length t
- Add allowReadingOnlyMatch:false to kanji token known-word lookups so 渓谷/けいこく no longer matches a mined 警告/けいこく card via reading-only index - Greedy name pre-pass yields when a strictly longer generic word starts at the same position (空 no longer splits 空気; ties still go to the name)
This commit is contained in:
@@ -37,8 +37,8 @@ export function createBuildTokenizerDepsMainHandler(deps: TokenizerMainDeps) {
|
||||
getYomitanParserInitPromise: () => deps.getYomitanParserInitPromise(),
|
||||
setYomitanParserInitPromise: (promise: Promise<boolean> | null) =>
|
||||
deps.setYomitanParserInitPromise(promise),
|
||||
isKnownWord: (text: string, reading?: string) => {
|
||||
const hit = deps.isKnownWord(text, reading);
|
||||
isKnownWord: (text, reading, options) => {
|
||||
const hit = deps.isKnownWord(text, reading, options);
|
||||
deps.recordLookup(hit);
|
||||
return hit;
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user