fix(tokenizer): block kanji reading collisions and name/generic length t

- Add allowReadingOnlyMatch:false to kanji token known-word lookups so 渓谷/けいこく no longer matches a mined 警告/けいこく card via reading-only index
- Greedy name pre-pass yields when a strictly longer generic word starts at the same position (空 no longer splits 空気; ties still go to the name)
This commit is contained in:
2026-07-09 01:03:58 -07:00
parent 6c251502b3
commit db4139ba0b
12 changed files with 237 additions and 15 deletions
@@ -37,8 +37,8 @@ export function createBuildTokenizerDepsMainHandler(deps: TokenizerMainDeps) {
getYomitanParserInitPromise: () => deps.getYomitanParserInitPromise(),
setYomitanParserInitPromise: (promise: Promise<boolean> | null) =>
deps.setYomitanParserInitPromise(promise),
isKnownWord: (text: string, reading?: string) => {
const hit = deps.isKnownWord(text, reading);
isKnownWord: (text, reading, options) => {
const hit = deps.isKnownWord(text, reading, options);
deps.recordLookup(hit);
return hit;
},