fix(tokenizer): block kanji reading collisions and name/generic length t

- Add allowReadingOnlyMatch:false to kanji token known-word lookups so 渓谷/けいこく no longer matches a mined 警告/けいこく card via reading-only index
- Greedy name pre-pass yields when a strictly longer generic word starts at the same position (空 no longer splits 空気; ties still go to the name)
This commit is contained in:
2026-07-09 01:03:58 -07:00
parent 6c251502b3
commit db4139ba0b
12 changed files with 237 additions and 15 deletions
@@ -734,6 +734,42 @@ test('KnownWordCacheManager disambiguates known words by note reading', async ()
}
});
test('KnownWordCacheManager suppresses reading-only matches when disallowed', async () => {
const config: AnkiConnectConfig = {
fields: {
word: 'Word',
},
knownWords: {
highlightEnabled: true,
},
};
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
try {
clientState.findNotesResult = [1];
clientState.notesInfoResult = [
{
noteId: 1,
fields: {
Word: { value: '警告' },
'Word Reading': { value: 'けいこく' },
},
},
];
await manager.refresh(true);
// Reading-only match stays available for kana subtitle text…
assert.equal(manager.isKnownWord('けいこく'), true);
// …but a kanji token's reading (渓谷/けいこく) must not borrow 警告's.
assert.equal(manager.isKnownWord('けいこく', undefined, { allowReadingOnlyMatch: false }), false);
// Mined word texts still match regardless of the flag.
assert.equal(manager.isKnownWord('警告', undefined, { allowReadingOnlyMatch: false }), true);
} finally {
cleanup();
}
});
test('KnownWordCacheManager does not match single-kana text by reading alone', async () => {
const config: AnkiConnectConfig = {
fields: {