fix(tokenizer): block kanji reading collisions and name/generic length t

- Add allowReadingOnlyMatch:false to kanji token known-word lookups so 渓谷/けいこく no longer matches a mined 警告/けいこく card via reading-only index
- Greedy name pre-pass yields when a strictly longer generic word starts at the same position (空 no longer splits 空気; ties still go to the name)
This commit is contained in:
2026-07-09 01:03:58 -07:00
parent 6c251502b3
commit db4139ba0b
12 changed files with 237 additions and 15 deletions
+13 -1
View File
@@ -142,7 +142,11 @@ export class KnownWordCacheManager {
);
}
isKnownWord(text: string, reading?: string): boolean {
isKnownWord(
text: string,
reading?: string,
options?: { allowReadingOnlyMatch?: boolean },
): boolean {
if (!this.isKnownWordCacheEnabled()) {
return false;
}
@@ -163,6 +167,14 @@ export class KnownWordCacheManager {
);
}
// Callers that look up a kanji token's reading (not subtitle text) must
// opt out of the reading-only fallback: readingCounts holds readings of
// every note including kanji words, so 渓谷's けいこく would match a
// mined 警告/けいこく.
if (options?.allowReadingOnlyMatch === false) {
return false;
}
// Reading-only fallback, except for single-kana text: particles and
// interjections (よ, ね, え…) would otherwise borrow the reading of an
// unrelated note (夜「よ」, 絵「え」) and count as known.