mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-29 07:21:33 -07:00
fix(tokenizer): block kanji reading collisions and name/generic length t
- Add allowReadingOnlyMatch:false to kanji token known-word lookups so 渓谷/けいこく no longer matches a mined 警告/けいこく card via reading-only index - Greedy name pre-pass yields when a strictly longer generic word starts at the same position (空 no longer splits 空気; ties still go to the name)
This commit is contained in:
@@ -142,7 +142,11 @@ export class KnownWordCacheManager {
|
||||
);
|
||||
}
|
||||
|
||||
isKnownWord(text: string, reading?: string): boolean {
|
||||
isKnownWord(
|
||||
text: string,
|
||||
reading?: string,
|
||||
options?: { allowReadingOnlyMatch?: boolean },
|
||||
): boolean {
|
||||
if (!this.isKnownWordCacheEnabled()) {
|
||||
return false;
|
||||
}
|
||||
@@ -163,6 +167,14 @@ export class KnownWordCacheManager {
|
||||
);
|
||||
}
|
||||
|
||||
// Callers that look up a kanji token's reading (not subtitle text) must
|
||||
// opt out of the reading-only fallback: readingCounts holds readings of
|
||||
// every note including kanji words, so 渓谷's けいこく would match a
|
||||
// mined 警告/けいこく.
|
||||
if (options?.allowReadingOnlyMatch === false) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Reading-only fallback, except for single-kana text: particles and
|
||||
// interjections (よ, ね, え…) would otherwise borrow the reading of an
|
||||
// unrelated note (夜「よ」, 絵「え」) and count as known.
|
||||
|
||||
Reference in New Issue
Block a user