fix(tokenizer): block kanji reading collisions and name/generic length t

- Add allowReadingOnlyMatch:false to kanji token known-word lookups so 渓谷/けいこく no longer matches a mined 警告/けいこく card via reading-only index
- Greedy name pre-pass yields when a strictly longer generic word starts at the same position (空 no longer splits 空気; ties still go to the name)
This commit is contained in:
2026-07-09 01:03:58 -07:00
parent 6c251502b3
commit db4139ba0b
12 changed files with 237 additions and 15 deletions
+12 -3
View File
@@ -33,6 +33,15 @@ type MecabTokenEnrichmentFn = (
mecabTokens: MergedToken[] | null,
) => Promise<MergedToken[]>;
// allowReadingOnlyMatch: false suppresses the cache's reading-only index for
// lookups that pass a kanji token's reading as the text (see
// computeTokenKnownStatus in annotation-stage).
export type KnownWordLookupFn = (
text: string,
reading?: string,
options?: { allowReadingOnlyMatch?: boolean },
) => boolean;
export interface TokenizerServiceDeps {
getYomitanExt: () => Extension | null;
getYomitanSession?: () => Session | null;
@@ -42,7 +51,7 @@ export interface TokenizerServiceDeps {
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
getYomitanParserInitPromise: () => Promise<boolean> | null;
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
isKnownWord: (text: string, reading?: string) => boolean;
isKnownWord: KnownWordLookupFn;
getKnownWordMatchMode: () => NPlusOneMatchMode;
getKnownWordsEnabled?: () => boolean;
getJlptLevel: (text: string) => JlptLevel | null;
@@ -77,7 +86,7 @@ export interface TokenizerDepsRuntimeOptions {
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
getYomitanParserInitPromise: () => Promise<boolean> | null;
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
isKnownWord: (text: string, reading?: string) => boolean;
isKnownWord: KnownWordLookupFn;
getKnownWordMatchMode: () => NPlusOneMatchMode;
getKnownWordsEnabled?: () => boolean;
getJlptLevel: (text: string) => JlptLevel | null;
@@ -129,7 +138,7 @@ const INVISIBLE_SEPARATOR_PATTERN = /[\u200b\u2060\ufeff]/g;
function getKnownWordLookup(
deps: TokenizerServiceDeps,
options: TokenizerAnnotationOptions,
): (text: string, reading?: string) => boolean {
): KnownWordLookupFn {
if (!options.knownWordsEnabled && !options.nPlusOneEnabled) {
return () => false;
}