mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-31 19:21:33 -07:00
fix(tokenizer): block kanji reading collisions and name/generic length t
- Add allowReadingOnlyMatch:false to kanji token known-word lookups so 渓谷/けいこく no longer matches a mined 警告/けいこく card via reading-only index - Greedy name pre-pass yields when a strictly longer generic word starts at the same position (空 no longer splits 空気; ties still go to the name)
This commit is contained in:
@@ -33,6 +33,15 @@ type MecabTokenEnrichmentFn = (
|
||||
mecabTokens: MergedToken[] | null,
|
||||
) => Promise<MergedToken[]>;
|
||||
|
||||
// allowReadingOnlyMatch: false suppresses the cache's reading-only index for
|
||||
// lookups that pass a kanji token's reading as the text (see
|
||||
// computeTokenKnownStatus in annotation-stage).
|
||||
export type KnownWordLookupFn = (
|
||||
text: string,
|
||||
reading?: string,
|
||||
options?: { allowReadingOnlyMatch?: boolean },
|
||||
) => boolean;
|
||||
|
||||
export interface TokenizerServiceDeps {
|
||||
getYomitanExt: () => Extension | null;
|
||||
getYomitanSession?: () => Session | null;
|
||||
@@ -42,7 +51,7 @@ export interface TokenizerServiceDeps {
|
||||
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
|
||||
getYomitanParserInitPromise: () => Promise<boolean> | null;
|
||||
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
|
||||
isKnownWord: (text: string, reading?: string) => boolean;
|
||||
isKnownWord: KnownWordLookupFn;
|
||||
getKnownWordMatchMode: () => NPlusOneMatchMode;
|
||||
getKnownWordsEnabled?: () => boolean;
|
||||
getJlptLevel: (text: string) => JlptLevel | null;
|
||||
@@ -77,7 +86,7 @@ export interface TokenizerDepsRuntimeOptions {
|
||||
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
|
||||
getYomitanParserInitPromise: () => Promise<boolean> | null;
|
||||
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
|
||||
isKnownWord: (text: string, reading?: string) => boolean;
|
||||
isKnownWord: KnownWordLookupFn;
|
||||
getKnownWordMatchMode: () => NPlusOneMatchMode;
|
||||
getKnownWordsEnabled?: () => boolean;
|
||||
getJlptLevel: (text: string) => JlptLevel | null;
|
||||
@@ -129,7 +138,7 @@ const INVISIBLE_SEPARATOR_PATTERN = /[\u200b\u2060\ufeff]/g;
|
||||
function getKnownWordLookup(
|
||||
deps: TokenizerServiceDeps,
|
||||
options: TokenizerAnnotationOptions,
|
||||
): (text: string, reading?: string) => boolean {
|
||||
): KnownWordLookupFn {
|
||||
if (!options.knownWordsEnabled && !options.nPlusOneEnabled) {
|
||||
return () => false;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user