fix(tokenizer): exclude unparsed-run tokens from annotations and N+1 (#153)

This commit is contained in:
2026-07-09 00:28:38 -07:00
committed by GitHub
parent ae40934d3a
commit 6c251502b3
13 changed files with 138 additions and 36 deletions
+6
View File
@@ -40,6 +40,12 @@ export interface MergedToken {
isMerged: boolean;
isKnown: boolean;
isNPlusOneTarget: boolean;
/**
* Text Yomitan had no dictionary entry for (e.g. ぅ~ elongation runs,
* truncated inflections). Kept as a token so it stays hoverable, but
* ignored by annotation, N+1, and vocabulary-stats logic.
*/
isUnparsedRun?: boolean;
isNameMatch?: boolean;
characterImage?: CharacterNameImage;
jlptLevel?: JlptLevel;