fix(tokenizer): exclude unparsed-run tokens from annotations and N+1 (#153)

This commit is contained in:
sudacode authored and GitHub committed 2026-07-09 00:28:38 -07:00
1 parent ae40934d3a
commit 6c251502b3
13 files changed
+138 -36

No files matched your search

@@ -934,6 +934,7 @@ test('requestYomitanScanTokens keeps scanner metadata for matching spans when pa
headword: 'っ ',
startPos: 2,
endPos: 4,
isUnparsedRun: true,
},
{
surface: 'ミナト',