fix(tokenizer): exclude unparsed-run tokens from annotations and N+1 (#153)

2026-09-22 17:16:19 -07:00 · 2026-07-09 00:28:38 -07:00
parent ae40934d3a
commit 6c251502b3
13 changed files with 138 additions and 36 deletions
@@ -934,6 +934,7 @@ test('requestYomitanScanTokens keeps scanner metadata for matching spans when pa
      headword: 'っ ',
      startPos: 2,
      endPos: 4,
+      isUnparsedRun: true,
    },
    {
      surface: 'ミナト',