fix(tokenizer): exclude unparsed-run tokens from annotations and N+1 (#153)

src/core/services/tokenizer.test.ts

@@ -28,6 +28,7 @@ interface YomitanTokenInput {
   frequencyRank?: number;
   isNameMatch?: boolean;
   wordClasses?: string[];
+  isUnparsedRun?: boolean;
 }
 
 function makeDepsFromYomitanTokens(
@@ -60,6 +61,7 @@ function makeDepsFromYomitanTokens(
                 isNameMatch: token.isNameMatch ?? false,
                 frequencyRank: token.frequencyRank,
                 wordClasses: token.wordClasses,
+                isUnparsedRun: token.isUnparsedRun,
               };
             });
           },
@@ -4223,6 +4225,38 @@ test('tokenizeSubtitle clears all annotations for explanatory pondering endings'
   );
 });
 
+test('tokenizeSubtitle ignores unparsed-run tokens for annotations and N+1', async () => {
+  // もう いるぅ～！: the ぅ～ elongation has no Yomitan dictionary entry; it must
+  // not become the sole N+1 candidate or receive frequency/JLPT annotations.
+  const result = await tokenizeSubtitle(
+    'もう いるぅ～！',
+    makeDepsFromYomitanTokens(
+      [
+        { surface: 'もう', reading: 'もう', headword: 'もう' },
+        { surface: 'いる', reading: 'いる', headword: 'いる' },
+        { surface: 'ぅ～', reading: '', headword: 'ぅ～', isUnparsedRun: true, frequencyRank: 999 },
+      ],
+      {
+        getFrequencyDictionaryEnabled: () => true,
+        getJlptLevel: (text) => (text === 'ぅ～' ? 'N5' : null),
+        isKnownWord: (text) => text === 'もう' || text === 'いる',
+        getMinSentenceWordsForNPlusOne: () => 2,
+        tokenizeWithMecab: async () => null,
+      },
+    ),
+  );
+
+  const filler = result.tokens?.find((token) => token.surface === 'ぅ～');
+  assert.ok(filler);
+  assert.equal(filler?.isNPlusOneTarget, false);
+  assert.equal(filler?.frequencyRank, undefined);
+  assert.equal(filler?.jlptLevel, undefined);
+  assert.equal(
+    result.tokens?.some((token) => token.isNPlusOneTarget),
+    false,
+  );
+});
+
 test('tokenizeSubtitle keeps frequency for content-led merged token with trailing colloquial suffixes', async () => {
   const result = await tokenizeSubtitle(
     '張り切ってんじゃ',