Restore multi-copy digit capture and add AniList selection (#56)

src/core/services/tokenizer/annotation-stage.test.ts

@@ -353,6 +353,19 @@ test('shouldExcludeTokenFromSubtitleAnnotations excludes kana-only demonstrative
   assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
 });
 
+test('shouldExcludeTokenFromSubtitleAnnotations excludes kana-only non-independent noun helper merges', () => {
+  const token = makeToken({
+    surface: 'ことに',
+    headword: '事',
+    reading: 'コトニ',
+    partOfSpeech: PartOfSpeech.noun,
+    pos1: '名詞|助詞',
+    pos2: '非自立|格助詞',
+  });
+
+  assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
+});
+
 test('stripSubtitleAnnotationMetadata keeps token hover data while clearing annotation fields', () => {
   const token = makeToken({
     surface: 'は',
@@ -812,3 +825,69 @@ test('annotateTokens applies one shared exclusion gate across known N+1 frequenc
   assert.equal(result[0]?.frequencyRank, undefined);
   assert.equal(result[0]?.jlptLevel, undefined);
 });
+
+test('annotateTokens clears all annotations for kana-only non-independent noun helper merges', () => {
+  const tokens = [
+    makeToken({
+      surface: 'ことに',
+      headword: '事',
+      reading: 'コトニ',
+      partOfSpeech: PartOfSpeech.noun,
+      pos1: '名詞|助詞',
+      pos2: '非自立|格助詞',
+      startPos: 0,
+      endPos: 3,
+      frequencyRank: 81,
+    }),
+  ];
+
+  const result = annotateTokens(
+    tokens,
+    makeDeps({
+      isKnownWord: (text) => text === '事',
+      getJlptLevel: (text) => (text === '事' ? 'N4' : null),
+    }),
+    { minSentenceWordsForNPlusOne: 1 },
+  );
+
+  assert.equal(result[0]?.isKnown, false);
+  assert.equal(result[0]?.isNPlusOneTarget, false);
+  assert.equal(result[0]?.frequencyRank, undefined);
+  assert.equal(result[0]?.jlptLevel, undefined);
+});
+
+test('annotateTokens clears all annotations from standalone あ interjections without POS tags', () => {
+  const tokens = [
+    makeToken({
+      surface: 'あ',
+      headword: 'あ',
+      reading: 'あ',
+      partOfSpeech: PartOfSpeech.other,
+      pos1: '',
+      pos2: '',
+      startPos: 0,
+      endPos: 1,
+      isKnown: true,
+      isNPlusOneTarget: true,
+      frequencyRank: 522,
+      jlptLevel: 'N5',
+    }),
+  ];
+
+  const result = annotateTokens(
+    tokens,
+    makeDeps({
+      isKnownWord: (text) => text === 'あ',
+      getJlptLevel: (text) => (text === 'あ' ? 'N5' : null),
+    }),
+    { minSentenceWordsForNPlusOne: 1 },
+  );
+
+  assert.equal(result[0]?.surface, 'あ');
+  assert.equal(result[0]?.headword, 'あ');
+  assert.equal(result[0]?.reading, 'あ');
+  assert.equal(result[0]?.isKnown, false);
+  assert.equal(result[0]?.isNPlusOneTarget, false);
+  assert.equal(result[0]?.frequencyRank, undefined);
+  assert.equal(result[0]?.jlptLevel, undefined);
+});

src/core/services/tokenizer/subtitle-annotation-filter.ts

@@ -14,6 +14,7 @@ const KATAKANA_CODEPOINT_START = 0x30a1;
 const KATAKANA_CODEPOINT_END = 0x30f6;
 
 const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
+  'あ',
   'ああ',
   'ええ',
   'うう',
@@ -70,6 +71,7 @@ const SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES = new Set([
   'ってば',
 ]);
 const AUXILIARY_STEM_GRAMMAR_TAIL_POS1 = new Set(['名詞', '助動詞', '助詞']);
+const NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1 = new Set(['助詞', '助動詞']);
 
 export interface SubtitleAnnotationFilterOptions {
   pos1Exclusions?: ReadonlySet<string>;
@@ -251,6 +253,31 @@ function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean {
   return pos3Parts.includes('助動詞語幹');
 }
 
+function isKanaOnlyNonIndependentNounHelperMerge(token: MergedToken): boolean {
+  const normalizedSurface = normalizeKana(token.surface);
+  const normalizedHeadword = normalizeKana(token.headword);
+  if (
+    !normalizedSurface ||
+    !normalizedHeadword ||
+    normalizedSurface === normalizedHeadword ||
+    ![...normalizedSurface].every(isKanaChar)
+  ) {
+    return false;
+  }
+
+  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  if (pos1Parts.length < 2 || pos1Parts[0] !== '名詞') {
+    return false;
+  }
+
+  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
+  if (pos2Parts[0] !== '非自立') {
+    return false;
+  }
+
+  return pos1Parts.slice(1).every((part) => NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1.has(part));
+}
+
 function isExcludedByTerm(token: MergedToken): boolean {
   const candidates = [token.surface, token.reading, token.headword].filter(
     (candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0,
@@ -334,6 +361,10 @@ export function shouldExcludeTokenFromSubtitleAnnotations(
     return true;
   }
 
+  if (isKanaOnlyNonIndependentNounHelperMerge(token)) {
+    return true;
+  }
+
   if (isExcludedTrailingParticleMergedToken(token)) {
     return true;
   }