refactor(tokenizer): extract subtitle annotation filter into rule table (#162)

changes/annotation-noise-rule-table.md

@@ -0,0 +1,4 @@
+type: fixed
+area: overlay
+
+- Applied configured primary POS exclusions consistently to merged trailing quote-particle tokens, preserved annotations for supplementary-plane kanji, and stopped treating katakana punctuation as kana-only annotation noise.

changes/kana-pos-content-classification.md

@@ -0,0 +1,4 @@
+type: fixed
+area: overlay
+
+- Kept kanji vocabulary tagged `名詞/非自立` eligible for N+1 highlighting, consistent with frequency, JLPT, and vocabulary persistence.

docs-site/subtitle-annotations.md

@@ -10,6 +10,8 @@ SubMiner's primary tokenizer is Yomitan itself - subtitle text is tokenized base
 
 Before any of those layers render, SubMiner strips annotation metadata from tokens that are usually just subtitle glue or annotation noise. Standalone particles, auxiliaries, adnominals, common explanatory endings like `んです` / `のだ`, merged trailing quote-particle forms like `...って`, auxiliary-stem grammar tails like `そうだ` (MeCab POS3 `助動詞語幹`), repeated kana interjections, and similar non-lexical helper tokens remain hoverable in the subtitle text, but they render as plain tokens without known-word, N+1, frequency, JLPT, or name-match annotation styling.
 
+Kanji vocabulary that MeCab labels `名詞/非自立`, such as `日` or `以外`, remains content for every annotation layer. The `非自立` exclusion only suppresses kana grammar nouns such as `こと` and `もの`.
+
 ## N+1 Word Highlighting
 
 N+1 highlighting identifies sentences where you know every word except one, making them ideal mining targets. When enabled, SubMiner builds a local cache of your known vocabulary from Anki and highlights tokens accordingly.
@@ -24,16 +26,16 @@ N+1 highlighting identifies sentences where you know every word except one, maki
 
 **Key settings:**
 
-| Option                                    | Default      | Description                                                                                                                                                   |
-| ----------------------------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `ankiConnect.knownWords.highlightEnabled` | `false`      | Enable known-word cache lookups used by N+1 highlighting                                                                                                      |
-| `ankiConnect.knownWords.refreshMinutes`   | `1440`       | Minutes between Anki cache refreshes                                                                                                                          |
-| `ankiConnect.knownWords.decks`            | `{}`         | Deck→fields map for known-word cache queries                                                                                                                  |
-| `ankiConnect.knownWords.matchMode`        | `"headword"` | `"headword"` (dictionary form) or `"surface"` (raw text)                                                                                                      |
-| `ankiConnect.nPlusOne.enabled`            | `false`      | Enable N+1 target highlighting                                                                                                                               |
-| `ankiConnect.nPlusOne.minSentenceWords`   | `3`          | Minimum tokens in a sentence for N+1 to trigger                                                                                                               |
-| `subtitleStyle.nPlusOneColor`             | `#c6a0f6`    | Color for the single unknown target word                                                                                                                      |
-| `subtitleStyle.knownWordColor`            | `#a6da95`    | Color for already-known tokens                                                                                                                                |
+| Option                                    | Default      | Description                                              |
+| ----------------------------------------- | ------------ | -------------------------------------------------------- |
+| `ankiConnect.knownWords.highlightEnabled` | `false`      | Enable known-word cache lookups used by N+1 highlighting |
+| `ankiConnect.knownWords.refreshMinutes`   | `1440`       | Minutes between Anki cache refreshes                     |
+| `ankiConnect.knownWords.decks`            | `{}`         | Deck→fields map for known-word cache queries             |
+| `ankiConnect.knownWords.matchMode`        | `"headword"` | `"headword"` (dictionary form) or `"surface"` (raw text) |
+| `ankiConnect.nPlusOne.enabled`            | `false`      | Enable N+1 target highlighting                           |
+| `ankiConnect.nPlusOne.minSentenceWords`   | `3`          | Minimum tokens in a sentence for N+1 to trigger          |
+| `subtitleStyle.nPlusOneColor`             | `#c6a0f6`    | Color for the single unknown target word                 |
+| `subtitleStyle.knownWordColor`            | `#a6da95`    | Color for already-known tokens                           |
 
 Prefer expression/word fields for `ankiConnect.knownWords.decks`. Reading-only fields can mark unrelated homophones as known, so only include them when that tradeoff is intentional.
 

src/core/services/tokenizer.ts

@@ -25,6 +25,7 @@ import {
   requestYomitanTermFrequencies,
 } from './tokenizer/yomitan-parser-runtime';
 import type { YomitanTermFrequency } from './tokenizer/yomitan-parser-runtime';
+import { isKanaChar } from './tokenizer/token-classification';
 
 const logger = createLogger('main:tokenizer');
 
@@ -322,20 +323,6 @@ function normalizeFrequencyLookupText(rawText: string): string {
   return rawText.trim().toLowerCase();
 }
 
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
 function getTrailingKanaSuffix(surface: string): string {
   const chars = Array.from(surface);
   let splitIndex = chars.length;

src/core/services/tokenizer/annotation-stage.test.ts

@@ -1563,6 +1563,7 @@ test('annotateTokens keeps frequency for unknown non-independent kanji noun toke
   );
 
   assert.equal(result[0]?.isKnown, false);
+  assert.equal(result[0]?.isNPlusOneTarget, true);
   assert.equal(result[0]?.frequencyRank, 718);
   assert.equal(result[0]?.jlptLevel, 'N4');
 });

src/core/services/tokenizer/annotation-stage.ts

@@ -10,14 +10,19 @@ import {
 import { JlptLevel, MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types';
 import { shouldIgnoreJlptByTerm, shouldIgnoreJlptForMecabPos1 } from '../jlpt-token-filter';
 import {
-  isKanjiNonIndependentNounToken,
   shouldExcludeTokenFromSubtitleAnnotations as sharedShouldExcludeTokenFromSubtitleAnnotations,
   stripSubtitleAnnotationMetadata as sharedStripSubtitleAnnotationMetadata,
 } from './subtitle-annotation-filter';
+import {
+  isKanaChar,
+  isKanaOnlyText,
+  isPosTagExcluded,
+  isTokenPos2Excluded,
+  normalizeKana,
+  normalizePosTag,
+  splitPosTag,
+} from './token-classification';
 
-const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
-const KATAKANA_CODEPOINT_START = 0x30a1;
-const KATAKANA_CODEPOINT_END = 0x30f6;
 const JLPT_LEVEL_LOOKUP_CACHE_LIMIT = 2048;
 
 const jlptLevelLookupCaches = new WeakMap<
@@ -55,30 +60,6 @@ function resolveKnownWordText(
   return matchMode === 'surface' ? surface : headword;
 }
 
-function normalizePos1Tag(pos1: string | undefined): string {
-  return typeof pos1 === 'string' ? pos1.trim() : '';
-}
-
-function splitNormalizedTagParts(normalizedTag: string): string[] {
-  if (!normalizedTag) {
-    return [];
-  }
-
-  return normalizedTag
-    .split('|')
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
-}
-
-function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<string>): boolean {
-  const parts = splitNormalizedTagParts(normalizedTag);
-  if (parts.length === 0) {
-    return false;
-  }
-
-  return parts.every((part) => exclusions.has(part));
-}
-
 function resolvePos1Exclusions(options: AnnotationStageOptions): ReadonlySet<string> {
   if (options.pos1Exclusions) {
     return options.pos1Exclusions;
@@ -95,10 +76,6 @@ function resolvePos2Exclusions(options: AnnotationStageOptions): ReadonlySet<str
   return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
 }
 
-function normalizePos2Tag(pos2: string | undefined): string {
-  return typeof pos2 === 'string' ? pos2.trim() : '';
-}
-
 function isExcludedComponent(
   pos1: string | undefined,
   pos2: string | undefined,
@@ -117,12 +94,12 @@ function shouldAllowContentLedMergedTokenFrequency(
   pos1Exclusions: ReadonlySet<string>,
   pos2Exclusions: ReadonlySet<string>,
 ): boolean {
-  const pos1Parts = splitNormalizedTagParts(normalizedPos1);
+  const pos1Parts = splitPosTag(normalizedPos1);
   if (pos1Parts.length < 2) {
     return false;
   }
 
-  const pos2Parts = splitNormalizedTagParts(normalizedPos2);
+  const pos2Parts = splitPosTag(normalizedPos2);
   if (isExcludedComponent(pos1Parts[0], pos2Parts[0], pos1Exclusions, pos2Exclusions)) {
     return false;
   }
@@ -144,8 +121,8 @@ function shouldAllowOrdinalPrefixNounFrequency(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
-  const pos2Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos2));
+  const pos1Parts = splitPosTag(token.pos1);
+  const pos2Parts = splitPosTag(token.pos2);
   return (
     pos1Parts.length >= 2 &&
     pos1Parts[0] === '接頭詞' &&
@@ -166,8 +143,8 @@ function shouldAllowHonorificPrefixNounFrequency(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
-  const pos2Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos2));
+  const pos1Parts = splitPosTag(token.pos1);
+  const pos2Parts = splitPosTag(token.pos2);
   return (
     pos1Parts.length >= 2 &&
     pos1Parts[0] === '接頭詞' &&
@@ -182,12 +159,12 @@ function shouldAllowDeterminerLedNounFrequency(
   pos1Exclusions: ReadonlySet<string>,
   pos2Exclusions: ReadonlySet<string>,
 ): boolean {
-  const pos1Parts = splitNormalizedTagParts(normalizedPos1);
+  const pos1Parts = splitPosTag(normalizedPos1);
   if (pos1Parts.length < 2 || pos1Parts[0] !== '連体詞') {
     return false;
   }
 
-  const pos2Parts = splitNormalizedTagParts(normalizedPos2);
+  const pos2Parts = splitPosTag(normalizedPos2);
   if (!isExcludedComponent(pos1Parts[0], pos2Parts[0], pos1Exclusions, pos2Exclusions)) {
     return false;
   }
@@ -218,9 +195,9 @@ function isFrequencyExcludedByPos(
     return true;
   }
 
-  const normalizedPos1 = normalizePos1Tag(token.pos1);
+  const normalizedPos1 = normalizePosTag(token.pos1);
   const hasPos1 = normalizedPos1.length > 0;
-  const normalizedPos2 = normalizePos2Tag(token.pos2);
+  const normalizedPos2 = normalizePosTag(token.pos2);
   const hasPos2 = normalizedPos2.length > 0;
   const allowContentLedMergedToken = shouldAllowContentLedMergedTokenFrequency(
     normalizedPos1,
@@ -238,7 +215,7 @@ function isFrequencyExcludedByPos(
   const allowHonorificPrefixNounToken = shouldAllowHonorificPrefixNounFrequency(token);
 
   if (
-    isExcludedByTagSet(normalizedPos1, pos1Exclusions) &&
+    isPosTagExcluded(normalizedPos1, pos1Exclusions) &&
     !allowContentLedMergedToken &&
     !allowDeterminerLedNounToken &&
     !allowOrdinalPrefixNounToken &&
@@ -248,7 +225,7 @@ function isFrequencyExcludedByPos(
   }
 
   if (
-    isExcludedByTagSet(normalizedPos2, pos2Exclusions) &&
+    isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions) &&
     !allowContentLedMergedToken &&
     !allowDeterminerLedNounToken &&
     !allowOrdinalPrefixNounToken &&
@@ -280,8 +257,7 @@ export function shouldExcludeTokenFromVocabularyPersistence(
 
   return (
     sharedShouldExcludeTokenFromSubtitleAnnotations(token, { pos1Exclusions, pos2Exclusions }) ||
-    (isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) &&
-      !isKanjiNonIndependentNounToken(token, pos1Exclusions))
+    isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions)
   );
 }
 
@@ -332,45 +308,6 @@ function resolveJlptLookupText(token: MergedToken): string {
   return token.surface;
 }
 
-function normalizeJlptTextForExclusion(text: string): string {
-  const raw = text.trim();
-  if (!raw) {
-    return '';
-  }
-
-  let normalized = '';
-  for (const char of raw) {
-    const code = char.codePointAt(0);
-    if (code === undefined) {
-      continue;
-    }
-
-    if (code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END) {
-      normalized += String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET);
-      continue;
-    }
-
-    normalized += char;
-  }
-
-  return normalized;
-}
-
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
 function isRepeatedKanaSfx(text: string): boolean {
   const normalized = text.trim();
   if (!normalized) {
@@ -406,7 +343,7 @@ function isRepeatedKanaSfx(text: string): boolean {
 }
 
 function isTrailingSmallTsuKanaSfx(text: string): boolean {
-  const normalized = normalizeJlptTextForExclusion(text);
+  const normalized = normalizeKana(text);
   if (!normalized) {
     return false;
   }
@@ -424,7 +361,7 @@ function isTrailingSmallTsuKanaSfx(text: string): boolean {
 }
 
 function isReduplicatedKanaSfx(text: string): boolean {
-  const normalized = normalizeJlptTextForExclusion(text);
+  const normalized = normalizeKana(text);
   if (!normalized) {
     return false;
   }
@@ -443,7 +380,7 @@ function isReduplicatedKanaSfx(text: string): boolean {
 }
 
 function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
-  const normalized = normalizeJlptTextForExclusion(text);
+  const normalized = normalizeKana(text);
   if (!normalized) {
     return false;
   }
@@ -460,7 +397,7 @@ function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
 }
 
 function hasAdjacentKanaRepeat(text: string): boolean {
-  const normalized = normalizeJlptTextForExclusion(text);
+  const normalized = normalizeKana(text);
   if (!normalized) {
     return false;
   }
@@ -490,7 +427,7 @@ function isLikelyFrequencyNoiseToken(token: MergedToken): boolean {
       continue;
     }
 
-    const normalizedCandidate = normalizeJlptTextForExclusion(trimmedCandidate);
+    const normalizedCandidate = normalizeKana(trimmedCandidate);
     if (!normalizedCandidate) {
       continue;
     }
@@ -528,19 +465,6 @@ function isSingleKanaFrequencyNoiseToken(text: string | undefined): boolean {
   return chars.length === 1 && isKanaChar(chars[0]!);
 }
 
-function isKanaOnlyText(text: string | undefined): boolean {
-  if (typeof text !== 'string') {
-    return false;
-  }
-
-  const normalized = text.trim();
-  if (!normalized) {
-    return false;
-  }
-
-  return [...normalized].every(isKanaChar);
-}
-
 function isKanaOnlyMixedFunctionContentToken(
   token: MergedToken,
   pos1Exclusions: ReadonlySet<string>,
@@ -549,7 +473,7 @@ function isKanaOnlyMixedFunctionContentToken(
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   const hasMixedFunctionContentParts =
     pos1Parts.length >= 2 &&
     pos1Parts.some((part) => pos1Exclusions.has(part)) &&
@@ -558,8 +482,8 @@ function isKanaOnlyMixedFunctionContentToken(
     return false;
   }
 
-  const normalizedReading = normalizeJlptTextForExclusion(token.reading);
-  const normalizedHeadwordReading = normalizeJlptTextForExclusion(token.headwordReading ?? '');
+  const normalizedReading = normalizeKana(token.reading);
+  const normalizedHeadwordReading = normalizeKana(token.headwordReading ?? '');
   return (
     !normalizedReading ||
     !normalizedHeadwordReading ||
@@ -577,7 +501,7 @@ function isJlptEligibleToken(token: MergedToken): boolean {
   );
 
   for (const candidate of candidates) {
-    const normalizedCandidate = normalizeJlptTextForExclusion(candidate);
+    const normalizedCandidate = normalizeKana(candidate);
     if (!normalizedCandidate) {
       continue;
     }
@@ -612,8 +536,8 @@ export function stripSubtitleAnnotationMetadata(
 // at least as many characters as the surface, with the surface's kana appearing
 // in order within the reading.
 function isCompleteReadingForSurface(surface: string, reading: string): boolean {
-  const surfaceChars = [...normalizeJlptTextForExclusion(surface)];
-  const readingChars = [...normalizeJlptTextForExclusion(reading)];
+  const surfaceChars = [...normalizeKana(surface)];
+  const readingChars = [...normalizeKana(reading)];
   if (readingChars.length < surfaceChars.length) {
     return false;
   }
@@ -697,10 +621,7 @@ function filterTokenFrequencyRank(
   pos1Exclusions: ReadonlySet<string>,
   pos2Exclusions: ReadonlySet<string>,
 ): number | undefined {
-  if (
-    isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) &&
-    !isKanjiNonIndependentNounToken(token, pos1Exclusions)
-  ) {
+  if (isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions)) {
     return undefined;
   }
 

src/core/services/tokenizer/parser-selection-stage.ts

@@ -1,5 +1,6 @@
 import { MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types';
 import { isStandaloneGrammarEndingText } from './grammar-ending';
+import { isKanaChar, isKanaOnlyText } from './token-classification';
 
 interface YomitanParseHeadword {
   term?: unknown;
@@ -41,21 +42,6 @@ function resolveKnownWordText(
   return matchMode === 'surface' ? surface : headword;
 }
 
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
 function isYomitanParseLine(value: unknown): value is YomitanParseLine {
   if (!Array.isArray(value)) {
     return false;
@@ -138,10 +124,6 @@ function selectMergedHeadword(
   return firstHeadword;
 }
 
-function isKanaOnlyText(text: string): boolean {
-  return text.length > 0 && Array.from(text).every((char) => isKanaChar(char));
-}
-
 function isStandaloneGrammarEndingSegment(segment: YomitanParseSegment): boolean {
   const surface = segment.text?.trim() ?? '';
   const headword = extractYomitanHeadword(segment).trim();

src/core/services/tokenizer/part-of-speech.ts

@@ -1,8 +1,5 @@
 import { PartOfSpeech } from '../../../types';
-
-function normalizePosTag(value: string | null | undefined): string {
-  return typeof value === 'string' ? value.trim() : '';
-}
+import { normalizePosTag, splitPosTag } from './token-classification';
 
 export function isPartOfSpeechValue(value: unknown): value is PartOfSpeech {
   return typeof value === 'string' && Object.values(PartOfSpeech).includes(value as PartOfSpeech);
@@ -35,10 +32,7 @@ export function deriveStoredPartOfSpeech(input: {
   partOfSpeech?: string | null;
   pos1?: string | null;
 }): PartOfSpeech {
-  const pos1Parts = normalizePosTag(input.pos1)
-    .split('|')
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
+  const pos1Parts = splitPosTag(input.pos1);
 
   if (pos1Parts.length > 0) {
     const derivedParts = [...new Set(pos1Parts.map((part) => mapMecabPos1ToPartOfSpeech(part)))];

src/core/services/tokenizer/subtitle-annotation-filter-rules.ts

@@ -0,0 +1,342 @@
+import {
+  DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG,
+  resolveAnnotationPos1ExclusionSet,
+} from '../../../token-pos1-exclusions';
+import {
+  DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG,
+  resolveAnnotationPos2ExclusionSet,
+} from '../../../token-pos2-exclusions';
+import { MergedToken } from '../../../types';
+import { shouldIgnoreJlptByTerm } from '../jlpt-token-filter';
+import { isSubtitleGrammarEndingText } from './grammar-ending';
+import {
+  isAuxiliaryOnlyHelperSpan,
+  isAuxiliaryStemGrammarTailToken,
+  isExcludedTrailingParticleMergedToken,
+  isKanaOnlyNonIndependentNounHelperMerge,
+  isReduplicatedKanaSfxWithOptionalTrailingTo,
+  isSingleKanaSurfaceFragment,
+  isStandaloneAuxiliaryInflectionFragment,
+  isStandaloneGrammarParticle,
+  isStandaloneSuruTeGrammarHelper,
+  isTrailingSmallTsuKanaSfx,
+} from './subtitle-annotation-filter-support';
+import {
+  isContentTokenByPos,
+  isPosTagExcluded,
+  isTokenPos2Excluded,
+  normalizeKana,
+  normalizePosTag,
+  splitPosTag,
+} from './token-classification';
+
+export interface SubtitleAnnotationFilterOptions {
+  pos1Exclusions?: ReadonlySet<string>;
+  pos2Exclusions?: ReadonlySet<string>;
+}
+
+export type SubtitleAnnotationRuleDecision = 'exclude' | 'keep' | 'pass';
+
+export interface SubtitleAnnotationRuleContext {
+  token: MergedToken;
+  pos1Exclusions: ReadonlySet<string>;
+  pos2Exclusions: ReadonlySet<string>;
+  normalizedPos1: string;
+  normalizedPos2: string;
+  hasPos1: boolean;
+  hasPos2: boolean;
+}
+
+type SubtitleAnnotationRuleData = Readonly<Record<string, readonly string[]>>;
+
+export interface SubtitleAnnotationRule {
+  readonly id: string;
+  readonly description: string;
+  readonly issueRef: string;
+  readonly data: SubtitleAnnotationRuleData;
+  test(context: SubtitleAnnotationRuleContext): SubtitleAnnotationRuleDecision;
+}
+
+function defineRule<TData extends SubtitleAnnotationRuleData>(definition: {
+  id: string;
+  description: string;
+  issueRef: string;
+  data: TData;
+  test: (context: SubtitleAnnotationRuleContext, data: TData) => SubtitleAnnotationRuleDecision;
+}): SubtitleAnnotationRule {
+  const data = Object.freeze(
+    Object.fromEntries(
+      Object.entries(definition.data).map(([key, values]) => [key, Object.freeze([...values])]),
+    ),
+  ) as TData;
+  return Object.freeze({
+    id: definition.id,
+    description: definition.description,
+    issueRef: definition.issueRef,
+    data,
+    test: (context: SubtitleAnnotationRuleContext) => definition.test(context, data),
+  });
+}
+
+export function createSubtitleAnnotationRuleContext(
+  token: MergedToken,
+  options: SubtitleAnnotationFilterOptions = {},
+): SubtitleAnnotationRuleContext {
+  const pos1Exclusions =
+    options.pos1Exclusions ??
+    resolveAnnotationPos1ExclusionSet(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG);
+  const pos2Exclusions =
+    options.pos2Exclusions ??
+    resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
+  const normalizedPos1 = normalizePosTag(token.pos1);
+  const normalizedPos2 = normalizePosTag(token.pos2);
+  return {
+    token,
+    pos1Exclusions,
+    pos2Exclusions,
+    normalizedPos1,
+    normalizedPos2,
+    hasPos1: normalizedPos1.length > 0,
+    hasPos2: normalizedPos2.length > 0,
+  };
+}
+
+function matchesExcludedTermOrPattern(token: MergedToken, terms: ReadonlySet<string>): boolean {
+  const candidates = [token.surface, token.reading, token.headword].filter(
+    (candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0,
+  );
+
+  for (const candidate of candidates) {
+    const trimmed = candidate.trim();
+    if (!trimmed) {
+      continue;
+    }
+    const normalized = normalizeKana(trimmed);
+    if (!normalized) {
+      continue;
+    }
+    if (
+      terms.has(trimmed) ||
+      terms.has(normalized) ||
+      isSubtitleGrammarEndingText(trimmed) ||
+      isSubtitleGrammarEndingText(normalized) ||
+      shouldIgnoreJlptByTerm(trimmed) ||
+      shouldIgnoreJlptByTerm(normalized) ||
+      isTrailingSmallTsuKanaSfx(trimmed) ||
+      isTrailingSmallTsuKanaSfx(normalized) ||
+      isReduplicatedKanaSfxWithOptionalTrailingTo(trimmed) ||
+      isReduplicatedKanaSfxWithOptionalTrailingTo(normalized)
+    ) {
+      return true;
+    }
+  }
+  return false;
+}
+
+export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
+  'あ',
+  'ああ',
+  'ある',
+  'あなた',
+  'あんた',
+  'ええ',
+  'うう',
+  'おお',
+  'おい',
+  'お前',
+  'こいつ',
+  'こっち',
+  'くれ',
+  'じゃない',
+  'そうだ',
+  'たち',
+  'である',
+  'どこか',
+  'なんか',
+  'べき',
+  'って',
+  'はあ',
+  'はぁ',
+  'はは',
+  'へえ',
+  'ふう',
+  'ほう',
+  '何か',
+  '何だ',
+  '何も',
+  '如何した',
+  '有る',
+  '在る',
+  '様',
+  '誰も',
+  '貴方',
+  'もんか',
+  'ものか',
+]);
+
+const excludedTermRule = defineRule({
+  id: 'excluded-term-or-pattern',
+  description:
+    'Exclude legacy subtitle stop terms, grammar endings, JLPT stop terms, and kana sound effects.',
+  issueRef: '#19, #33, #57',
+  data: {},
+  test: ({ token }) =>
+    matchesExcludedTermOrPattern(token, SUBTITLE_ANNOTATION_EXCLUDED_TERMS) ? 'exclude' : 'pass',
+});
+
+// Ordered, first-match-wins. issueRef records the introducing/fixing PR; early
+// rules are legacy behavior inherited from the original #19 filter.
+export const SUBTITLE_ANNOTATION_RULES: readonly SubtitleAnnotationRule[] = Object.freeze([
+  defineRule({
+    id: 'unparsed-run',
+    description: 'Exclude hoverable parser gaps that have no Yomitan dictionary entry.',
+    issueRef: '#153',
+    data: {},
+    test: ({ token }) => (token.isUnparsedRun === true ? 'exclude' : 'pass'),
+  }),
+  defineRule({
+    id: 'configured-pos1-exclusion',
+    description: 'Apply the configured primary part-of-speech exclusions.',
+    issueRef: '#19',
+    data: {},
+    test: ({ token, pos1Exclusions }) =>
+      isPosTagExcluded(token.pos1, pos1Exclusions) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'configured-pos2-exclusion',
+    description: 'Apply configured secondary POS exclusions, preserving #150 kanji nouns.',
+    issueRef: '#150',
+    data: {},
+    test: ({ token, pos1Exclusions, pos2Exclusions }) =>
+      isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'coarse-grammar-pos-fallback',
+    description: 'Exclude coarse grammar POS when detailed MeCab tags are unavailable.',
+    issueRef: '#19',
+    data: {},
+    test: ({ token, hasPos1, hasPos2, pos1Exclusions, pos2Exclusions }) =>
+      !hasPos1 && !hasPos2 && !isContentTokenByPos(token, pos1Exclusions, pos2Exclusions)
+        ? 'exclude'
+        : 'pass',
+  }),
+  defineRule({
+    id: 'auxiliary-stem-grammar-tail',
+    description: 'Exclude merged grammar tails containing an auxiliary stem.',
+    issueRef: '#19',
+    data: { allowedPos1: ['名詞', '助動詞', '助詞'] },
+    test: ({ token }, { allowedPos1 }) =>
+      isAuxiliaryStemGrammarTailToken(token, allowedPos1) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'kana-non-independent-noun-helper',
+    description: 'Exclude kana non-independent nouns merged with grammar helpers.',
+    issueRef: '#56',
+    data: { tailPos1: ['助詞', '助動詞'] },
+    test: ({ token }, { tailPos1 }) =>
+      isKanaOnlyNonIndependentNounHelperMerge(token, tailPos1) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'standalone-auxiliary-inflection',
+    description: 'Exclude standalone kana auxiliary inflection fragments.',
+    issueRef: '#57',
+    data: { trailingPos1: ['助動詞'] },
+    test: ({ token }, { trailingPos1 }) =>
+      isStandaloneAuxiliaryInflectionFragment(token, trailingPos1) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'auxiliary-only-helper-span',
+    description: 'Exclude kana helper spans without an independent lexical verb.',
+    issueRef: '#57',
+    data: { allowedPos1: ['助詞', '助動詞', '動詞'], lexicalVerbPos2: ['自立'] },
+    test: ({ token }, { allowedPos1, lexicalVerbPos2 }) =>
+      isAuxiliaryOnlyHelperSpan(token, allowedPos1, lexicalVerbPos2) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'standalone-suru-te-helper',
+    description: 'Exclude standalone して grammar-helper fragments.',
+    issueRef: '#57',
+    data: {},
+    test: ({ token }) => (isStandaloneSuruTeGrammarHelper(token) ? 'exclude' : 'pass'),
+  }),
+  defineRule({
+    id: 'standalone-grammar-particle',
+    description: 'Exclude standalone particle surfaces and connective particle phrases.',
+    issueRef: '#57',
+    data: {
+      surfaces: [
+        'か',
+        'が',
+        'さ',
+        'し',
+        'ぞ',
+        'ぜ',
+        'と',
+        'な',
+        'に',
+        'ね',
+        'の',
+        'は',
+        'へ',
+        'も',
+        'や',
+        'よ',
+        'を',
+      ],
+      phrases: ['たって', 'だって'],
+    },
+    test: ({ token }, { surfaces, phrases }) =>
+      isStandaloneGrammarParticle(token, surfaces, phrases) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'single-kana-fragment',
+    description: 'Exclude isolated one-kana parser fragments.',
+    issueRef: '#57',
+    data: {},
+    test: ({ token }) => (isSingleKanaSurfaceFragment(token) ? 'exclude' : 'pass'),
+  }),
+  defineRule({
+    id: 'merged-trailing-quote-particle',
+    description: 'Exclude lexical tokens merged only with a trailing quote-particle suffix.',
+    issueRef: '#19',
+    data: { suffixes: ['って', 'ってよ', 'ってね', 'ってな', 'ってさ', 'ってか', 'ってば'] },
+    test: ({ token, pos1Exclusions }, { suffixes }) =>
+      isExcludedTrailingParticleMergedToken(token, suffixes, pos1Exclusions) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'lexical-kureru-keep',
+    description: 'Keep lexical くれる before the legacy bare-くれ stop-term rule.',
+    issueRef: '#57',
+    data: {
+      surfaces: ['くれ'],
+      headwords: ['くれる'],
+      pos1: ['動詞'],
+      pos2: ['自立'],
+    },
+    test: ({ token }, data) => {
+      const pos1 = splitPosTag(token.pos1);
+      const pos2 = splitPosTag(token.pos2);
+      return data.surfaces.includes(normalizeKana(token.surface)) &&
+        data.headwords.includes(normalizeKana(token.headword)) &&
+        pos1.length === 1 &&
+        data.pos1.includes(pos1[0] ?? '') &&
+        pos2.length === 1 &&
+        data.pos2.includes(pos2[0] ?? '')
+        ? 'keep'
+        : 'pass';
+    },
+  }),
+  excludedTermRule,
+]);
+
+export function evaluateSubtitleAnnotationRules(
+  context: SubtitleAnnotationRuleContext,
+): SubtitleAnnotationRuleDecision {
+  for (const rule of SUBTITLE_ANNOTATION_RULES) {
+    const decision = rule.test(context);
+    if (decision !== 'pass') {
+      return decision;
+    }
+  }
+  return 'pass';
+}

src/core/services/tokenizer/subtitle-annotation-filter-support.ts

@@ -0,0 +1,153 @@
+import { MergedToken } from '../../../types';
+import { isKanaChar, isKanaOnlyText, normalizeKana, splitPosTag } from './token-classification';
+
+export function isTrailingSmallTsuKanaSfx(text: string): boolean {
+  const chars = [...normalizeKana(text)];
+  return (
+    chars.length >= 2 &&
+    chars.length <= 4 &&
+    chars.every(isKanaChar) &&
+    chars[chars.length - 1] === 'っ'
+  );
+}
+
+function isReduplicatedKanaSfx(text: string): boolean {
+  const chars = [...normalizeKana(text)];
+  if (chars.length < 4 || chars.length % 2 !== 0 || !chars.every(isKanaChar)) {
+    return false;
+  }
+  const half = chars.length / 2;
+  return chars.slice(0, half).join('') === chars.slice(half).join('');
+}
+
+export function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
+  const normalized = normalizeKana(text);
+  if (!normalized) {
+    return false;
+  }
+  if (isReduplicatedKanaSfx(normalized)) {
+    return true;
+  }
+  return normalized.length > 1 && normalized.endsWith('と')
+    ? isReduplicatedKanaSfx(normalized.slice(0, -1))
+    : false;
+}
+
+export function isExcludedTrailingParticleMergedToken(
+  token: MergedToken,
+  suffixes: readonly string[],
+  leadingPos1Exclusions: ReadonlySet<string>,
+): boolean {
+  const surface = normalizeKana(token.surface);
+  const headword = normalizeKana(token.headword);
+  if (!surface || !headword || !surface.startsWith(headword)) {
+    return false;
+  }
+  if (!suffixes.includes(surface.slice(headword.length))) {
+    return false;
+  }
+
+  const [leadingPos1, ...trailingPos1] = splitPosTag(token.pos1);
+  if (!leadingPos1 || leadingPos1Exclusions.has(leadingPos1)) {
+    return false;
+  }
+  return trailingPos1.length > 0 && trailingPos1.every((part) => part === '助詞');
+}
+
+export function isAuxiliaryStemGrammarTailToken(
+  token: MergedToken,
+  allowedPos1: readonly string[],
+): boolean {
+  const pos1Parts = splitPosTag(token.pos1);
+  if (pos1Parts.length === 0 || !pos1Parts.every((part) => allowedPos1.includes(part))) {
+    return false;
+  }
+  return splitPosTag(token.pos3).includes('助動詞語幹');
+}
+
+export function isKanaOnlyNonIndependentNounHelperMerge(
+  token: MergedToken,
+  tailPos1: readonly string[],
+): boolean {
+  const surface = normalizeKana(token.surface);
+  const headword = normalizeKana(token.headword);
+  if (!surface || !headword || surface === headword || ![...surface].every(isKanaChar)) {
+    return false;
+  }
+
+  const pos1Parts = splitPosTag(token.pos1);
+  if (pos1Parts.length < 2 || pos1Parts[0] !== '名詞') {
+    return false;
+  }
+  const pos2Parts = splitPosTag(token.pos2);
+  return pos2Parts[0] === '非自立' && pos1Parts.slice(1).every((part) => tailPos1.includes(part));
+}
+
+export function isStandaloneAuxiliaryInflectionFragment(
+  token: MergedToken,
+  trailingPos1: readonly string[],
+): boolean {
+  if (!isKanaOnlyText(token.surface)) {
+    return false;
+  }
+  const pos1Parts = splitPosTag(token.pos1);
+  if (pos1Parts.length === 0) {
+    return false;
+  }
+  if (pos1Parts.every((part) => part === '助動詞')) {
+    return true;
+  }
+  const pos2Parts = splitPosTag(token.pos2);
+  return (
+    pos1Parts[0] === '動詞' &&
+    pos2Parts[0] === '接尾' &&
+    pos1Parts.slice(1).every((part) => trailingPos1.includes(part))
+  );
+}
+
+export function isAuxiliaryOnlyHelperSpan(
+  token: MergedToken,
+  allowedPos1: readonly string[],
+  lexicalVerbPos2: readonly string[],
+): boolean {
+  if (!isKanaOnlyText(token.surface) || !isKanaOnlyText(token.headword)) {
+    return false;
+  }
+  const pos1Parts = splitPosTag(token.pos1);
+  if (
+    pos1Parts.length === 0 ||
+    !pos1Parts.every((part) => allowedPos1.includes(part)) ||
+    !pos1Parts.includes('助詞') ||
+    !pos1Parts.includes('動詞')
+  ) {
+    return false;
+  }
+  return !splitPosTag(token.pos2).some((part) => lexicalVerbPos2.includes(part));
+}
+
+export function isStandaloneSuruTeGrammarHelper(token: MergedToken): boolean {
+  const surface = normalizeKana(token.surface);
+  const headword = normalizeKana(token.headword);
+  if (!surface.startsWith('して') || headword !== 'する') {
+    return false;
+  }
+  const pos1Parts = splitPosTag(token.pos1);
+  return isKanaOnlyText(surface) && (pos1Parts.length === 0 || pos1Parts.includes('動詞'));
+}
+
+export function isStandaloneGrammarParticle(
+  token: MergedToken,
+  surfaces: readonly string[],
+  phrases: readonly string[],
+): boolean {
+  const surface = normalizeKana(token.surface);
+  return (
+    surface === normalizeKana(token.headword) &&
+    (surfaces.includes(surface) || phrases.includes(surface))
+  );
+}
+
+export function isSingleKanaSurfaceFragment(token: MergedToken): boolean {
+  const chars = [...normalizeKana(token.surface)];
+  return chars.length === 1 && chars.every(isKanaChar);
+}

src/core/services/tokenizer/subtitle-annotation-filter.test.ts

@@ -0,0 +1,183 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { MergedToken, PartOfSpeech } from '../../../types';
+import {
+  createSubtitleAnnotationRuleContext,
+  shouldExcludeTokenFromSubtitleAnnotations,
+  SUBTITLE_ANNOTATION_EXCLUDED_TERMS,
+  SUBTITLE_ANNOTATION_RULES,
+} from './subtitle-annotation-filter';
+import { isKanaChar } from './token-classification';
+
+function makeToken(overrides: Partial<MergedToken> = {}): MergedToken {
+  return {
+    surface: '猫',
+    reading: 'ネコ',
+    headword: '猫',
+    startPos: 0,
+    endPos: 1,
+    partOfSpeech: PartOfSpeech.noun,
+    isMerged: false,
+    isKnown: false,
+    isNPlusOneTarget: false,
+    ...overrides,
+  };
+}
+
+test('subtitle annotation rules expose stable ordered provenance', () => {
+  assert.deepEqual(
+    SUBTITLE_ANNOTATION_RULES.map(({ id, issueRef }) => ({ id, issueRef })),
+    [
+      { id: 'unparsed-run', issueRef: '#153' },
+      { id: 'configured-pos1-exclusion', issueRef: '#19' },
+      { id: 'configured-pos2-exclusion', issueRef: '#150' },
+      { id: 'coarse-grammar-pos-fallback', issueRef: '#19' },
+      { id: 'auxiliary-stem-grammar-tail', issueRef: '#19' },
+      { id: 'kana-non-independent-noun-helper', issueRef: '#56' },
+      { id: 'standalone-auxiliary-inflection', issueRef: '#57' },
+      { id: 'auxiliary-only-helper-span', issueRef: '#57' },
+      { id: 'standalone-suru-te-helper', issueRef: '#57' },
+      { id: 'standalone-grammar-particle', issueRef: '#57' },
+      { id: 'single-kana-fragment', issueRef: '#57' },
+      { id: 'merged-trailing-quote-particle', issueRef: '#19' },
+      { id: 'lexical-kureru-keep', issueRef: '#57' },
+      { id: 'excluded-term-or-pattern', issueRef: '#19, #33, #57' },
+    ],
+  );
+  assert.equal(new Set(SUBTITLE_ANNOTATION_RULES.map(({ id }) => id)).size, 14);
+  assert.ok(SUBTITLE_ANNOTATION_RULES.every(({ description }) => description.length > 0));
+  assert.ok(
+    SUBTITLE_ANNOTATION_RULES.every(
+      ({ data }) => Object.isFrozen(data) && Object.values(data).every(Object.isFrozen),
+    ),
+  );
+});
+
+test('lexical kureru keep rule precedes and overrides the excluded-term rule', () => {
+  const token = makeToken({
+    surface: 'くれ',
+    headword: 'くれる',
+    reading: 'クレ',
+    partOfSpeech: PartOfSpeech.verb,
+    pos1: '動詞',
+    pos2: '自立',
+  });
+  const context = createSubtitleAnnotationRuleContext(token);
+  const keepIndex = SUBTITLE_ANNOTATION_RULES.findIndex(({ id }) => id === 'lexical-kureru-keep');
+  const termIndex = SUBTITLE_ANNOTATION_RULES.findIndex(
+    ({ id }) => id === 'excluded-term-or-pattern',
+  );
+
+  assert.ok(keepIndex >= 0 && keepIndex < termIndex);
+  assert.ok(SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has('くれ'));
+  assert.equal(SUBTITLE_ANNOTATION_RULES[keepIndex]?.test(context), 'keep');
+  assert.equal(SUBTITLE_ANNOTATION_RULES[termIndex]?.test(context), 'exclude');
+  assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
+});
+
+test('adding an exported excluded term changes annotation matching', () => {
+  const term = '超語彙';
+  const token = makeToken({
+    surface: term,
+    headword: term,
+    reading: term,
+    pos1: '名詞',
+    pos2: '一般',
+  });
+
+  SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term);
+  try {
+    assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
+    SUBTITLE_ANNOTATION_EXCLUDED_TERMS.add(term);
+    assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
+  } finally {
+    SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term);
+  }
+});
+
+test('deleting an exported excluded term changes annotation matching', () => {
+  const term = 'あなた';
+  const token = makeToken({
+    surface: term,
+    headword: term,
+    reading: term,
+    pos1: '名詞',
+    pos2: '一般',
+  });
+
+  assert.ok(SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(term));
+  try {
+    SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term);
+    assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
+  } finally {
+    SUBTITLE_ANNOTATION_EXCLUDED_TERMS.add(term);
+  }
+});
+
+test('configured POS rules consume the context exclusion sets in table order', () => {
+  const token = makeToken({ pos1: '名詞', pos2: '一般' });
+  const context = createSubtitleAnnotationRuleContext(token, {
+    pos1Exclusions: new Set(['名詞']),
+    pos2Exclusions: new Set(['一般']),
+  });
+  const pos1Rule = SUBTITLE_ANNOTATION_RULES.find(({ id }) => id === 'configured-pos1-exclusion');
+  const pos2Rule = SUBTITLE_ANNOTATION_RULES.find(({ id }) => id === 'configured-pos2-exclusion');
+
+  assert.equal(pos1Rule?.test(context), 'exclude');
+  assert.equal(pos2Rule?.test(context), 'exclude');
+  assert.equal(
+    shouldExcludeTokenFromSubtitleAnnotations(token, {
+      pos1Exclusions: context.pos1Exclusions,
+      pos2Exclusions: context.pos2Exclusions,
+    }),
+    true,
+  );
+});
+
+test('configured POS2 rule preserves the kanji non-independent noun exception', () => {
+  const token = makeToken({ surface: '以外', headword: '以外', pos1: '名詞', pos2: '非自立' });
+  const context = createSubtitleAnnotationRuleContext(token, {
+    pos1Exclusions: new Set(),
+    pos2Exclusions: new Set(['非自立']),
+  });
+  const pos2Rule = SUBTITLE_ANNOTATION_RULES.find(({ id }) => id === 'configured-pos2-exclusion');
+
+  assert.equal(pos2Rule?.test(context), 'pass');
+  assert.equal(
+    shouldExcludeTokenFromSubtitleAnnotations(token, {
+      pos1Exclusions: context.pos1Exclusions,
+      pos2Exclusions: context.pos2Exclusions,
+    }),
+    false,
+  );
+});
+
+test('trailing quote-particle rule honors configured POS1 exclusions', () => {
+  const token = makeToken({ surface: '猫って', headword: '猫', pos1: '名詞|助詞' });
+  const context = createSubtitleAnnotationRuleContext(token, {
+    pos1Exclusions: new Set(['名詞']),
+  });
+  const trailingParticleRule = SUBTITLE_ANNOTATION_RULES.find(
+    ({ id }) => id === 'merged-trailing-quote-particle',
+  );
+
+  assert.equal(trailingParticleRule?.test(context), 'pass');
+});
+
+test('configured POS2 rule preserves supplementary-plane kanji nouns', () => {
+  const token = makeToken({ surface: '𠮟', headword: '𠮟', pos1: '名詞', pos2: '非自立' });
+
+  assert.equal(
+    shouldExcludeTokenFromSubtitleAnnotations(token, {
+      pos1Exclusions: new Set(),
+      pos2Exclusions: new Set(['非自立']),
+    }),
+    false,
+  );
+});
+
+test('kana detection excludes katakana punctuation boundaries', () => {
+  assert.equal(isKanaChar('゠'), false);
+  assert.equal(isKanaChar('・'), false);
+  assert.equal(isKanaChar('ァ'), true);
+});

src/core/services/tokenizer/subtitle-annotation-filter.ts

@@ -1,551 +1,31 @@
+import { MergedToken } from '../../../types';
 import {
-  DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG,
-  resolveAnnotationPos1ExclusionSet,
-} from '../../../token-pos1-exclusions';
-import {
-  DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG,
-  resolveAnnotationPos2ExclusionSet,
-} from '../../../token-pos2-exclusions';
-import { MergedToken, PartOfSpeech } from '../../../types';
-import { shouldIgnoreJlptByTerm } from '../jlpt-token-filter';
-import { isSubtitleGrammarEndingText } from './grammar-ending';
-
-const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
-const KATAKANA_CODEPOINT_START = 0x30a1;
-const KATAKANA_CODEPOINT_END = 0x30f6;
-
-const STANDALONE_GRAMMAR_PARTICLE_PHRASES = ['たって', 'だって'] as const;
-const STANDALONE_GRAMMAR_PARTICLE_PHRASES_SET: ReadonlySet<string> = new Set(
-  STANDALONE_GRAMMAR_PARTICLE_PHRASES,
-);
-
-export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
-  'あ',
-  'ああ',
-  'ある',
-  'あなた',
-  'あんた',
-  'ええ',
-  'うう',
-  'おお',
-  'おい',
-  'お前',
-  'こいつ',
-  'こっち',
-  'くれ',
-  'じゃない',
-  'そうだ',
-  'たち',
-  'である',
-  'どこか',
-  'なんか',
-  'べき',
-  'って',
-  'はあ',
-  'はぁ',
-  'はは',
-  'へえ',
-  'ふう',
-  'ほう',
-  '何か',
-  '何だ',
-  '何も',
-  '如何した',
-  '有る',
-  '在る',
-  '様',
-  '誰も',
-  '貴方',
-  'もんか',
-  'ものか',
-]);
-const SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES = new Set([
-  'って',
-  'ってよ',
-  'ってね',
-  'ってな',
-  'ってさ',
-  'ってか',
-  'ってば',
-]);
-const AUXILIARY_STEM_GRAMMAR_TAIL_POS1 = new Set(['名詞', '助動詞', '助詞']);
-const NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1 = new Set(['助詞', '助動詞']);
-const AUXILIARY_INFLECTION_TRAILING_POS1 = new Set(['助動詞']);
-const AUXILIARY_HELPER_SPAN_POS1 = new Set(['助詞', '助動詞', '動詞']);
-const LEXICAL_VERB_POS2 = new Set(['自立']);
-const STANDALONE_GRAMMAR_PARTICLE_SURFACES = new Set([
-  'か',
-  'が',
-  'さ',
-  'し',
-  'ぞ',
-  'ぜ',
-  'と',
-  'な',
-  'に',
-  'ね',
-  'の',
-  'は',
-  'へ',
-  'も',
-  'や',
-  'よ',
-  'を',
-]);
-export interface SubtitleAnnotationFilterOptions {
-  pos1Exclusions?: ReadonlySet<string>;
-  pos2Exclusions?: ReadonlySet<string>;
-}
-
-function normalizePosTag(pos: string | undefined): string {
-  return typeof pos === 'string' ? pos.trim() : '';
-}
-
-function splitNormalizedTagParts(normalizedTag: string): string[] {
-  if (!normalizedTag) {
-    return [];
-  }
-
-  return normalizedTag
-    .split('|')
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
-}
-
-function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<string>): boolean {
-  const parts = splitNormalizedTagParts(normalizedTag);
-  if (parts.length === 0) {
-    return false;
-  }
-
-  return parts.every((part) => exclusions.has(part));
-}
-
-function resolvePos1Exclusions(options: SubtitleAnnotationFilterOptions = {}): ReadonlySet<string> {
-  if (options.pos1Exclusions) {
-    return options.pos1Exclusions;
-  }
-
-  return resolveAnnotationPos1ExclusionSet(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG);
-}
-
-function resolvePos2Exclusions(options: SubtitleAnnotationFilterOptions = {}): ReadonlySet<string> {
-  if (options.pos2Exclusions) {
-    return options.pos2Exclusions;
-  }
-
-  return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
-}
-
-function hasKanjiChar(text: string): boolean {
-  for (const char of text) {
-    const code = char.codePointAt(0);
-    if (code === undefined) {
-      continue;
-    }
-    if (
-      (code >= 0x3400 && code <= 0x4dbf) ||
-      (code >= 0x4e00 && code <= 0x9fff) ||
-      (code >= 0xf900 && code <= 0xfaff)
-    ) {
-      return true;
-    }
-  }
-  return false;
-}
-
-// Kanji-bearing non-independent nouns (日, 方, 上, …) are real vocabulary that
-// Yomitan segments as standalone tokens; MeCab's 非自立 tag exists to suppress
-// kana grammar nouns (こと, もの, とき) and must not hide these.
-export function isKanjiNonIndependentNounToken(
-  token: MergedToken,
-  pos1Exclusions: ReadonlySet<string>,
-): boolean {
-  if (pos1Exclusions.has('名詞')) {
-    return false;
-  }
-
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
-  if (pos1Parts.length !== 1 || pos2Parts.length !== 1) {
-    return false;
-  }
-  if (pos1Parts[0] !== '名詞' || pos2Parts[0] !== '非自立') {
-    return false;
-  }
-
-  return hasKanjiChar(token.surface) || hasKanjiChar(token.headword);
-}
-
-function normalizeKana(text: string): string {
-  const raw = text.trim();
-  if (!raw) {
-    return '';
-  }
-
-  let normalized = '';
-  for (const char of raw) {
-    const code = char.codePointAt(0);
-    if (code === undefined) {
-      continue;
-    }
-
-    if (code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END) {
-      normalized += String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET);
-      continue;
-    }
-
-    normalized += char;
-  }
-
-  return normalized;
-}
-
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
-function isTrailingSmallTsuKanaSfx(text: string): boolean {
-  const normalized = normalizeKana(text);
-  if (!normalized) {
-    return false;
-  }
-
-  const chars = [...normalized];
-  if (chars.length < 2 || chars.length > 4) {
-    return false;
-  }
-
-  if (!chars.every(isKanaChar)) {
-    return false;
-  }
-
-  return chars[chars.length - 1] === 'っ';
-}
-
-function isReduplicatedKanaSfx(text: string): boolean {
-  const normalized = normalizeKana(text);
-  if (!normalized) {
-    return false;
-  }
-
-  const chars = [...normalized];
-  if (chars.length < 4 || chars.length % 2 !== 0) {
-    return false;
-  }
-
-  if (!chars.every(isKanaChar)) {
-    return false;
-  }
-
-  const half = chars.length / 2;
-  return chars.slice(0, half).join('') === chars.slice(half).join('');
-}
-
-function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
-  const normalized = normalizeKana(text);
-  if (!normalized) {
-    return false;
-  }
-
-  if (isReduplicatedKanaSfx(normalized)) {
-    return true;
-  }
-
-  if (normalized.length <= 1 || !normalized.endsWith('と')) {
-    return false;
-  }
-
-  return isReduplicatedKanaSfx(normalized.slice(0, -1));
-}
-
-function isExcludedTrailingParticleMergedToken(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const normalizedHeadword = normalizeKana(token.headword);
-  if (
-    !normalizedSurface ||
-    !normalizedHeadword ||
-    !normalizedSurface.startsWith(normalizedHeadword)
-  ) {
-    return false;
-  }
-
-  const suffix = normalizedSurface.slice(normalizedHeadword.length);
-  if (!SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES.has(suffix)) {
-    return false;
-  }
-
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
-  if (pos1Parts.length < 2) {
-    return false;
-  }
-
-  const [leadingPos1, ...trailingPos1] = pos1Parts;
-  if (!leadingPos1 || resolvePos1Exclusions().has(leadingPos1)) {
-    return false;
-  }
-
-  return trailingPos1.length > 0 && trailingPos1.every((part) => part === '助詞');
-}
-
-function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean {
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
-  if (
-    pos1Parts.length === 0 ||
-    !pos1Parts.every((part) => AUXILIARY_STEM_GRAMMAR_TAIL_POS1.has(part))
-  ) {
-    return false;
-  }
-
-  const pos3Parts = splitNormalizedTagParts(normalizePosTag(token.pos3));
-  return pos3Parts.includes('助動詞語幹');
-}
-
-function isKanaOnlyNonIndependentNounHelperMerge(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const normalizedHeadword = normalizeKana(token.headword);
-  if (
-    !normalizedSurface ||
-    !normalizedHeadword ||
-    normalizedSurface === normalizedHeadword ||
-    ![...normalizedSurface].every(isKanaChar)
-  ) {
-    return false;
-  }
-
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
-  if (pos1Parts.length < 2 || pos1Parts[0] !== '名詞') {
-    return false;
-  }
-
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
-  if (pos2Parts[0] !== '非自立') {
-    return false;
-  }
-
-  return pos1Parts.slice(1).every((part) => NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1.has(part));
-}
-
-function isKanaOnlyText(text: string): boolean {
-  const normalized = normalizeKana(text);
-  return normalized.length > 0 && [...normalized].every(isKanaChar);
-}
-
-function isLexicalKureruVerb(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const normalizedHeadword = normalizeKana(token.headword);
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
-  return (
-    normalizedSurface === 'くれ' &&
-    normalizedHeadword === 'くれる' &&
-    pos1Parts.length === 1 &&
-    pos1Parts[0] === '動詞' &&
-    pos2Parts.length === 1 &&
-    pos2Parts[0] === '自立'
-  );
-}
-
-function isStandaloneAuxiliaryInflectionFragment(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  if (!isKanaOnlyText(normalizedSurface)) {
-    return false;
-  }
-
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
-  if (pos1Parts.length === 0) {
-    return false;
-  }
-
-  if (pos1Parts.every((part) => part === '助動詞')) {
-    return true;
-  }
-
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
-  return (
-    pos1Parts[0] === '動詞' &&
-    pos2Parts[0] === '接尾' &&
-    pos1Parts.slice(1).every((part) => AUXILIARY_INFLECTION_TRAILING_POS1.has(part))
-  );
-}
-
-function isAuxiliaryOnlyHelperSpan(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const normalizedHeadword = normalizeKana(token.headword);
-  if (!isKanaOnlyText(normalizedSurface) || !isKanaOnlyText(normalizedHeadword)) {
-    return false;
-  }
-
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
-  if (
-    pos1Parts.length === 0 ||
-    !pos1Parts.every((part) => AUXILIARY_HELPER_SPAN_POS1.has(part)) ||
-    !pos1Parts.includes('助詞') ||
-    !pos1Parts.includes('動詞')
-  ) {
-    return false;
-  }
-
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
-  return !pos2Parts.some((part) => LEXICAL_VERB_POS2.has(part));
-}
-
-function isStandaloneSuruTeGrammarHelper(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const normalizedHeadword = normalizeKana(token.headword);
-  if (!normalizedSurface.startsWith('して') || normalizedHeadword !== 'する') {
-    return false;
-  }
-
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
-  return (
-    isKanaOnlyText(normalizedSurface) && (pos1Parts.length === 0 || pos1Parts.includes('動詞'))
-  );
-}
-
-function isStandaloneGrammarParticle(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const normalizedHeadword = normalizeKana(token.headword);
-  return (
-    normalizedSurface === normalizedHeadword &&
-    (STANDALONE_GRAMMAR_PARTICLE_SURFACES.has(normalizedSurface) ||
-      STANDALONE_GRAMMAR_PARTICLE_PHRASES_SET.has(normalizedSurface))
-  );
-}
-
-function isSingleKanaSurfaceFragment(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const chars = [...normalizedSurface];
-  return chars.length === 1 && chars.every(isKanaChar);
-}
-
-function isExcludedByTerm(token: MergedToken): boolean {
-  const candidates = [token.surface, token.reading, token.headword].filter(
-    (candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0,
-  );
-
-  for (const candidate of candidates) {
-    const trimmed = candidate.trim();
-    if (!trimmed) {
-      continue;
-    }
-
-    const normalized = normalizeKana(trimmed);
-    if (!normalized) {
-      continue;
-    }
-
-    if (
-      SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(trimmed) ||
-      SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(normalized) ||
-      isSubtitleGrammarEndingText(trimmed) ||
-      isSubtitleGrammarEndingText(normalized) ||
-      shouldIgnoreJlptByTerm(trimmed) ||
-      shouldIgnoreJlptByTerm(normalized)
-    ) {
-      return true;
-    }
-
-    if (
-      isTrailingSmallTsuKanaSfx(trimmed) ||
-      isTrailingSmallTsuKanaSfx(normalized) ||
-      isReduplicatedKanaSfxWithOptionalTrailingTo(trimmed) ||
-      isReduplicatedKanaSfxWithOptionalTrailingTo(normalized)
-    ) {
-      return true;
-    }
-  }
-
-  return false;
-}
+  createSubtitleAnnotationRuleContext,
+  evaluateSubtitleAnnotationRules,
+} from './subtitle-annotation-filter-rules';
+import type { SubtitleAnnotationFilterOptions } from './subtitle-annotation-filter-rules';
+
+export {
+  createSubtitleAnnotationRuleContext,
+  SUBTITLE_ANNOTATION_EXCLUDED_TERMS,
+  SUBTITLE_ANNOTATION_RULES,
+} from './subtitle-annotation-filter-rules';
+export type {
+  SubtitleAnnotationFilterOptions,
+  SubtitleAnnotationRule,
+  SubtitleAnnotationRuleContext,
+  SubtitleAnnotationRuleDecision,
+} from './subtitle-annotation-filter-rules';
+export { isKanjiNonIndependentNounToken } from './token-classification';
 
 export function shouldExcludeTokenFromSubtitleAnnotations(
   token: MergedToken,
   options: SubtitleAnnotationFilterOptions = {},
 ): boolean {
-  // No Yomitan dictionary entry backs this token (ぅ～ elongations, truncated
-  // inflections) — it exists only to stay hoverable and must never receive
-  // annotations or count in the N+1 math.
-  if (token.isUnparsedRun === true) {
-    return true;
-  }
-
-  const pos1Exclusions = resolvePos1Exclusions(options);
-  const pos2Exclusions = resolvePos2Exclusions(options);
-  const normalizedPos1 = normalizePosTag(token.pos1);
-  const normalizedPos2 = normalizePosTag(token.pos2);
-  const hasPos1 = normalizedPos1.length > 0;
-  const hasPos2 = normalizedPos2.length > 0;
-
-  if (isExcludedByTagSet(normalizedPos1, pos1Exclusions)) {
-    return true;
-  }
-
-  if (
-    isExcludedByTagSet(normalizedPos2, pos2Exclusions) &&
-    !isKanjiNonIndependentNounToken(token, pos1Exclusions)
-  ) {
-    return true;
-  }
-
-  if (
-    !hasPos1 &&
-    !hasPos2 &&
-    (token.partOfSpeech === PartOfSpeech.particle ||
-      token.partOfSpeech === PartOfSpeech.bound_auxiliary ||
-      token.partOfSpeech === PartOfSpeech.symbol)
-  ) {
-    return true;
-  }
-
-  if (isAuxiliaryStemGrammarTailToken(token)) {
-    return true;
-  }
-
-  if (isKanaOnlyNonIndependentNounHelperMerge(token)) {
-    return true;
-  }
-
-  if (isStandaloneAuxiliaryInflectionFragment(token)) {
-    return true;
-  }
-
-  if (isAuxiliaryOnlyHelperSpan(token)) {
-    return true;
-  }
-
-  if (isStandaloneSuruTeGrammarHelper(token)) {
-    return true;
-  }
-
-  if (isStandaloneGrammarParticle(token)) {
-    return true;
-  }
-
-  if (isSingleKanaSurfaceFragment(token)) {
-    return true;
-  }
-
-  if (isExcludedTrailingParticleMergedToken(token)) {
-    return true;
-  }
-
-  if (isLexicalKureruVerb(token)) {
-    return false;
-  }
-
-  return isExcludedByTerm(token);
+  return (
+    evaluateSubtitleAnnotationRules(createSubtitleAnnotationRuleContext(token, options)) ===
+    'exclude'
+  );
 }
 
 export function stripSubtitleAnnotationMetadata(

src/core/services/tokenizer/token-classification.test.ts

@@ -0,0 +1,57 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { MergedToken, PartOfSpeech } from '../../../types';
+import {
+  isContentTokenByPos,
+  isKanaCandidateIgnorableChar,
+  isKanaCandidateText,
+  isKanaChar,
+  isKanaOnlyText,
+  isTokenPos2Excluded,
+} from './token-classification';
+
+const POS1_EXCLUSIONS = new Set(['助詞']);
+const POS2_EXCLUSIONS = new Set(['非自立']);
+
+function makeNoun(surface: string): MergedToken {
+  return {
+    surface,
+    reading: surface,
+    headword: surface,
+    startPos: 0,
+    endPos: surface.length,
+    partOfSpeech: PartOfSpeech.noun,
+    pos1: '名詞',
+    pos2: '非自立',
+    isMerged: false,
+    isKnown: false,
+    isNPlusOneTarget: false,
+  };
+}
+
+test('kana classification excludes the katakana-hiragana double hyphen', () => {
+  assert.equal(isKanaChar('゠'), false);
+  assert.equal(isKanaOnlyText('゠'), false);
+});
+
+test('POS classification keeps kanji non-independent nouns as content', () => {
+  const token = makeNoun('日');
+
+  assert.equal(isTokenPos2Excluded(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), false);
+  assert.equal(isContentTokenByPos(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), true);
+});
+
+test('POS classification excludes kana non-independent nouns', () => {
+  const token = makeNoun('こと');
+
+  assert.equal(isTokenPos2Excluded(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), true);
+  assert.equal(isContentTokenByPos(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), false);
+});
+
+test('kana candidate classification allows punctuation around kana only', () => {
+  assert.equal(isKanaCandidateIgnorableChar('！'), true);
+  assert.equal(isKanaCandidateIgnorableChar('猫'), false);
+  assert.equal(isKanaCandidateText('「かな！？」'), true);
+  assert.equal(isKanaCandidateText('「！？」'), false);
+  assert.equal(isKanaCandidateText('かな猫'), false);
+});

src/core/services/tokenizer/token-classification.ts

@@ -0,0 +1,169 @@
+import { MergedToken, PartOfSpeech } from '../../../types';
+
+const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
+const KATAKANA_CODEPOINT_START = 0x30a1;
+const KATAKANA_CODEPOINT_END = 0x30f6;
+
+export function normalizeKana(text: string): string {
+  const raw = text.trim();
+  if (!raw) {
+    return '';
+  }
+
+  let normalized = '';
+  for (const char of raw) {
+    const code = char.codePointAt(0);
+    if (code === undefined) {
+      continue;
+    }
+
+    normalized +=
+      code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END
+        ? String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET)
+        : char;
+  }
+
+  return normalized;
+}
+
+export function isKanaChar(char: string): boolean {
+  const code = char.codePointAt(0);
+  if (code === undefined) {
+    return false;
+  }
+
+  return (
+    (code >= 0x3041 && code <= 0x3096) ||
+    (code >= 0x309b && code <= 0x309f) ||
+    code === 0x30fc ||
+    (code >= 0x30a1 && code <= 0x30fa) ||
+    (code >= 0x30fd && code <= 0x30ff)
+  );
+}
+
+export function isKanaOnlyText(text: string | null | undefined): boolean {
+  if (typeof text !== 'string') {
+    return false;
+  }
+
+  const normalized = normalizeKana(text);
+  return normalized.length > 0 && [...normalized].every(isKanaChar);
+}
+
+export function isKanaCandidateIgnorableChar(char: string): boolean {
+  return /^[\s.,!?;:()[\]{}"'`、。！？…‥・「」『』（）［］｛｝〈〉《》【】―-]$/u.test(char);
+}
+
+export function isKanaCandidateText(text: string): boolean {
+  const normalized = text.trim();
+  if (normalized.length === 0) {
+    return false;
+  }
+
+  let hasKana = false;
+  for (const char of normalized) {
+    if (isKanaChar(char)) {
+      hasKana = true;
+      continue;
+    }
+    if (!isKanaCandidateIgnorableChar(char)) {
+      return false;
+    }
+  }
+
+  return hasKana;
+}
+
+export function normalizePosTag(value: string | null | undefined): string {
+  return typeof value === 'string' ? value.trim() : '';
+}
+
+export function splitPosTag(value: string | null | undefined): string[] {
+  const normalized = normalizePosTag(value);
+  if (!normalized) {
+    return [];
+  }
+
+  return normalized
+    .split('|')
+    .map((part) => part.trim())
+    .filter((part) => part.length > 0);
+}
+
+export function isPosTagExcluded(
+  value: string | null | undefined,
+  exclusions: ReadonlySet<string>,
+): boolean {
+  const parts = splitPosTag(value);
+  return parts.length > 0 && parts.every((part) => exclusions.has(part));
+}
+
+function hasKanjiChar(text: string): boolean {
+  for (const char of text) {
+    const code = char.codePointAt(0);
+    if (
+      code !== undefined &&
+      ((code >= 0x3400 && code <= 0x4dbf) ||
+        (code >= 0x4e00 && code <= 0x9fff) ||
+        (code >= 0xf900 && code <= 0xfaff) ||
+        (code >= 0x20000 && code <= 0x2fa1f) ||
+        (code >= 0x30000 && code <= 0x323af))
+    ) {
+      return true;
+    }
+  }
+  return false;
+}
+
+// MeCab's 非自立 tag suppresses kana grammar nouns (こと, もの, とき), but
+// Yomitan can segment kanji-bearing nouns (日, 方, 上, …) as real vocabulary.
+export function isKanjiNonIndependentNounToken(
+  token: MergedToken,
+  pos1Exclusions: ReadonlySet<string>,
+): boolean {
+  if (pos1Exclusions.has('名詞')) {
+    return false;
+  }
+
+  const pos1Parts = splitPosTag(token.pos1);
+  const pos2Parts = splitPosTag(token.pos2);
+  return (
+    pos1Parts.length === 1 &&
+    pos1Parts[0] === '名詞' &&
+    pos2Parts.length === 1 &&
+    pos2Parts[0] === '非自立' &&
+    (hasKanjiChar(token.surface) || hasKanjiChar(token.headword))
+  );
+}
+
+export function isTokenPos2Excluded(
+  token: MergedToken,
+  pos1Exclusions: ReadonlySet<string>,
+  pos2Exclusions: ReadonlySet<string>,
+): boolean {
+  return (
+    isPosTagExcluded(token.pos2, pos2Exclusions) &&
+    !isKanjiNonIndependentNounToken(token, pos1Exclusions)
+  );
+}
+
+export function isContentTokenByPos(
+  token: MergedToken,
+  pos1Exclusions: ReadonlySet<string>,
+  pos2Exclusions: ReadonlySet<string>,
+): boolean {
+  if (
+    isPosTagExcluded(token.pos1, pos1Exclusions) ||
+    isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions)
+  ) {
+    return false;
+  }
+
+  if (splitPosTag(token.pos1).length > 0 || splitPosTag(token.pos2).length > 0) {
+    return true;
+  }
+
+  return ![PartOfSpeech.particle, PartOfSpeech.bound_auxiliary, PartOfSpeech.symbol].includes(
+    token.partOfSpeech,
+  );
+}

src/token-merger.ts

@@ -20,6 +20,7 @@ import { PartOfSpeech, Token, MergedToken } from './types';
 import { DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG } from './token-pos1-exclusions';
 import { DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG } from './token-pos2-exclusions';
 import { shouldExcludeTokenFromSubtitleAnnotations } from './core/services/tokenizer/subtitle-annotation-filter';
+import { isKanaCandidateText } from './core/services/tokenizer/token-classification';
 
 export function isNoun(tok: Token): boolean {
   return tok.partOfSpeech === PartOfSpeech.noun;
@@ -261,67 +262,6 @@ const SENTENCE_BOUNDARY_SURFACES = new Set(['。', '？', '！', '?', '!', '…'
 const N_PLUS_ONE_IGNORED_POS1 = new Set(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG.defaults);
 const N_PLUS_ONE_IGNORED_POS2 = new Set(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG.defaults);
 
-function normalizePos1Tag(pos1: string | undefined): string {
-  return typeof pos1 === 'string' ? pos1.trim() : '';
-}
-
-function normalizePos2Tag(pos2: string | undefined): string {
-  return typeof pos2 === 'string' ? pos2.trim() : '';
-}
-
-function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<string>): boolean {
-  if (!normalizedTag) {
-    return false;
-  }
-  const parts = normalizedTag
-    .split('|')
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
-  if (parts.length === 0) {
-    return false;
-  }
-  return parts.every((part) => exclusions.has(part));
-}
-
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
-function isKanaCandidateIgnorableChar(char: string): boolean {
-  return /^[\s.,!?;:()[\]{}"'`、。！？…‥・「」『』（）［］｛｝〈〉《》【】―-]$/u.test(char);
-}
-
-function isKanaOnlyText(text: string): boolean {
-  const normalized = text.trim();
-  if (normalized.length === 0) {
-    return false;
-  }
-
-  let hasKana = false;
-  for (const char of normalized) {
-    if (isKanaChar(char)) {
-      hasKana = true;
-      continue;
-    }
-    if (!isKanaCandidateIgnorableChar(char)) {
-      return false;
-    }
-  }
-
-  return hasKana;
-}
-
 function normalizeSourceTextForTokenOffsets(sourceText: string | undefined): string | undefined {
   return typeof sourceText === 'string' ? sourceText.replace(/\r?\n/g, ' ').trim() : undefined;
 }
@@ -334,7 +274,7 @@ export function isNPlusOneCandidateToken(
   if (token.isKnown) {
     return false;
   }
-  if (isKanaOnlyText(token.surface)) {
+  if (isKanaCandidateText(token.surface)) {
     return false;
   }
   return isNPlusOneWordCountToken(token, pos1Exclusions, pos2Exclusions);
@@ -349,28 +289,6 @@ function isNPlusOneWordCountToken(
     return false;
   }
 
-  const normalizedPos1 = normalizePos1Tag(token.pos1);
-  const hasPos1 = normalizedPos1.length > 0;
-  if (isExcludedByTagSet(normalizedPos1, pos1Exclusions)) {
-    return false;
-  }
-
-  const normalizedPos2 = normalizePos2Tag(token.pos2);
-  const hasPos2 = normalizedPos2.length > 0;
-  if (isExcludedByTagSet(normalizedPos2, pos2Exclusions)) {
-    return false;
-  }
-
-  if (
-    !hasPos1 &&
-    !hasPos2 &&
-    (token.partOfSpeech === PartOfSpeech.particle ||
-      token.partOfSpeech === PartOfSpeech.bound_auxiliary ||
-      token.partOfSpeech === PartOfSpeech.symbol)
-  ) {
-    return false;
-  }
-
   if (token.partOfSpeech === PartOfSpeech.noun && token.pos2 === '固有名詞') {
     return false;
   }