refactor(tokenizer): extract subtitle annotation filter into rule table (#162)

changes/annotation-noise-rule-table.md

@@ -0,0 +1,4 @@
+type: fixed
+area: overlay
+
+- Applied configured primary POS exclusions consistently to merged trailing quote-particle tokens, preserved annotations for supplementary-plane kanji, and stopped treating katakana punctuation as kana-only annotation noise.

src/core/services/tokenizer/subtitle-annotation-filter-rules.ts

@@ -0,0 +1,342 @@
+import {
+  DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG,
+  resolveAnnotationPos1ExclusionSet,
+} from '../../../token-pos1-exclusions';
+import {
+  DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG,
+  resolveAnnotationPos2ExclusionSet,
+} from '../../../token-pos2-exclusions';
+import { MergedToken } from '../../../types';
+import { shouldIgnoreJlptByTerm } from '../jlpt-token-filter';
+import { isSubtitleGrammarEndingText } from './grammar-ending';
+import {
+  isAuxiliaryOnlyHelperSpan,
+  isAuxiliaryStemGrammarTailToken,
+  isExcludedTrailingParticleMergedToken,
+  isKanaOnlyNonIndependentNounHelperMerge,
+  isReduplicatedKanaSfxWithOptionalTrailingTo,
+  isSingleKanaSurfaceFragment,
+  isStandaloneAuxiliaryInflectionFragment,
+  isStandaloneGrammarParticle,
+  isStandaloneSuruTeGrammarHelper,
+  isTrailingSmallTsuKanaSfx,
+} from './subtitle-annotation-filter-support';
+import {
+  isContentTokenByPos,
+  isPosTagExcluded,
+  isTokenPos2Excluded,
+  normalizeKana,
+  normalizePosTag,
+  splitPosTag,
+} from './token-classification';
+
+export interface SubtitleAnnotationFilterOptions {
+  pos1Exclusions?: ReadonlySet<string>;
+  pos2Exclusions?: ReadonlySet<string>;
+}
+
+export type SubtitleAnnotationRuleDecision = 'exclude' | 'keep' | 'pass';
+
+export interface SubtitleAnnotationRuleContext {
+  token: MergedToken;
+  pos1Exclusions: ReadonlySet<string>;
+  pos2Exclusions: ReadonlySet<string>;
+  normalizedPos1: string;
+  normalizedPos2: string;
+  hasPos1: boolean;
+  hasPos2: boolean;
+}
+
+type SubtitleAnnotationRuleData = Readonly<Record<string, readonly string[]>>;
+
+export interface SubtitleAnnotationRule {
+  readonly id: string;
+  readonly description: string;
+  readonly issueRef: string;
+  readonly data: SubtitleAnnotationRuleData;
+  test(context: SubtitleAnnotationRuleContext): SubtitleAnnotationRuleDecision;
+}
+
+function defineRule<TData extends SubtitleAnnotationRuleData>(definition: {
+  id: string;
+  description: string;
+  issueRef: string;
+  data: TData;
+  test: (context: SubtitleAnnotationRuleContext, data: TData) => SubtitleAnnotationRuleDecision;
+}): SubtitleAnnotationRule {
+  const data = Object.freeze(
+    Object.fromEntries(
+      Object.entries(definition.data).map(([key, values]) => [key, Object.freeze([...values])]),
+    ),
+  ) as TData;
+  return Object.freeze({
+    id: definition.id,
+    description: definition.description,
+    issueRef: definition.issueRef,
+    data,
+    test: (context: SubtitleAnnotationRuleContext) => definition.test(context, data),
+  });
+}
+
+export function createSubtitleAnnotationRuleContext(
+  token: MergedToken,
+  options: SubtitleAnnotationFilterOptions = {},
+): SubtitleAnnotationRuleContext {
+  const pos1Exclusions =
+    options.pos1Exclusions ??
+    resolveAnnotationPos1ExclusionSet(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG);
+  const pos2Exclusions =
+    options.pos2Exclusions ??
+    resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
+  const normalizedPos1 = normalizePosTag(token.pos1);
+  const normalizedPos2 = normalizePosTag(token.pos2);
+  return {
+    token,
+    pos1Exclusions,
+    pos2Exclusions,
+    normalizedPos1,
+    normalizedPos2,
+    hasPos1: normalizedPos1.length > 0,
+    hasPos2: normalizedPos2.length > 0,
+  };
+}
+
+function matchesExcludedTermOrPattern(token: MergedToken, terms: ReadonlySet<string>): boolean {
+  const candidates = [token.surface, token.reading, token.headword].filter(
+    (candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0,
+  );
+
+  for (const candidate of candidates) {
+    const trimmed = candidate.trim();
+    if (!trimmed) {
+      continue;
+    }
+    const normalized = normalizeKana(trimmed);
+    if (!normalized) {
+      continue;
+    }
+    if (
+      terms.has(trimmed) ||
+      terms.has(normalized) ||
+      isSubtitleGrammarEndingText(trimmed) ||
+      isSubtitleGrammarEndingText(normalized) ||
+      shouldIgnoreJlptByTerm(trimmed) ||
+      shouldIgnoreJlptByTerm(normalized) ||
+      isTrailingSmallTsuKanaSfx(trimmed) ||
+      isTrailingSmallTsuKanaSfx(normalized) ||
+      isReduplicatedKanaSfxWithOptionalTrailingTo(trimmed) ||
+      isReduplicatedKanaSfxWithOptionalTrailingTo(normalized)
+    ) {
+      return true;
+    }
+  }
+  return false;
+}
+
+export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
+  'あ',
+  'ああ',
+  'ある',
+  'あなた',
+  'あんた',
+  'ええ',
+  'うう',
+  'おお',
+  'おい',
+  'お前',
+  'こいつ',
+  'こっち',
+  'くれ',
+  'じゃない',
+  'そうだ',
+  'たち',
+  'である',
+  'どこか',
+  'なんか',
+  'べき',
+  'って',
+  'はあ',
+  'はぁ',
+  'はは',
+  'へえ',
+  'ふう',
+  'ほう',
+  '何か',
+  '何だ',
+  '何も',
+  '如何した',
+  '有る',
+  '在る',
+  '様',
+  '誰も',
+  '貴方',
+  'もんか',
+  'ものか',
+]);
+
+const excludedTermRule = defineRule({
+  id: 'excluded-term-or-pattern',
+  description:
+    'Exclude legacy subtitle stop terms, grammar endings, JLPT stop terms, and kana sound effects.',
+  issueRef: '#19, #33, #57',
+  data: {},
+  test: ({ token }) =>
+    matchesExcludedTermOrPattern(token, SUBTITLE_ANNOTATION_EXCLUDED_TERMS) ? 'exclude' : 'pass',
+});
+
+// Ordered, first-match-wins. issueRef records the introducing/fixing PR; early
+// rules are legacy behavior inherited from the original #19 filter.
+export const SUBTITLE_ANNOTATION_RULES: readonly SubtitleAnnotationRule[] = Object.freeze([
+  defineRule({
+    id: 'unparsed-run',
+    description: 'Exclude hoverable parser gaps that have no Yomitan dictionary entry.',
+    issueRef: '#153',
+    data: {},
+    test: ({ token }) => (token.isUnparsedRun === true ? 'exclude' : 'pass'),
+  }),
+  defineRule({
+    id: 'configured-pos1-exclusion',
+    description: 'Apply the configured primary part-of-speech exclusions.',
+    issueRef: '#19',
+    data: {},
+    test: ({ token, pos1Exclusions }) =>
+      isPosTagExcluded(token.pos1, pos1Exclusions) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'configured-pos2-exclusion',
+    description: 'Apply configured secondary POS exclusions, preserving #150 kanji nouns.',
+    issueRef: '#150',
+    data: {},
+    test: ({ token, pos1Exclusions, pos2Exclusions }) =>
+      isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'coarse-grammar-pos-fallback',
+    description: 'Exclude coarse grammar POS when detailed MeCab tags are unavailable.',
+    issueRef: '#19',
+    data: {},
+    test: ({ token, hasPos1, hasPos2, pos1Exclusions, pos2Exclusions }) =>
+      !hasPos1 && !hasPos2 && !isContentTokenByPos(token, pos1Exclusions, pos2Exclusions)
+        ? 'exclude'
+        : 'pass',
+  }),
+  defineRule({
+    id: 'auxiliary-stem-grammar-tail',
+    description: 'Exclude merged grammar tails containing an auxiliary stem.',
+    issueRef: '#19',
+    data: { allowedPos1: ['名詞', '助動詞', '助詞'] },
+    test: ({ token }, { allowedPos1 }) =>
+      isAuxiliaryStemGrammarTailToken(token, allowedPos1) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'kana-non-independent-noun-helper',
+    description: 'Exclude kana non-independent nouns merged with grammar helpers.',
+    issueRef: '#56',
+    data: { tailPos1: ['助詞', '助動詞'] },
+    test: ({ token }, { tailPos1 }) =>
+      isKanaOnlyNonIndependentNounHelperMerge(token, tailPos1) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'standalone-auxiliary-inflection',
+    description: 'Exclude standalone kana auxiliary inflection fragments.',
+    issueRef: '#57',
+    data: { trailingPos1: ['助動詞'] },
+    test: ({ token }, { trailingPos1 }) =>
+      isStandaloneAuxiliaryInflectionFragment(token, trailingPos1) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'auxiliary-only-helper-span',
+    description: 'Exclude kana helper spans without an independent lexical verb.',
+    issueRef: '#57',
+    data: { allowedPos1: ['助詞', '助動詞', '動詞'], lexicalVerbPos2: ['自立'] },
+    test: ({ token }, { allowedPos1, lexicalVerbPos2 }) =>
+      isAuxiliaryOnlyHelperSpan(token, allowedPos1, lexicalVerbPos2) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'standalone-suru-te-helper',
+    description: 'Exclude standalone して grammar-helper fragments.',
+    issueRef: '#57',
+    data: {},
+    test: ({ token }) => (isStandaloneSuruTeGrammarHelper(token) ? 'exclude' : 'pass'),
+  }),
+  defineRule({
+    id: 'standalone-grammar-particle',
+    description: 'Exclude standalone particle surfaces and connective particle phrases.',
+    issueRef: '#57',
+    data: {
+      surfaces: [
+        'か',
+        'が',
+        'さ',
+        'し',
+        'ぞ',
+        'ぜ',
+        'と',
+        'な',
+        'に',
+        'ね',
+        'の',
+        'は',
+        'へ',
+        'も',
+        'や',
+        'よ',
+        'を',
+      ],
+      phrases: ['たって', 'だって'],
+    },
+    test: ({ token }, { surfaces, phrases }) =>
+      isStandaloneGrammarParticle(token, surfaces, phrases) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'single-kana-fragment',
+    description: 'Exclude isolated one-kana parser fragments.',
+    issueRef: '#57',
+    data: {},
+    test: ({ token }) => (isSingleKanaSurfaceFragment(token) ? 'exclude' : 'pass'),
+  }),
+  defineRule({
+    id: 'merged-trailing-quote-particle',
+    description: 'Exclude lexical tokens merged only with a trailing quote-particle suffix.',
+    issueRef: '#19',
+    data: { suffixes: ['って', 'ってよ', 'ってね', 'ってな', 'ってさ', 'ってか', 'ってば'] },
+    test: ({ token, pos1Exclusions }, { suffixes }) =>
+      isExcludedTrailingParticleMergedToken(token, suffixes, pos1Exclusions) ? 'exclude' : 'pass',
+  }),
+  defineRule({
+    id: 'lexical-kureru-keep',
+    description: 'Keep lexical くれる before the legacy bare-くれ stop-term rule.',
+    issueRef: '#57',
+    data: {
+      surfaces: ['くれ'],
+      headwords: ['くれる'],
+      pos1: ['動詞'],
+      pos2: ['自立'],
+    },
+    test: ({ token }, data) => {
+      const pos1 = splitPosTag(token.pos1);
+      const pos2 = splitPosTag(token.pos2);
+      return data.surfaces.includes(normalizeKana(token.surface)) &&
+        data.headwords.includes(normalizeKana(token.headword)) &&
+        pos1.length === 1 &&
+        data.pos1.includes(pos1[0] ?? '') &&
+        pos2.length === 1 &&
+        data.pos2.includes(pos2[0] ?? '')
+        ? 'keep'
+        : 'pass';
+    },
+  }),
+  excludedTermRule,
+]);
+
+export function evaluateSubtitleAnnotationRules(
+  context: SubtitleAnnotationRuleContext,
+): SubtitleAnnotationRuleDecision {
+  for (const rule of SUBTITLE_ANNOTATION_RULES) {
+    const decision = rule.test(context);
+    if (decision !== 'pass') {
+      return decision;
+    }
+  }
+  return 'pass';
+}

src/core/services/tokenizer/subtitle-annotation-filter-support.ts

@@ -0,0 +1,153 @@
+import { MergedToken } from '../../../types';
+import { isKanaChar, isKanaOnlyText, normalizeKana, splitPosTag } from './token-classification';
+
+export function isTrailingSmallTsuKanaSfx(text: string): boolean {
+  const chars = [...normalizeKana(text)];
+  return (
+    chars.length >= 2 &&
+    chars.length <= 4 &&
+    chars.every(isKanaChar) &&
+    chars[chars.length - 1] === 'っ'
+  );
+}
+
+function isReduplicatedKanaSfx(text: string): boolean {
+  const chars = [...normalizeKana(text)];
+  if (chars.length < 4 || chars.length % 2 !== 0 || !chars.every(isKanaChar)) {
+    return false;
+  }
+  const half = chars.length / 2;
+  return chars.slice(0, half).join('') === chars.slice(half).join('');
+}
+
+export function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
+  const normalized = normalizeKana(text);
+  if (!normalized) {
+    return false;
+  }
+  if (isReduplicatedKanaSfx(normalized)) {
+    return true;
+  }
+  return normalized.length > 1 && normalized.endsWith('と')
+    ? isReduplicatedKanaSfx(normalized.slice(0, -1))
+    : false;
+}
+
+export function isExcludedTrailingParticleMergedToken(
+  token: MergedToken,
+  suffixes: readonly string[],
+  leadingPos1Exclusions: ReadonlySet<string>,
+): boolean {
+  const surface = normalizeKana(token.surface);
+  const headword = normalizeKana(token.headword);
+  if (!surface || !headword || !surface.startsWith(headword)) {
+    return false;
+  }
+  if (!suffixes.includes(surface.slice(headword.length))) {
+    return false;
+  }
+
+  const [leadingPos1, ...trailingPos1] = splitPosTag(token.pos1);
+  if (!leadingPos1 || leadingPos1Exclusions.has(leadingPos1)) {
+    return false;
+  }
+  return trailingPos1.length > 0 && trailingPos1.every((part) => part === '助詞');
+}
+
+export function isAuxiliaryStemGrammarTailToken(
+  token: MergedToken,
+  allowedPos1: readonly string[],
+): boolean {
+  const pos1Parts = splitPosTag(token.pos1);
+  if (pos1Parts.length === 0 || !pos1Parts.every((part) => allowedPos1.includes(part))) {
+    return false;
+  }
+  return splitPosTag(token.pos3).includes('助動詞語幹');
+}
+
+export function isKanaOnlyNonIndependentNounHelperMerge(
+  token: MergedToken,
+  tailPos1: readonly string[],
+): boolean {
+  const surface = normalizeKana(token.surface);
+  const headword = normalizeKana(token.headword);
+  if (!surface || !headword || surface === headword || ![...surface].every(isKanaChar)) {
+    return false;
+  }
+
+  const pos1Parts = splitPosTag(token.pos1);
+  if (pos1Parts.length < 2 || pos1Parts[0] !== '名詞') {
+    return false;
+  }
+  const pos2Parts = splitPosTag(token.pos2);
+  return pos2Parts[0] === '非自立' && pos1Parts.slice(1).every((part) => tailPos1.includes(part));
+}
+
+export function isStandaloneAuxiliaryInflectionFragment(
+  token: MergedToken,
+  trailingPos1: readonly string[],
+): boolean {
+  if (!isKanaOnlyText(token.surface)) {
+    return false;
+  }
+  const pos1Parts = splitPosTag(token.pos1);
+  if (pos1Parts.length === 0) {
+    return false;
+  }
+  if (pos1Parts.every((part) => part === '助動詞')) {
+    return true;
+  }
+  const pos2Parts = splitPosTag(token.pos2);
+  return (
+    pos1Parts[0] === '動詞' &&
+    pos2Parts[0] === '接尾' &&
+    pos1Parts.slice(1).every((part) => trailingPos1.includes(part))
+  );
+}
+
+export function isAuxiliaryOnlyHelperSpan(
+  token: MergedToken,
+  allowedPos1: readonly string[],
+  lexicalVerbPos2: readonly string[],
+): boolean {
+  if (!isKanaOnlyText(token.surface) || !isKanaOnlyText(token.headword)) {
+    return false;
+  }
+  const pos1Parts = splitPosTag(token.pos1);
+  if (
+    pos1Parts.length === 0 ||
+    !pos1Parts.every((part) => allowedPos1.includes(part)) ||
+    !pos1Parts.includes('助詞') ||
+    !pos1Parts.includes('動詞')
+  ) {
+    return false;
+  }
+  return !splitPosTag(token.pos2).some((part) => lexicalVerbPos2.includes(part));
+}
+
+export function isStandaloneSuruTeGrammarHelper(token: MergedToken): boolean {
+  const surface = normalizeKana(token.surface);
+  const headword = normalizeKana(token.headword);
+  if (!surface.startsWith('して') || headword !== 'する') {
+    return false;
+  }
+  const pos1Parts = splitPosTag(token.pos1);
+  return isKanaOnlyText(surface) && (pos1Parts.length === 0 || pos1Parts.includes('動詞'));
+}
+
+export function isStandaloneGrammarParticle(
+  token: MergedToken,
+  surfaces: readonly string[],
+  phrases: readonly string[],
+): boolean {
+  const surface = normalizeKana(token.surface);
+  return (
+    surface === normalizeKana(token.headword) &&
+    (surfaces.includes(surface) || phrases.includes(surface))
+  );
+}
+
+export function isSingleKanaSurfaceFragment(token: MergedToken): boolean {
+  const chars = [...normalizeKana(token.surface)];
+  return chars.length === 1 && chars.every(isKanaChar);
+}

src/core/services/tokenizer/subtitle-annotation-filter.test.ts

@@ -0,0 +1,183 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { MergedToken, PartOfSpeech } from '../../../types';
+import {
+  createSubtitleAnnotationRuleContext,
+  shouldExcludeTokenFromSubtitleAnnotations,
+  SUBTITLE_ANNOTATION_EXCLUDED_TERMS,
+  SUBTITLE_ANNOTATION_RULES,
+} from './subtitle-annotation-filter';
+import { isKanaChar } from './token-classification';
+
+function makeToken(overrides: Partial<MergedToken> = {}): MergedToken {
+  return {
+    surface: '猫',
+    reading: 'ネコ',
+    headword: '猫',
+    startPos: 0,
+    endPos: 1,
+    partOfSpeech: PartOfSpeech.noun,
+    isMerged: false,
+    isKnown: false,
+    isNPlusOneTarget: false,
+    ...overrides,
+  };
+}
+
+test('subtitle annotation rules expose stable ordered provenance', () => {
+  assert.deepEqual(
+    SUBTITLE_ANNOTATION_RULES.map(({ id, issueRef }) => ({ id, issueRef })),
+    [
+      { id: 'unparsed-run', issueRef: '#153' },
+      { id: 'configured-pos1-exclusion', issueRef: '#19' },
+      { id: 'configured-pos2-exclusion', issueRef: '#150' },
+      { id: 'coarse-grammar-pos-fallback', issueRef: '#19' },
+      { id: 'auxiliary-stem-grammar-tail', issueRef: '#19' },
+      { id: 'kana-non-independent-noun-helper', issueRef: '#56' },
+      { id: 'standalone-auxiliary-inflection', issueRef: '#57' },
+      { id: 'auxiliary-only-helper-span', issueRef: '#57' },
+      { id: 'standalone-suru-te-helper', issueRef: '#57' },
+      { id: 'standalone-grammar-particle', issueRef: '#57' },
+      { id: 'single-kana-fragment', issueRef: '#57' },
+      { id: 'merged-trailing-quote-particle', issueRef: '#19' },
+      { id: 'lexical-kureru-keep', issueRef: '#57' },
+      { id: 'excluded-term-or-pattern', issueRef: '#19, #33, #57' },
+    ],
+  );
+  assert.equal(new Set(SUBTITLE_ANNOTATION_RULES.map(({ id }) => id)).size, 14);
+  assert.ok(SUBTITLE_ANNOTATION_RULES.every(({ description }) => description.length > 0));
+  assert.ok(
+    SUBTITLE_ANNOTATION_RULES.every(
+      ({ data }) => Object.isFrozen(data) && Object.values(data).every(Object.isFrozen),
+    ),
+  );
+});
+
+test('lexical kureru keep rule precedes and overrides the excluded-term rule', () => {
+  const token = makeToken({
+    surface: 'くれ',
+    headword: 'くれる',
+    reading: 'クレ',
+    partOfSpeech: PartOfSpeech.verb,
+    pos1: '動詞',
+    pos2: '自立',
+  });
+  const context = createSubtitleAnnotationRuleContext(token);
+  const keepIndex = SUBTITLE_ANNOTATION_RULES.findIndex(({ id }) => id === 'lexical-kureru-keep');
+  const termIndex = SUBTITLE_ANNOTATION_RULES.findIndex(
+    ({ id }) => id === 'excluded-term-or-pattern',
+  );
+
+  assert.ok(keepIndex >= 0 && keepIndex < termIndex);
+  assert.ok(SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has('くれ'));
+  assert.equal(SUBTITLE_ANNOTATION_RULES[keepIndex]?.test(context), 'keep');
+  assert.equal(SUBTITLE_ANNOTATION_RULES[termIndex]?.test(context), 'exclude');
+  assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
+});
+
+test('adding an exported excluded term changes annotation matching', () => {
+  const term = '超語彙';
+  const token = makeToken({
+    surface: term,
+    headword: term,
+    reading: term,
+    pos1: '名詞',
+    pos2: '一般',
+  });
+
+  SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term);
+  try {
+    assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
+    SUBTITLE_ANNOTATION_EXCLUDED_TERMS.add(term);
+    assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
+  } finally {
+    SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term);
+  }
+});
+
+test('deleting an exported excluded term changes annotation matching', () => {
+  const term = 'あなた';
+  const token = makeToken({
+    surface: term,
+    headword: term,
+    reading: term,
+    pos1: '名詞',
+    pos2: '一般',
+  });
+
+  assert.ok(SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(term));
+  try {
+    SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term);
+    assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
+  } finally {
+    SUBTITLE_ANNOTATION_EXCLUDED_TERMS.add(term);
+  }
+});
+
+test('configured POS rules consume the context exclusion sets in table order', () => {
+  const token = makeToken({ pos1: '名詞', pos2: '一般' });
+  const context = createSubtitleAnnotationRuleContext(token, {
+    pos1Exclusions: new Set(['名詞']),
+    pos2Exclusions: new Set(['一般']),
+  });
+  const pos1Rule = SUBTITLE_ANNOTATION_RULES.find(({ id }) => id === 'configured-pos1-exclusion');
+  const pos2Rule = SUBTITLE_ANNOTATION_RULES.find(({ id }) => id === 'configured-pos2-exclusion');
+
+  assert.equal(pos1Rule?.test(context), 'exclude');
+  assert.equal(pos2Rule?.test(context), 'exclude');
+  assert.equal(
+    shouldExcludeTokenFromSubtitleAnnotations(token, {
+      pos1Exclusions: context.pos1Exclusions,
+      pos2Exclusions: context.pos2Exclusions,
+    }),
+    true,
+  );
+});
+
+test('configured POS2 rule preserves the kanji non-independent noun exception', () => {
+  const token = makeToken({ surface: '以外', headword: '以外', pos1: '名詞', pos2: '非自立' });
+  const context = createSubtitleAnnotationRuleContext(token, {
+    pos1Exclusions: new Set(),
+    pos2Exclusions: new Set(['非自立']),
+  });
+  const pos2Rule = SUBTITLE_ANNOTATION_RULES.find(({ id }) => id === 'configured-pos2-exclusion');
+
+  assert.equal(pos2Rule?.test(context), 'pass');
+  assert.equal(
+    shouldExcludeTokenFromSubtitleAnnotations(token, {
+      pos1Exclusions: context.pos1Exclusions,
+      pos2Exclusions: context.pos2Exclusions,
+    }),
+    false,
+  );
+});
+
+test('trailing quote-particle rule honors configured POS1 exclusions', () => {
+  const token = makeToken({ surface: '猫って', headword: '猫', pos1: '名詞|助詞' });
+  const context = createSubtitleAnnotationRuleContext(token, {
+    pos1Exclusions: new Set(['名詞']),
+  });
+  const trailingParticleRule = SUBTITLE_ANNOTATION_RULES.find(
+    ({ id }) => id === 'merged-trailing-quote-particle',
+  );
+
+  assert.equal(trailingParticleRule?.test(context), 'pass');
+});
+
+test('configured POS2 rule preserves supplementary-plane kanji nouns', () => {
+  const token = makeToken({ surface: '𠮟', headword: '𠮟', pos1: '名詞', pos2: '非自立' });
+
+  assert.equal(
+    shouldExcludeTokenFromSubtitleAnnotations(token, {
+      pos1Exclusions: new Set(),
+      pos2Exclusions: new Set(['非自立']),
+    }),
+    false,
+  );
+});
+
+test('kana detection excludes katakana punctuation boundaries', () => {
+  assert.equal(isKanaChar('゠'), false);
+  assert.equal(isKanaChar('・'), false);
+  assert.equal(isKanaChar('ァ'), true);
+});

src/core/services/tokenizer/subtitle-annotation-filter.ts

@@ -1,424 +1,31 @@
-import {
-  DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG,
-  resolveAnnotationPos1ExclusionSet,
-} from '../../../token-pos1-exclusions';
-import {
-  DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG,
-  resolveAnnotationPos2ExclusionSet,
-} from '../../../token-pos2-exclusions';
 import { MergedToken } from '../../../types';
-import { shouldIgnoreJlptByTerm } from '../jlpt-token-filter';
-import { isSubtitleGrammarEndingText } from './grammar-ending';
 import {
-  isContentTokenByPos,
+  createSubtitleAnnotationRuleContext,
-  isKanaChar,
+  evaluateSubtitleAnnotationRules,
-  isKanaOnlyText,
+} from './subtitle-annotation-filter-rules';
-  normalizeKana,
+import type { SubtitleAnnotationFilterOptions } from './subtitle-annotation-filter-rules';
-  splitPosTag,
-} from './token-classification';
 
-const STANDALONE_GRAMMAR_PARTICLE_PHRASES = ['たって', 'だって'] as const;
+export {
-const STANDALONE_GRAMMAR_PARTICLE_PHRASES_SET: ReadonlySet<string> = new Set(
+  createSubtitleAnnotationRuleContext,
-  STANDALONE_GRAMMAR_PARTICLE_PHRASES,
+  SUBTITLE_ANNOTATION_EXCLUDED_TERMS,
-);
+  SUBTITLE_ANNOTATION_RULES,
-
+} from './subtitle-annotation-filter-rules';
-export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
+export type {
-  'あ',
+  SubtitleAnnotationFilterOptions,
-  'ああ',
+  SubtitleAnnotationRule,
-  'ある',
+  SubtitleAnnotationRuleContext,
-  'あなた',
+  SubtitleAnnotationRuleDecision,
-  'あんた',
+} from './subtitle-annotation-filter-rules';
-  'ええ',
+export { isKanjiNonIndependentNounToken } from './token-classification';
-  'うう',
-  'おお',
-  'おい',
-  'お前',
-  'こいつ',
-  'こっち',
-  'くれ',
-  'じゃない',
-  'そうだ',
-  'たち',
-  'である',
-  'どこか',
-  'なんか',
-  'べき',
-  'って',
-  'はあ',
-  'はぁ',
-  'はは',
-  'へえ',
-  'ふう',
-  'ほう',
-  '何か',
-  '何だ',
-  '何も',
-  '如何した',
-  '有る',
-  '在る',
-  '様',
-  '誰も',
-  '貴方',
-  'もんか',
-  'ものか',
-]);
-const SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES = new Set([
-  'って',
-  'ってよ',
-  'ってね',
-  'ってな',
-  'ってさ',
-  'ってか',
-  'ってば',
-]);
-const AUXILIARY_STEM_GRAMMAR_TAIL_POS1 = new Set(['名詞', '助動詞', '助詞']);
-const NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1 = new Set(['助詞', '助動詞']);
-const AUXILIARY_INFLECTION_TRAILING_POS1 = new Set(['助動詞']);
-const AUXILIARY_HELPER_SPAN_POS1 = new Set(['助詞', '助動詞', '動詞']);
-const LEXICAL_VERB_POS2 = new Set(['自立']);
-const STANDALONE_GRAMMAR_PARTICLE_SURFACES = new Set([
-  'か',
-  'が',
-  'さ',
-  'し',
-  'ぞ',
-  'ぜ',
-  'と',
-  'な',
-  'に',
-  'ね',
-  'の',
-  'は',
-  'へ',
-  'も',
-  'や',
-  'よ',
-  'を',
-]);
-export interface SubtitleAnnotationFilterOptions {
-  pos1Exclusions?: ReadonlySet<string>;
-  pos2Exclusions?: ReadonlySet<string>;
-}
-
-function resolvePos1Exclusions(options: SubtitleAnnotationFilterOptions = {}): ReadonlySet<string> {
-  if (options.pos1Exclusions) {
-    return options.pos1Exclusions;
-  }
-
-  return resolveAnnotationPos1ExclusionSet(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG);
-}
-
-function resolvePos2Exclusions(options: SubtitleAnnotationFilterOptions = {}): ReadonlySet<string> {
-  if (options.pos2Exclusions) {
-    return options.pos2Exclusions;
-  }
-
-  return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
-}
-
-function isTrailingSmallTsuKanaSfx(text: string): boolean {
-  const normalized = normalizeKana(text);
-  if (!normalized) {
-    return false;
-  }
-
-  const chars = [...normalized];
-  if (chars.length < 2 || chars.length > 4) {
-    return false;
-  }
-
-  if (!chars.every(isKanaChar)) {
-    return false;
-  }
-
-  return chars[chars.length - 1] === 'っ';
-}
-
-function isReduplicatedKanaSfx(text: string): boolean {
-  const normalized = normalizeKana(text);
-  if (!normalized) {
-    return false;
-  }
-
-  const chars = [...normalized];
-  if (chars.length < 4 || chars.length % 2 !== 0) {
-    return false;
-  }
-
-  if (!chars.every(isKanaChar)) {
-    return false;
-  }
-
-  const half = chars.length / 2;
-  return chars.slice(0, half).join('') === chars.slice(half).join('');
-}
-
-function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
-  const normalized = normalizeKana(text);
-  if (!normalized) {
-    return false;
-  }
-
-  if (isReduplicatedKanaSfx(normalized)) {
-    return true;
-  }
-
-  if (normalized.length <= 1 || !normalized.endsWith('と')) {
-    return false;
-  }
-
-  return isReduplicatedKanaSfx(normalized.slice(0, -1));
-}
-
-function isExcludedTrailingParticleMergedToken(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const normalizedHeadword = normalizeKana(token.headword);
-  if (
-    !normalizedSurface ||
-    !normalizedHeadword ||
-    !normalizedSurface.startsWith(normalizedHeadword)
-  ) {
-    return false;
-  }
-
-  const suffix = normalizedSurface.slice(normalizedHeadword.length);
-  if (!SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES.has(suffix)) {
-    return false;
-  }
-
-  const pos1Parts = splitPosTag(token.pos1);
-  if (pos1Parts.length < 2) {
-    return false;
-  }
-
-  const [leadingPos1, ...trailingPos1] = pos1Parts;
-  if (!leadingPos1 || resolvePos1Exclusions().has(leadingPos1)) {
-    return false;
-  }
-
-  return trailingPos1.length > 0 && trailingPos1.every((part) => part === '助詞');
-}
-
-function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean {
-  const pos1Parts = splitPosTag(token.pos1);
-  if (
-    pos1Parts.length === 0 ||
-    !pos1Parts.every((part) => AUXILIARY_STEM_GRAMMAR_TAIL_POS1.has(part))
-  ) {
-    return false;
-  }
-
-  const pos3Parts = splitPosTag(token.pos3);
-  return pos3Parts.includes('助動詞語幹');
-}
-
-function isKanaOnlyNonIndependentNounHelperMerge(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const normalizedHeadword = normalizeKana(token.headword);
-  if (
-    !normalizedSurface ||
-    !normalizedHeadword ||
-    normalizedSurface === normalizedHeadword ||
-    ![...normalizedSurface].every(isKanaChar)
-  ) {
-    return false;
-  }
-
-  const pos1Parts = splitPosTag(token.pos1);
-  if (pos1Parts.length < 2 || pos1Parts[0] !== '名詞') {
-    return false;
-  }
-
-  const pos2Parts = splitPosTag(token.pos2);
-  if (pos2Parts[0] !== '非自立') {
-    return false;
-  }
-
-  return pos1Parts.slice(1).every((part) => NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1.has(part));
-}
-
-function isLexicalKureruVerb(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const normalizedHeadword = normalizeKana(token.headword);
-  const pos1Parts = splitPosTag(token.pos1);
-  const pos2Parts = splitPosTag(token.pos2);
-  return (
-    normalizedSurface === 'くれ' &&
-    normalizedHeadword === 'くれる' &&
-    pos1Parts.length === 1 &&
-    pos1Parts[0] === '動詞' &&
-    pos2Parts.length === 1 &&
-    pos2Parts[0] === '自立'
-  );
-}
-
-function isStandaloneAuxiliaryInflectionFragment(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  if (!isKanaOnlyText(normalizedSurface)) {
-    return false;
-  }
-
-  const pos1Parts = splitPosTag(token.pos1);
-  if (pos1Parts.length === 0) {
-    return false;
-  }
-
-  if (pos1Parts.every((part) => part === '助動詞')) {
-    return true;
-  }
-
-  const pos2Parts = splitPosTag(token.pos2);
-  return (
-    pos1Parts[0] === '動詞' &&
-    pos2Parts[0] === '接尾' &&
-    pos1Parts.slice(1).every((part) => AUXILIARY_INFLECTION_TRAILING_POS1.has(part))
-  );
-}
-
-function isAuxiliaryOnlyHelperSpan(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const normalizedHeadword = normalizeKana(token.headword);
-  if (!isKanaOnlyText(normalizedSurface) || !isKanaOnlyText(normalizedHeadword)) {
-    return false;
-  }
-
-  const pos1Parts = splitPosTag(token.pos1);
-  if (
-    pos1Parts.length === 0 ||
-    !pos1Parts.every((part) => AUXILIARY_HELPER_SPAN_POS1.has(part)) ||
-    !pos1Parts.includes('助詞') ||
-    !pos1Parts.includes('動詞')
-  ) {
-    return false;
-  }
-
-  const pos2Parts = splitPosTag(token.pos2);
-  return !pos2Parts.some((part) => LEXICAL_VERB_POS2.has(part));
-}
-
-function isStandaloneSuruTeGrammarHelper(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const normalizedHeadword = normalizeKana(token.headword);
-  if (!normalizedSurface.startsWith('して') || normalizedHeadword !== 'する') {
-    return false;
-  }
-
-  const pos1Parts = splitPosTag(token.pos1);
-  return (
-    isKanaOnlyText(normalizedSurface) && (pos1Parts.length === 0 || pos1Parts.includes('動詞'))
-  );
-}
-
-function isStandaloneGrammarParticle(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const normalizedHeadword = normalizeKana(token.headword);
-  return (
-    normalizedSurface === normalizedHeadword &&
-    (STANDALONE_GRAMMAR_PARTICLE_SURFACES.has(normalizedSurface) ||
-      STANDALONE_GRAMMAR_PARTICLE_PHRASES_SET.has(normalizedSurface))
-  );
-}
-
-function isSingleKanaSurfaceFragment(token: MergedToken): boolean {
-  const normalizedSurface = normalizeKana(token.surface);
-  const chars = [...normalizedSurface];
-  return chars.length === 1 && chars.every(isKanaChar);
-}
-
-function isExcludedByTerm(token: MergedToken): boolean {
-  const candidates = [token.surface, token.reading, token.headword].filter(
-    (candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0,
-  );
-
-  for (const candidate of candidates) {
-    const trimmed = candidate.trim();
-    if (!trimmed) {
-      continue;
-    }
-
-    const normalized = normalizeKana(trimmed);
-    if (!normalized) {
-      continue;
-    }
-
-    if (
-      SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(trimmed) ||
-      SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(normalized) ||
-      isSubtitleGrammarEndingText(trimmed) ||
-      isSubtitleGrammarEndingText(normalized) ||
-      shouldIgnoreJlptByTerm(trimmed) ||
-      shouldIgnoreJlptByTerm(normalized)
-    ) {
-      return true;
-    }
-
-    if (
-      isTrailingSmallTsuKanaSfx(trimmed) ||
-      isTrailingSmallTsuKanaSfx(normalized) ||
-      isReduplicatedKanaSfxWithOptionalTrailingTo(trimmed) ||
-      isReduplicatedKanaSfxWithOptionalTrailingTo(normalized)
-    ) {
-      return true;
-    }
-  }
-
-  return false;
-}
 
 export function shouldExcludeTokenFromSubtitleAnnotations(
   token: MergedToken,
   options: SubtitleAnnotationFilterOptions = {},
 ): boolean {
-  // No Yomitan dictionary entry backs this token (ぅ～ elongations, truncated
+  return (
-  // inflections) — it exists only to stay hoverable and must never receive
+    evaluateSubtitleAnnotationRules(createSubtitleAnnotationRuleContext(token, options)) ===
-  // annotations or count in the N+1 math.
+    'exclude'
-  if (token.isUnparsedRun === true) {
+  );
-    return true;
-  }
-
-  const pos1Exclusions = resolvePos1Exclusions(options);
-  const pos2Exclusions = resolvePos2Exclusions(options);
-  if (!isContentTokenByPos(token, pos1Exclusions, pos2Exclusions)) {
-    return true;
-  }
-
-  if (isAuxiliaryStemGrammarTailToken(token)) {
-    return true;
-  }
-
-  if (isKanaOnlyNonIndependentNounHelperMerge(token)) {
-    return true;
-  }
-
-  if (isStandaloneAuxiliaryInflectionFragment(token)) {
-    return true;
-  }
-
-  if (isAuxiliaryOnlyHelperSpan(token)) {
-    return true;
-  }
-
-  if (isStandaloneSuruTeGrammarHelper(token)) {
-    return true;
-  }
-
-  if (isStandaloneGrammarParticle(token)) {
-    return true;
-  }
-
-  if (isSingleKanaSurfaceFragment(token)) {
-    return true;
-  }
-
-  if (isExcludedTrailingParticleMergedToken(token)) {
-    return true;
-  }
-
-  if (isLexicalKureruVerb(token)) {
-    return false;
-  }
-
-  return isExcludedByTerm(token);
 }
 
 export function stripSubtitleAnnotationMetadata(

src/core/services/tokenizer/token-classification.ts

@@ -105,7 +105,9 @@ function hasKanjiChar(text: string): boolean {
       code !== undefined &&
       ((code >= 0x3400 && code <= 0x4dbf) ||
         (code >= 0x4e00 && code <= 0x9fff) ||
-        (code >= 0xf900 && code <= 0xfaff))
+        (code >= 0xf900 && code <= 0xfaff) ||
+        (code >= 0x20000 && code <= 0x2fa1f) ||
+        (code >= 0x30000 && code <= 0x323af))
     ) {
       return true;
     }
@@ -115,7 +117,7 @@ function hasKanjiChar(text: string): boolean {
 
 // MeCab's 非自立 tag suppresses kana grammar nouns (こと, もの, とき), but
 // Yomitan can segment kanji-bearing nouns (日, 方, 上, …) as real vocabulary.
-function isKanjiNonIndependentNounToken(
+export function isKanjiNonIndependentNounToken(
   token: MergedToken,
   pos1Exclusions: ReadonlySet<string>,
 ): boolean {