From 0ab840b3628889dc084e95c6e8c946ec0b7749fc Mon Sep 17 00:00:00 2001 From: sudacode Date: Sun, 12 Jul 2026 23:54:13 -0700 Subject: [PATCH] refactor(tokenizer): extract subtitle annotation filter into rule table (#162) --- changes/annotation-noise-rule-table.md | 4 + .../subtitle-annotation-filter-rules.ts | 342 ++++++++++++++ .../subtitle-annotation-filter-support.ts | 153 +++++++ .../subtitle-annotation-filter.test.ts | 183 ++++++++ .../tokenizer/subtitle-annotation-filter.ts | 433 +----------------- .../tokenizer/token-classification.ts | 6 +- 6 files changed, 706 insertions(+), 415 deletions(-) create mode 100644 changes/annotation-noise-rule-table.md create mode 100644 src/core/services/tokenizer/subtitle-annotation-filter-rules.ts create mode 100644 src/core/services/tokenizer/subtitle-annotation-filter-support.ts create mode 100644 src/core/services/tokenizer/subtitle-annotation-filter.test.ts diff --git a/changes/annotation-noise-rule-table.md b/changes/annotation-noise-rule-table.md new file mode 100644 index 00000000..2c35f034 --- /dev/null +++ b/changes/annotation-noise-rule-table.md @@ -0,0 +1,4 @@ +type: fixed +area: overlay + +- Applied configured primary POS exclusions consistently to merged trailing quote-particle tokens, preserved annotations for supplementary-plane kanji, and stopped treating katakana punctuation as kana-only annotation noise. diff --git a/src/core/services/tokenizer/subtitle-annotation-filter-rules.ts b/src/core/services/tokenizer/subtitle-annotation-filter-rules.ts new file mode 100644 index 00000000..a3a36b3f --- /dev/null +++ b/src/core/services/tokenizer/subtitle-annotation-filter-rules.ts @@ -0,0 +1,342 @@ +import { + DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG, + resolveAnnotationPos1ExclusionSet, +} from '../../../token-pos1-exclusions'; +import { + DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG, + resolveAnnotationPos2ExclusionSet, +} from '../../../token-pos2-exclusions'; +import { MergedToken } from '../../../types'; +import { shouldIgnoreJlptByTerm } from '../jlpt-token-filter'; +import { isSubtitleGrammarEndingText } from './grammar-ending'; +import { + isAuxiliaryOnlyHelperSpan, + isAuxiliaryStemGrammarTailToken, + isExcludedTrailingParticleMergedToken, + isKanaOnlyNonIndependentNounHelperMerge, + isReduplicatedKanaSfxWithOptionalTrailingTo, + isSingleKanaSurfaceFragment, + isStandaloneAuxiliaryInflectionFragment, + isStandaloneGrammarParticle, + isStandaloneSuruTeGrammarHelper, + isTrailingSmallTsuKanaSfx, +} from './subtitle-annotation-filter-support'; +import { + isContentTokenByPos, + isPosTagExcluded, + isTokenPos2Excluded, + normalizeKana, + normalizePosTag, + splitPosTag, +} from './token-classification'; + +export interface SubtitleAnnotationFilterOptions { + pos1Exclusions?: ReadonlySet; + pos2Exclusions?: ReadonlySet; +} + +export type SubtitleAnnotationRuleDecision = 'exclude' | 'keep' | 'pass'; + +export interface SubtitleAnnotationRuleContext { + token: MergedToken; + pos1Exclusions: ReadonlySet; + pos2Exclusions: ReadonlySet; + normalizedPos1: string; + normalizedPos2: string; + hasPos1: boolean; + hasPos2: boolean; +} + +type SubtitleAnnotationRuleData = Readonly>; + +export interface SubtitleAnnotationRule { + readonly id: string; + readonly description: string; + readonly issueRef: string; + readonly data: SubtitleAnnotationRuleData; + test(context: SubtitleAnnotationRuleContext): SubtitleAnnotationRuleDecision; +} + +function defineRule(definition: { + id: string; + description: string; + issueRef: string; + data: TData; + test: (context: SubtitleAnnotationRuleContext, data: TData) => SubtitleAnnotationRuleDecision; +}): SubtitleAnnotationRule { + const data = Object.freeze( + Object.fromEntries( + Object.entries(definition.data).map(([key, values]) => [key, Object.freeze([...values])]), + ), + ) as TData; + return Object.freeze({ + id: definition.id, + description: definition.description, + issueRef: definition.issueRef, + data, + test: (context: SubtitleAnnotationRuleContext) => definition.test(context, data), + }); +} + +export function createSubtitleAnnotationRuleContext( + token: MergedToken, + options: SubtitleAnnotationFilterOptions = {}, +): SubtitleAnnotationRuleContext { + const pos1Exclusions = + options.pos1Exclusions ?? + resolveAnnotationPos1ExclusionSet(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG); + const pos2Exclusions = + options.pos2Exclusions ?? + resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG); + const normalizedPos1 = normalizePosTag(token.pos1); + const normalizedPos2 = normalizePosTag(token.pos2); + return { + token, + pos1Exclusions, + pos2Exclusions, + normalizedPos1, + normalizedPos2, + hasPos1: normalizedPos1.length > 0, + hasPos2: normalizedPos2.length > 0, + }; +} + +function matchesExcludedTermOrPattern(token: MergedToken, terms: ReadonlySet): boolean { + const candidates = [token.surface, token.reading, token.headword].filter( + (candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0, + ); + + for (const candidate of candidates) { + const trimmed = candidate.trim(); + if (!trimmed) { + continue; + } + const normalized = normalizeKana(trimmed); + if (!normalized) { + continue; + } + if ( + terms.has(trimmed) || + terms.has(normalized) || + isSubtitleGrammarEndingText(trimmed) || + isSubtitleGrammarEndingText(normalized) || + shouldIgnoreJlptByTerm(trimmed) || + shouldIgnoreJlptByTerm(normalized) || + isTrailingSmallTsuKanaSfx(trimmed) || + isTrailingSmallTsuKanaSfx(normalized) || + isReduplicatedKanaSfxWithOptionalTrailingTo(trimmed) || + isReduplicatedKanaSfxWithOptionalTrailingTo(normalized) + ) { + return true; + } + } + return false; +} + +export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([ + 'あ', + 'ああ', + 'ある', + 'あなた', + 'あんた', + 'ええ', + 'うう', + 'おお', + 'おい', + 'お前', + 'こいつ', + 'こっち', + 'くれ', + 'じゃない', + 'そうだ', + 'たち', + 'である', + 'どこか', + 'なんか', + 'べき', + 'って', + 'はあ', + 'はぁ', + 'はは', + 'へえ', + 'ふう', + 'ほう', + '何か', + '何だ', + '何も', + '如何した', + '有る', + '在る', + '様', + '誰も', + '貴方', + 'もんか', + 'ものか', +]); + +const excludedTermRule = defineRule({ + id: 'excluded-term-or-pattern', + description: + 'Exclude legacy subtitle stop terms, grammar endings, JLPT stop terms, and kana sound effects.', + issueRef: '#19, #33, #57', + data: {}, + test: ({ token }) => + matchesExcludedTermOrPattern(token, SUBTITLE_ANNOTATION_EXCLUDED_TERMS) ? 'exclude' : 'pass', +}); + +// Ordered, first-match-wins. issueRef records the introducing/fixing PR; early +// rules are legacy behavior inherited from the original #19 filter. +export const SUBTITLE_ANNOTATION_RULES: readonly SubtitleAnnotationRule[] = Object.freeze([ + defineRule({ + id: 'unparsed-run', + description: 'Exclude hoverable parser gaps that have no Yomitan dictionary entry.', + issueRef: '#153', + data: {}, + test: ({ token }) => (token.isUnparsedRun === true ? 'exclude' : 'pass'), + }), + defineRule({ + id: 'configured-pos1-exclusion', + description: 'Apply the configured primary part-of-speech exclusions.', + issueRef: '#19', + data: {}, + test: ({ token, pos1Exclusions }) => + isPosTagExcluded(token.pos1, pos1Exclusions) ? 'exclude' : 'pass', + }), + defineRule({ + id: 'configured-pos2-exclusion', + description: 'Apply configured secondary POS exclusions, preserving #150 kanji nouns.', + issueRef: '#150', + data: {}, + test: ({ token, pos1Exclusions, pos2Exclusions }) => + isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions) ? 'exclude' : 'pass', + }), + defineRule({ + id: 'coarse-grammar-pos-fallback', + description: 'Exclude coarse grammar POS when detailed MeCab tags are unavailable.', + issueRef: '#19', + data: {}, + test: ({ token, hasPos1, hasPos2, pos1Exclusions, pos2Exclusions }) => + !hasPos1 && !hasPos2 && !isContentTokenByPos(token, pos1Exclusions, pos2Exclusions) + ? 'exclude' + : 'pass', + }), + defineRule({ + id: 'auxiliary-stem-grammar-tail', + description: 'Exclude merged grammar tails containing an auxiliary stem.', + issueRef: '#19', + data: { allowedPos1: ['名詞', '助動詞', '助詞'] }, + test: ({ token }, { allowedPos1 }) => + isAuxiliaryStemGrammarTailToken(token, allowedPos1) ? 'exclude' : 'pass', + }), + defineRule({ + id: 'kana-non-independent-noun-helper', + description: 'Exclude kana non-independent nouns merged with grammar helpers.', + issueRef: '#56', + data: { tailPos1: ['助詞', '助動詞'] }, + test: ({ token }, { tailPos1 }) => + isKanaOnlyNonIndependentNounHelperMerge(token, tailPos1) ? 'exclude' : 'pass', + }), + defineRule({ + id: 'standalone-auxiliary-inflection', + description: 'Exclude standalone kana auxiliary inflection fragments.', + issueRef: '#57', + data: { trailingPos1: ['助動詞'] }, + test: ({ token }, { trailingPos1 }) => + isStandaloneAuxiliaryInflectionFragment(token, trailingPos1) ? 'exclude' : 'pass', + }), + defineRule({ + id: 'auxiliary-only-helper-span', + description: 'Exclude kana helper spans without an independent lexical verb.', + issueRef: '#57', + data: { allowedPos1: ['助詞', '助動詞', '動詞'], lexicalVerbPos2: ['自立'] }, + test: ({ token }, { allowedPos1, lexicalVerbPos2 }) => + isAuxiliaryOnlyHelperSpan(token, allowedPos1, lexicalVerbPos2) ? 'exclude' : 'pass', + }), + defineRule({ + id: 'standalone-suru-te-helper', + description: 'Exclude standalone して grammar-helper fragments.', + issueRef: '#57', + data: {}, + test: ({ token }) => (isStandaloneSuruTeGrammarHelper(token) ? 'exclude' : 'pass'), + }), + defineRule({ + id: 'standalone-grammar-particle', + description: 'Exclude standalone particle surfaces and connective particle phrases.', + issueRef: '#57', + data: { + surfaces: [ + 'か', + 'が', + 'さ', + 'し', + 'ぞ', + 'ぜ', + 'と', + 'な', + 'に', + 'ね', + 'の', + 'は', + 'へ', + 'も', + 'や', + 'よ', + 'を', + ], + phrases: ['たって', 'だって'], + }, + test: ({ token }, { surfaces, phrases }) => + isStandaloneGrammarParticle(token, surfaces, phrases) ? 'exclude' : 'pass', + }), + defineRule({ + id: 'single-kana-fragment', + description: 'Exclude isolated one-kana parser fragments.', + issueRef: '#57', + data: {}, + test: ({ token }) => (isSingleKanaSurfaceFragment(token) ? 'exclude' : 'pass'), + }), + defineRule({ + id: 'merged-trailing-quote-particle', + description: 'Exclude lexical tokens merged only with a trailing quote-particle suffix.', + issueRef: '#19', + data: { suffixes: ['って', 'ってよ', 'ってね', 'ってな', 'ってさ', 'ってか', 'ってば'] }, + test: ({ token, pos1Exclusions }, { suffixes }) => + isExcludedTrailingParticleMergedToken(token, suffixes, pos1Exclusions) ? 'exclude' : 'pass', + }), + defineRule({ + id: 'lexical-kureru-keep', + description: 'Keep lexical くれる before the legacy bare-くれ stop-term rule.', + issueRef: '#57', + data: { + surfaces: ['くれ'], + headwords: ['くれる'], + pos1: ['動詞'], + pos2: ['自立'], + }, + test: ({ token }, data) => { + const pos1 = splitPosTag(token.pos1); + const pos2 = splitPosTag(token.pos2); + return data.surfaces.includes(normalizeKana(token.surface)) && + data.headwords.includes(normalizeKana(token.headword)) && + pos1.length === 1 && + data.pos1.includes(pos1[0] ?? '') && + pos2.length === 1 && + data.pos2.includes(pos2[0] ?? '') + ? 'keep' + : 'pass'; + }, + }), + excludedTermRule, +]); + +export function evaluateSubtitleAnnotationRules( + context: SubtitleAnnotationRuleContext, +): SubtitleAnnotationRuleDecision { + for (const rule of SUBTITLE_ANNOTATION_RULES) { + const decision = rule.test(context); + if (decision !== 'pass') { + return decision; + } + } + return 'pass'; +} diff --git a/src/core/services/tokenizer/subtitle-annotation-filter-support.ts b/src/core/services/tokenizer/subtitle-annotation-filter-support.ts new file mode 100644 index 00000000..4fc7f131 --- /dev/null +++ b/src/core/services/tokenizer/subtitle-annotation-filter-support.ts @@ -0,0 +1,153 @@ +import { MergedToken } from '../../../types'; +import { isKanaChar, isKanaOnlyText, normalizeKana, splitPosTag } from './token-classification'; + +export function isTrailingSmallTsuKanaSfx(text: string): boolean { + const chars = [...normalizeKana(text)]; + return ( + chars.length >= 2 && + chars.length <= 4 && + chars.every(isKanaChar) && + chars[chars.length - 1] === 'っ' + ); +} + +function isReduplicatedKanaSfx(text: string): boolean { + const chars = [...normalizeKana(text)]; + if (chars.length < 4 || chars.length % 2 !== 0 || !chars.every(isKanaChar)) { + return false; + } + const half = chars.length / 2; + return chars.slice(0, half).join('') === chars.slice(half).join(''); +} + +export function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean { + const normalized = normalizeKana(text); + if (!normalized) { + return false; + } + if (isReduplicatedKanaSfx(normalized)) { + return true; + } + return normalized.length > 1 && normalized.endsWith('と') + ? isReduplicatedKanaSfx(normalized.slice(0, -1)) + : false; +} + +export function isExcludedTrailingParticleMergedToken( + token: MergedToken, + suffixes: readonly string[], + leadingPos1Exclusions: ReadonlySet, +): boolean { + const surface = normalizeKana(token.surface); + const headword = normalizeKana(token.headword); + if (!surface || !headword || !surface.startsWith(headword)) { + return false; + } + if (!suffixes.includes(surface.slice(headword.length))) { + return false; + } + + const [leadingPos1, ...trailingPos1] = splitPosTag(token.pos1); + if (!leadingPos1 || leadingPos1Exclusions.has(leadingPos1)) { + return false; + } + return trailingPos1.length > 0 && trailingPos1.every((part) => part === '助詞'); +} + +export function isAuxiliaryStemGrammarTailToken( + token: MergedToken, + allowedPos1: readonly string[], +): boolean { + const pos1Parts = splitPosTag(token.pos1); + if (pos1Parts.length === 0 || !pos1Parts.every((part) => allowedPos1.includes(part))) { + return false; + } + return splitPosTag(token.pos3).includes('助動詞語幹'); +} + +export function isKanaOnlyNonIndependentNounHelperMerge( + token: MergedToken, + tailPos1: readonly string[], +): boolean { + const surface = normalizeKana(token.surface); + const headword = normalizeKana(token.headword); + if (!surface || !headword || surface === headword || ![...surface].every(isKanaChar)) { + return false; + } + + const pos1Parts = splitPosTag(token.pos1); + if (pos1Parts.length < 2 || pos1Parts[0] !== '名詞') { + return false; + } + const pos2Parts = splitPosTag(token.pos2); + return pos2Parts[0] === '非自立' && pos1Parts.slice(1).every((part) => tailPos1.includes(part)); +} + +export function isStandaloneAuxiliaryInflectionFragment( + token: MergedToken, + trailingPos1: readonly string[], +): boolean { + if (!isKanaOnlyText(token.surface)) { + return false; + } + const pos1Parts = splitPosTag(token.pos1); + if (pos1Parts.length === 0) { + return false; + } + if (pos1Parts.every((part) => part === '助動詞')) { + return true; + } + const pos2Parts = splitPosTag(token.pos2); + return ( + pos1Parts[0] === '動詞' && + pos2Parts[0] === '接尾' && + pos1Parts.slice(1).every((part) => trailingPos1.includes(part)) + ); +} + +export function isAuxiliaryOnlyHelperSpan( + token: MergedToken, + allowedPos1: readonly string[], + lexicalVerbPos2: readonly string[], +): boolean { + if (!isKanaOnlyText(token.surface) || !isKanaOnlyText(token.headword)) { + return false; + } + const pos1Parts = splitPosTag(token.pos1); + if ( + pos1Parts.length === 0 || + !pos1Parts.every((part) => allowedPos1.includes(part)) || + !pos1Parts.includes('助詞') || + !pos1Parts.includes('動詞') + ) { + return false; + } + return !splitPosTag(token.pos2).some((part) => lexicalVerbPos2.includes(part)); +} + +export function isStandaloneSuruTeGrammarHelper(token: MergedToken): boolean { + const surface = normalizeKana(token.surface); + const headword = normalizeKana(token.headword); + if (!surface.startsWith('して') || headword !== 'する') { + return false; + } + const pos1Parts = splitPosTag(token.pos1); + return isKanaOnlyText(surface) && (pos1Parts.length === 0 || pos1Parts.includes('動詞')); +} + +export function isStandaloneGrammarParticle( + token: MergedToken, + surfaces: readonly string[], + phrases: readonly string[], +): boolean { + const surface = normalizeKana(token.surface); + return ( + surface === normalizeKana(token.headword) && + (surfaces.includes(surface) || phrases.includes(surface)) + ); +} + +export function isSingleKanaSurfaceFragment(token: MergedToken): boolean { + const chars = [...normalizeKana(token.surface)]; + return chars.length === 1 && chars.every(isKanaChar); +} diff --git a/src/core/services/tokenizer/subtitle-annotation-filter.test.ts b/src/core/services/tokenizer/subtitle-annotation-filter.test.ts new file mode 100644 index 00000000..898c36ae --- /dev/null +++ b/src/core/services/tokenizer/subtitle-annotation-filter.test.ts @@ -0,0 +1,183 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { MergedToken, PartOfSpeech } from '../../../types'; +import { + createSubtitleAnnotationRuleContext, + shouldExcludeTokenFromSubtitleAnnotations, + SUBTITLE_ANNOTATION_EXCLUDED_TERMS, + SUBTITLE_ANNOTATION_RULES, +} from './subtitle-annotation-filter'; +import { isKanaChar } from './token-classification'; + +function makeToken(overrides: Partial = {}): MergedToken { + return { + surface: '猫', + reading: 'ネコ', + headword: '猫', + startPos: 0, + endPos: 1, + partOfSpeech: PartOfSpeech.noun, + isMerged: false, + isKnown: false, + isNPlusOneTarget: false, + ...overrides, + }; +} + +test('subtitle annotation rules expose stable ordered provenance', () => { + assert.deepEqual( + SUBTITLE_ANNOTATION_RULES.map(({ id, issueRef }) => ({ id, issueRef })), + [ + { id: 'unparsed-run', issueRef: '#153' }, + { id: 'configured-pos1-exclusion', issueRef: '#19' }, + { id: 'configured-pos2-exclusion', issueRef: '#150' }, + { id: 'coarse-grammar-pos-fallback', issueRef: '#19' }, + { id: 'auxiliary-stem-grammar-tail', issueRef: '#19' }, + { id: 'kana-non-independent-noun-helper', issueRef: '#56' }, + { id: 'standalone-auxiliary-inflection', issueRef: '#57' }, + { id: 'auxiliary-only-helper-span', issueRef: '#57' }, + { id: 'standalone-suru-te-helper', issueRef: '#57' }, + { id: 'standalone-grammar-particle', issueRef: '#57' }, + { id: 'single-kana-fragment', issueRef: '#57' }, + { id: 'merged-trailing-quote-particle', issueRef: '#19' }, + { id: 'lexical-kureru-keep', issueRef: '#57' }, + { id: 'excluded-term-or-pattern', issueRef: '#19, #33, #57' }, + ], + ); + assert.equal(new Set(SUBTITLE_ANNOTATION_RULES.map(({ id }) => id)).size, 14); + assert.ok(SUBTITLE_ANNOTATION_RULES.every(({ description }) => description.length > 0)); + assert.ok( + SUBTITLE_ANNOTATION_RULES.every( + ({ data }) => Object.isFrozen(data) && Object.values(data).every(Object.isFrozen), + ), + ); +}); + +test('lexical kureru keep rule precedes and overrides the excluded-term rule', () => { + const token = makeToken({ + surface: 'くれ', + headword: 'くれる', + reading: 'クレ', + partOfSpeech: PartOfSpeech.verb, + pos1: '動詞', + pos2: '自立', + }); + const context = createSubtitleAnnotationRuleContext(token); + const keepIndex = SUBTITLE_ANNOTATION_RULES.findIndex(({ id }) => id === 'lexical-kureru-keep'); + const termIndex = SUBTITLE_ANNOTATION_RULES.findIndex( + ({ id }) => id === 'excluded-term-or-pattern', + ); + + assert.ok(keepIndex >= 0 && keepIndex < termIndex); + assert.ok(SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has('くれ')); + assert.equal(SUBTITLE_ANNOTATION_RULES[keepIndex]?.test(context), 'keep'); + assert.equal(SUBTITLE_ANNOTATION_RULES[termIndex]?.test(context), 'exclude'); + assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false); +}); + +test('adding an exported excluded term changes annotation matching', () => { + const term = '超語彙'; + const token = makeToken({ + surface: term, + headword: term, + reading: term, + pos1: '名詞', + pos2: '一般', + }); + + SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term); + try { + assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false); + SUBTITLE_ANNOTATION_EXCLUDED_TERMS.add(term); + assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true); + } finally { + SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term); + } +}); + +test('deleting an exported excluded term changes annotation matching', () => { + const term = 'あなた'; + const token = makeToken({ + surface: term, + headword: term, + reading: term, + pos1: '名詞', + pos2: '一般', + }); + + assert.ok(SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(term)); + try { + SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term); + assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false); + } finally { + SUBTITLE_ANNOTATION_EXCLUDED_TERMS.add(term); + } +}); + +test('configured POS rules consume the context exclusion sets in table order', () => { + const token = makeToken({ pos1: '名詞', pos2: '一般' }); + const context = createSubtitleAnnotationRuleContext(token, { + pos1Exclusions: new Set(['名詞']), + pos2Exclusions: new Set(['一般']), + }); + const pos1Rule = SUBTITLE_ANNOTATION_RULES.find(({ id }) => id === 'configured-pos1-exclusion'); + const pos2Rule = SUBTITLE_ANNOTATION_RULES.find(({ id }) => id === 'configured-pos2-exclusion'); + + assert.equal(pos1Rule?.test(context), 'exclude'); + assert.equal(pos2Rule?.test(context), 'exclude'); + assert.equal( + shouldExcludeTokenFromSubtitleAnnotations(token, { + pos1Exclusions: context.pos1Exclusions, + pos2Exclusions: context.pos2Exclusions, + }), + true, + ); +}); + +test('configured POS2 rule preserves the kanji non-independent noun exception', () => { + const token = makeToken({ surface: '以外', headword: '以外', pos1: '名詞', pos2: '非自立' }); + const context = createSubtitleAnnotationRuleContext(token, { + pos1Exclusions: new Set(), + pos2Exclusions: new Set(['非自立']), + }); + const pos2Rule = SUBTITLE_ANNOTATION_RULES.find(({ id }) => id === 'configured-pos2-exclusion'); + + assert.equal(pos2Rule?.test(context), 'pass'); + assert.equal( + shouldExcludeTokenFromSubtitleAnnotations(token, { + pos1Exclusions: context.pos1Exclusions, + pos2Exclusions: context.pos2Exclusions, + }), + false, + ); +}); + +test('trailing quote-particle rule honors configured POS1 exclusions', () => { + const token = makeToken({ surface: '猫って', headword: '猫', pos1: '名詞|助詞' }); + const context = createSubtitleAnnotationRuleContext(token, { + pos1Exclusions: new Set(['名詞']), + }); + const trailingParticleRule = SUBTITLE_ANNOTATION_RULES.find( + ({ id }) => id === 'merged-trailing-quote-particle', + ); + + assert.equal(trailingParticleRule?.test(context), 'pass'); +}); + +test('configured POS2 rule preserves supplementary-plane kanji nouns', () => { + const token = makeToken({ surface: '𠮟', headword: '𠮟', pos1: '名詞', pos2: '非自立' }); + + assert.equal( + shouldExcludeTokenFromSubtitleAnnotations(token, { + pos1Exclusions: new Set(), + pos2Exclusions: new Set(['非自立']), + }), + false, + ); +}); + +test('kana detection excludes katakana punctuation boundaries', () => { + assert.equal(isKanaChar('゠'), false); + assert.equal(isKanaChar('・'), false); + assert.equal(isKanaChar('ァ'), true); +}); diff --git a/src/core/services/tokenizer/subtitle-annotation-filter.ts b/src/core/services/tokenizer/subtitle-annotation-filter.ts index ab06afc2..cb119473 100644 --- a/src/core/services/tokenizer/subtitle-annotation-filter.ts +++ b/src/core/services/tokenizer/subtitle-annotation-filter.ts @@ -1,424 +1,31 @@ -import { - DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG, - resolveAnnotationPos1ExclusionSet, -} from '../../../token-pos1-exclusions'; -import { - DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG, - resolveAnnotationPos2ExclusionSet, -} from '../../../token-pos2-exclusions'; import { MergedToken } from '../../../types'; -import { shouldIgnoreJlptByTerm } from '../jlpt-token-filter'; -import { isSubtitleGrammarEndingText } from './grammar-ending'; import { - isContentTokenByPos, - isKanaChar, - isKanaOnlyText, - normalizeKana, - splitPosTag, -} from './token-classification'; + createSubtitleAnnotationRuleContext, + evaluateSubtitleAnnotationRules, +} from './subtitle-annotation-filter-rules'; +import type { SubtitleAnnotationFilterOptions } from './subtitle-annotation-filter-rules'; -const STANDALONE_GRAMMAR_PARTICLE_PHRASES = ['たって', 'だって'] as const; -const STANDALONE_GRAMMAR_PARTICLE_PHRASES_SET: ReadonlySet = new Set( - STANDALONE_GRAMMAR_PARTICLE_PHRASES, -); - -export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([ - 'あ', - 'ああ', - 'ある', - 'あなた', - 'あんた', - 'ええ', - 'うう', - 'おお', - 'おい', - 'お前', - 'こいつ', - 'こっち', - 'くれ', - 'じゃない', - 'そうだ', - 'たち', - 'である', - 'どこか', - 'なんか', - 'べき', - 'って', - 'はあ', - 'はぁ', - 'はは', - 'へえ', - 'ふう', - 'ほう', - '何か', - '何だ', - '何も', - '如何した', - '有る', - '在る', - '様', - '誰も', - '貴方', - 'もんか', - 'ものか', -]); -const SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES = new Set([ - 'って', - 'ってよ', - 'ってね', - 'ってな', - 'ってさ', - 'ってか', - 'ってば', -]); -const AUXILIARY_STEM_GRAMMAR_TAIL_POS1 = new Set(['名詞', '助動詞', '助詞']); -const NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1 = new Set(['助詞', '助動詞']); -const AUXILIARY_INFLECTION_TRAILING_POS1 = new Set(['助動詞']); -const AUXILIARY_HELPER_SPAN_POS1 = new Set(['助詞', '助動詞', '動詞']); -const LEXICAL_VERB_POS2 = new Set(['自立']); -const STANDALONE_GRAMMAR_PARTICLE_SURFACES = new Set([ - 'か', - 'が', - 'さ', - 'し', - 'ぞ', - 'ぜ', - 'と', - 'な', - 'に', - 'ね', - 'の', - 'は', - 'へ', - 'も', - 'や', - 'よ', - 'を', -]); -export interface SubtitleAnnotationFilterOptions { - pos1Exclusions?: ReadonlySet; - pos2Exclusions?: ReadonlySet; -} - -function resolvePos1Exclusions(options: SubtitleAnnotationFilterOptions = {}): ReadonlySet { - if (options.pos1Exclusions) { - return options.pos1Exclusions; - } - - return resolveAnnotationPos1ExclusionSet(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG); -} - -function resolvePos2Exclusions(options: SubtitleAnnotationFilterOptions = {}): ReadonlySet { - if (options.pos2Exclusions) { - return options.pos2Exclusions; - } - - return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG); -} - -function isTrailingSmallTsuKanaSfx(text: string): boolean { - const normalized = normalizeKana(text); - if (!normalized) { - return false; - } - - const chars = [...normalized]; - if (chars.length < 2 || chars.length > 4) { - return false; - } - - if (!chars.every(isKanaChar)) { - return false; - } - - return chars[chars.length - 1] === 'っ'; -} - -function isReduplicatedKanaSfx(text: string): boolean { - const normalized = normalizeKana(text); - if (!normalized) { - return false; - } - - const chars = [...normalized]; - if (chars.length < 4 || chars.length % 2 !== 0) { - return false; - } - - if (!chars.every(isKanaChar)) { - return false; - } - - const half = chars.length / 2; - return chars.slice(0, half).join('') === chars.slice(half).join(''); -} - -function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean { - const normalized = normalizeKana(text); - if (!normalized) { - return false; - } - - if (isReduplicatedKanaSfx(normalized)) { - return true; - } - - if (normalized.length <= 1 || !normalized.endsWith('と')) { - return false; - } - - return isReduplicatedKanaSfx(normalized.slice(0, -1)); -} - -function isExcludedTrailingParticleMergedToken(token: MergedToken): boolean { - const normalizedSurface = normalizeKana(token.surface); - const normalizedHeadword = normalizeKana(token.headword); - if ( - !normalizedSurface || - !normalizedHeadword || - !normalizedSurface.startsWith(normalizedHeadword) - ) { - return false; - } - - const suffix = normalizedSurface.slice(normalizedHeadword.length); - if (!SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES.has(suffix)) { - return false; - } - - const pos1Parts = splitPosTag(token.pos1); - if (pos1Parts.length < 2) { - return false; - } - - const [leadingPos1, ...trailingPos1] = pos1Parts; - if (!leadingPos1 || resolvePos1Exclusions().has(leadingPos1)) { - return false; - } - - return trailingPos1.length > 0 && trailingPos1.every((part) => part === '助詞'); -} - -function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean { - const pos1Parts = splitPosTag(token.pos1); - if ( - pos1Parts.length === 0 || - !pos1Parts.every((part) => AUXILIARY_STEM_GRAMMAR_TAIL_POS1.has(part)) - ) { - return false; - } - - const pos3Parts = splitPosTag(token.pos3); - return pos3Parts.includes('助動詞語幹'); -} - -function isKanaOnlyNonIndependentNounHelperMerge(token: MergedToken): boolean { - const normalizedSurface = normalizeKana(token.surface); - const normalizedHeadword = normalizeKana(token.headword); - if ( - !normalizedSurface || - !normalizedHeadword || - normalizedSurface === normalizedHeadword || - ![...normalizedSurface].every(isKanaChar) - ) { - return false; - } - - const pos1Parts = splitPosTag(token.pos1); - if (pos1Parts.length < 2 || pos1Parts[0] !== '名詞') { - return false; - } - - const pos2Parts = splitPosTag(token.pos2); - if (pos2Parts[0] !== '非自立') { - return false; - } - - return pos1Parts.slice(1).every((part) => NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1.has(part)); -} - -function isLexicalKureruVerb(token: MergedToken): boolean { - const normalizedSurface = normalizeKana(token.surface); - const normalizedHeadword = normalizeKana(token.headword); - const pos1Parts = splitPosTag(token.pos1); - const pos2Parts = splitPosTag(token.pos2); - return ( - normalizedSurface === 'くれ' && - normalizedHeadword === 'くれる' && - pos1Parts.length === 1 && - pos1Parts[0] === '動詞' && - pos2Parts.length === 1 && - pos2Parts[0] === '自立' - ); -} - -function isStandaloneAuxiliaryInflectionFragment(token: MergedToken): boolean { - const normalizedSurface = normalizeKana(token.surface); - if (!isKanaOnlyText(normalizedSurface)) { - return false; - } - - const pos1Parts = splitPosTag(token.pos1); - if (pos1Parts.length === 0) { - return false; - } - - if (pos1Parts.every((part) => part === '助動詞')) { - return true; - } - - const pos2Parts = splitPosTag(token.pos2); - return ( - pos1Parts[0] === '動詞' && - pos2Parts[0] === '接尾' && - pos1Parts.slice(1).every((part) => AUXILIARY_INFLECTION_TRAILING_POS1.has(part)) - ); -} - -function isAuxiliaryOnlyHelperSpan(token: MergedToken): boolean { - const normalizedSurface = normalizeKana(token.surface); - const normalizedHeadword = normalizeKana(token.headword); - if (!isKanaOnlyText(normalizedSurface) || !isKanaOnlyText(normalizedHeadword)) { - return false; - } - - const pos1Parts = splitPosTag(token.pos1); - if ( - pos1Parts.length === 0 || - !pos1Parts.every((part) => AUXILIARY_HELPER_SPAN_POS1.has(part)) || - !pos1Parts.includes('助詞') || - !pos1Parts.includes('動詞') - ) { - return false; - } - - const pos2Parts = splitPosTag(token.pos2); - return !pos2Parts.some((part) => LEXICAL_VERB_POS2.has(part)); -} - -function isStandaloneSuruTeGrammarHelper(token: MergedToken): boolean { - const normalizedSurface = normalizeKana(token.surface); - const normalizedHeadword = normalizeKana(token.headword); - if (!normalizedSurface.startsWith('して') || normalizedHeadword !== 'する') { - return false; - } - - const pos1Parts = splitPosTag(token.pos1); - return ( - isKanaOnlyText(normalizedSurface) && (pos1Parts.length === 0 || pos1Parts.includes('動詞')) - ); -} - -function isStandaloneGrammarParticle(token: MergedToken): boolean { - const normalizedSurface = normalizeKana(token.surface); - const normalizedHeadword = normalizeKana(token.headword); - return ( - normalizedSurface === normalizedHeadword && - (STANDALONE_GRAMMAR_PARTICLE_SURFACES.has(normalizedSurface) || - STANDALONE_GRAMMAR_PARTICLE_PHRASES_SET.has(normalizedSurface)) - ); -} - -function isSingleKanaSurfaceFragment(token: MergedToken): boolean { - const normalizedSurface = normalizeKana(token.surface); - const chars = [...normalizedSurface]; - return chars.length === 1 && chars.every(isKanaChar); -} - -function isExcludedByTerm(token: MergedToken): boolean { - const candidates = [token.surface, token.reading, token.headword].filter( - (candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0, - ); - - for (const candidate of candidates) { - const trimmed = candidate.trim(); - if (!trimmed) { - continue; - } - - const normalized = normalizeKana(trimmed); - if (!normalized) { - continue; - } - - if ( - SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(trimmed) || - SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(normalized) || - isSubtitleGrammarEndingText(trimmed) || - isSubtitleGrammarEndingText(normalized) || - shouldIgnoreJlptByTerm(trimmed) || - shouldIgnoreJlptByTerm(normalized) - ) { - return true; - } - - if ( - isTrailingSmallTsuKanaSfx(trimmed) || - isTrailingSmallTsuKanaSfx(normalized) || - isReduplicatedKanaSfxWithOptionalTrailingTo(trimmed) || - isReduplicatedKanaSfxWithOptionalTrailingTo(normalized) - ) { - return true; - } - } - - return false; -} +export { + createSubtitleAnnotationRuleContext, + SUBTITLE_ANNOTATION_EXCLUDED_TERMS, + SUBTITLE_ANNOTATION_RULES, +} from './subtitle-annotation-filter-rules'; +export type { + SubtitleAnnotationFilterOptions, + SubtitleAnnotationRule, + SubtitleAnnotationRuleContext, + SubtitleAnnotationRuleDecision, +} from './subtitle-annotation-filter-rules'; +export { isKanjiNonIndependentNounToken } from './token-classification'; export function shouldExcludeTokenFromSubtitleAnnotations( token: MergedToken, options: SubtitleAnnotationFilterOptions = {}, ): boolean { - // No Yomitan dictionary entry backs this token (ぅ～ elongations, truncated - // inflections) — it exists only to stay hoverable and must never receive - // annotations or count in the N+1 math. - if (token.isUnparsedRun === true) { - return true; - } - - const pos1Exclusions = resolvePos1Exclusions(options); - const pos2Exclusions = resolvePos2Exclusions(options); - if (!isContentTokenByPos(token, pos1Exclusions, pos2Exclusions)) { - return true; - } - - if (isAuxiliaryStemGrammarTailToken(token)) { - return true; - } - - if (isKanaOnlyNonIndependentNounHelperMerge(token)) { - return true; - } - - if (isStandaloneAuxiliaryInflectionFragment(token)) { - return true; - } - - if (isAuxiliaryOnlyHelperSpan(token)) { - return true; - } - - if (isStandaloneSuruTeGrammarHelper(token)) { - return true; - } - - if (isStandaloneGrammarParticle(token)) { - return true; - } - - if (isSingleKanaSurfaceFragment(token)) { - return true; - } - - if (isExcludedTrailingParticleMergedToken(token)) { - return true; - } - - if (isLexicalKureruVerb(token)) { - return false; - } - - return isExcludedByTerm(token); + return ( + evaluateSubtitleAnnotationRules(createSubtitleAnnotationRuleContext(token, options)) === + 'exclude' + ); } export function stripSubtitleAnnotationMetadata( diff --git a/src/core/services/tokenizer/token-classification.ts b/src/core/services/tokenizer/token-classification.ts index 259b5e58..d5c46cf8 100644 --- a/src/core/services/tokenizer/token-classification.ts +++ b/src/core/services/tokenizer/token-classification.ts @@ -105,7 +105,9 @@ function hasKanjiChar(text: string): boolean { code !== undefined && ((code >= 0x3400 && code <= 0x4dbf) || (code >= 0x4e00 && code <= 0x9fff) || - (code >= 0xf900 && code <= 0xfaff)) + (code >= 0xf900 && code <= 0xfaff) || + (code >= 0x20000 && code <= 0x2fa1f) || + (code >= 0x30000 && code <= 0x323af)) ) { return true; } @@ -115,7 +117,7 @@ function hasKanjiChar(text: string): boolean { // MeCab's 非自立 tag suppresses kana grammar nouns (こと, もの, とき), but // Yomitan can segment kanji-bearing nouns (日, 方, 上, …) as real vocabulary. -function isKanjiNonIndependentNounToken( +export function isKanjiNonIndependentNounToken( token: MergedToken, pos1Exclusions: ReadonlySet, ): boolean {