mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-27 04:49:49 -07:00
refactor(tokenizer): extract subtitle annotation filter into rule table (#162)
This commit is contained in:
GitHub
@@ -0,0 +1,4 @@
|
||||
type: fixed
|
||||
area: overlay
|
||||
|
||||
- Applied configured primary POS exclusions consistently to merged trailing quote-particle tokens, preserved annotations for supplementary-plane kanji, and stopped treating katakana punctuation as kana-only annotation noise.
|
||||
@@ -0,0 +1,342 @@
|
||||
import {
|
||||
DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG,
|
||||
resolveAnnotationPos1ExclusionSet,
|
||||
} from '../../../token-pos1-exclusions';
|
||||
import {
|
||||
DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG,
|
||||
resolveAnnotationPos2ExclusionSet,
|
||||
} from '../../../token-pos2-exclusions';
|
||||
import { MergedToken } from '../../../types';
|
||||
import { shouldIgnoreJlptByTerm } from '../jlpt-token-filter';
|
||||
import { isSubtitleGrammarEndingText } from './grammar-ending';
|
||||
import {
|
||||
isAuxiliaryOnlyHelperSpan,
|
||||
isAuxiliaryStemGrammarTailToken,
|
||||
isExcludedTrailingParticleMergedToken,
|
||||
isKanaOnlyNonIndependentNounHelperMerge,
|
||||
isReduplicatedKanaSfxWithOptionalTrailingTo,
|
||||
isSingleKanaSurfaceFragment,
|
||||
isStandaloneAuxiliaryInflectionFragment,
|
||||
isStandaloneGrammarParticle,
|
||||
isStandaloneSuruTeGrammarHelper,
|
||||
isTrailingSmallTsuKanaSfx,
|
||||
} from './subtitle-annotation-filter-support';
|
||||
import {
|
||||
isContentTokenByPos,
|
||||
isPosTagExcluded,
|
||||
isTokenPos2Excluded,
|
||||
normalizeKana,
|
||||
normalizePosTag,
|
||||
splitPosTag,
|
||||
} from './token-classification';
|
||||
|
||||
export interface SubtitleAnnotationFilterOptions {
|
||||
pos1Exclusions?: ReadonlySet<string>;
|
||||
pos2Exclusions?: ReadonlySet<string>;
|
||||
}
|
||||
|
||||
export type SubtitleAnnotationRuleDecision = 'exclude' | 'keep' | 'pass';
|
||||
|
||||
export interface SubtitleAnnotationRuleContext {
|
||||
token: MergedToken;
|
||||
pos1Exclusions: ReadonlySet<string>;
|
||||
pos2Exclusions: ReadonlySet<string>;
|
||||
normalizedPos1: string;
|
||||
normalizedPos2: string;
|
||||
hasPos1: boolean;
|
||||
hasPos2: boolean;
|
||||
}
|
||||
|
||||
type SubtitleAnnotationRuleData = Readonly<Record<string, readonly string[]>>;
|
||||
|
||||
export interface SubtitleAnnotationRule {
|
||||
readonly id: string;
|
||||
readonly description: string;
|
||||
readonly issueRef: string;
|
||||
readonly data: SubtitleAnnotationRuleData;
|
||||
test(context: SubtitleAnnotationRuleContext): SubtitleAnnotationRuleDecision;
|
||||
}
|
||||
|
||||
function defineRule<TData extends SubtitleAnnotationRuleData>(definition: {
|
||||
id: string;
|
||||
description: string;
|
||||
issueRef: string;
|
||||
data: TData;
|
||||
test: (context: SubtitleAnnotationRuleContext, data: TData) => SubtitleAnnotationRuleDecision;
|
||||
}): SubtitleAnnotationRule {
|
||||
const data = Object.freeze(
|
||||
Object.fromEntries(
|
||||
Object.entries(definition.data).map(([key, values]) => [key, Object.freeze([...values])]),
|
||||
),
|
||||
) as TData;
|
||||
return Object.freeze({
|
||||
id: definition.id,
|
||||
description: definition.description,
|
||||
issueRef: definition.issueRef,
|
||||
data,
|
||||
test: (context: SubtitleAnnotationRuleContext) => definition.test(context, data),
|
||||
});
|
||||
}
|
||||
|
||||
export function createSubtitleAnnotationRuleContext(
|
||||
token: MergedToken,
|
||||
options: SubtitleAnnotationFilterOptions = {},
|
||||
): SubtitleAnnotationRuleContext {
|
||||
const pos1Exclusions =
|
||||
options.pos1Exclusions ??
|
||||
resolveAnnotationPos1ExclusionSet(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG);
|
||||
const pos2Exclusions =
|
||||
options.pos2Exclusions ??
|
||||
resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
|
||||
const normalizedPos1 = normalizePosTag(token.pos1);
|
||||
const normalizedPos2 = normalizePosTag(token.pos2);
|
||||
return {
|
||||
token,
|
||||
pos1Exclusions,
|
||||
pos2Exclusions,
|
||||
normalizedPos1,
|
||||
normalizedPos2,
|
||||
hasPos1: normalizedPos1.length > 0,
|
||||
hasPos2: normalizedPos2.length > 0,
|
||||
};
|
||||
}
|
||||
|
||||
function matchesExcludedTermOrPattern(token: MergedToken, terms: ReadonlySet<string>): boolean {
|
||||
const candidates = [token.surface, token.reading, token.headword].filter(
|
||||
(candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0,
|
||||
);
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const trimmed = candidate.trim();
|
||||
if (!trimmed) {
|
||||
continue;
|
||||
}
|
||||
const normalized = normalizeKana(trimmed);
|
||||
if (!normalized) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
terms.has(trimmed) ||
|
||||
terms.has(normalized) ||
|
||||
isSubtitleGrammarEndingText(trimmed) ||
|
||||
isSubtitleGrammarEndingText(normalized) ||
|
||||
shouldIgnoreJlptByTerm(trimmed) ||
|
||||
shouldIgnoreJlptByTerm(normalized) ||
|
||||
isTrailingSmallTsuKanaSfx(trimmed) ||
|
||||
isTrailingSmallTsuKanaSfx(normalized) ||
|
||||
isReduplicatedKanaSfxWithOptionalTrailingTo(trimmed) ||
|
||||
isReduplicatedKanaSfxWithOptionalTrailingTo(normalized)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
|
||||
'あ',
|
||||
'ああ',
|
||||
'ある',
|
||||
'あなた',
|
||||
'あんた',
|
||||
'ええ',
|
||||
'うう',
|
||||
'おお',
|
||||
'おい',
|
||||
'お前',
|
||||
'こいつ',
|
||||
'こっち',
|
||||
'くれ',
|
||||
'じゃない',
|
||||
'そうだ',
|
||||
'たち',
|
||||
'である',
|
||||
'どこか',
|
||||
'なんか',
|
||||
'べき',
|
||||
'って',
|
||||
'はあ',
|
||||
'はぁ',
|
||||
'はは',
|
||||
'へえ',
|
||||
'ふう',
|
||||
'ほう',
|
||||
'何か',
|
||||
'何だ',
|
||||
'何も',
|
||||
'如何した',
|
||||
'有る',
|
||||
'在る',
|
||||
'様',
|
||||
'誰も',
|
||||
'貴方',
|
||||
'もんか',
|
||||
'ものか',
|
||||
]);
|
||||
|
||||
const excludedTermRule = defineRule({
|
||||
id: 'excluded-term-or-pattern',
|
||||
description:
|
||||
'Exclude legacy subtitle stop terms, grammar endings, JLPT stop terms, and kana sound effects.',
|
||||
issueRef: '#19, #33, #57',
|
||||
data: {},
|
||||
test: ({ token }) =>
|
||||
matchesExcludedTermOrPattern(token, SUBTITLE_ANNOTATION_EXCLUDED_TERMS) ? 'exclude' : 'pass',
|
||||
});
|
||||
|
||||
// Ordered, first-match-wins. issueRef records the introducing/fixing PR; early
|
||||
// rules are legacy behavior inherited from the original #19 filter.
|
||||
export const SUBTITLE_ANNOTATION_RULES: readonly SubtitleAnnotationRule[] = Object.freeze([
|
||||
defineRule({
|
||||
id: 'unparsed-run',
|
||||
description: 'Exclude hoverable parser gaps that have no Yomitan dictionary entry.',
|
||||
issueRef: '#153',
|
||||
data: {},
|
||||
test: ({ token }) => (token.isUnparsedRun === true ? 'exclude' : 'pass'),
|
||||
}),
|
||||
defineRule({
|
||||
id: 'configured-pos1-exclusion',
|
||||
description: 'Apply the configured primary part-of-speech exclusions.',
|
||||
issueRef: '#19',
|
||||
data: {},
|
||||
test: ({ token, pos1Exclusions }) =>
|
||||
isPosTagExcluded(token.pos1, pos1Exclusions) ? 'exclude' : 'pass',
|
||||
}),
|
||||
defineRule({
|
||||
id: 'configured-pos2-exclusion',
|
||||
description: 'Apply configured secondary POS exclusions, preserving #150 kanji nouns.',
|
||||
issueRef: '#150',
|
||||
data: {},
|
||||
test: ({ token, pos1Exclusions, pos2Exclusions }) =>
|
||||
isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions) ? 'exclude' : 'pass',
|
||||
}),
|
||||
defineRule({
|
||||
id: 'coarse-grammar-pos-fallback',
|
||||
description: 'Exclude coarse grammar POS when detailed MeCab tags are unavailable.',
|
||||
issueRef: '#19',
|
||||
data: {},
|
||||
test: ({ token, hasPos1, hasPos2, pos1Exclusions, pos2Exclusions }) =>
|
||||
!hasPos1 && !hasPos2 && !isContentTokenByPos(token, pos1Exclusions, pos2Exclusions)
|
||||
? 'exclude'
|
||||
: 'pass',
|
||||
}),
|
||||
defineRule({
|
||||
id: 'auxiliary-stem-grammar-tail',
|
||||
description: 'Exclude merged grammar tails containing an auxiliary stem.',
|
||||
issueRef: '#19',
|
||||
data: { allowedPos1: ['名詞', '助動詞', '助詞'] },
|
||||
test: ({ token }, { allowedPos1 }) =>
|
||||
isAuxiliaryStemGrammarTailToken(token, allowedPos1) ? 'exclude' : 'pass',
|
||||
}),
|
||||
defineRule({
|
||||
id: 'kana-non-independent-noun-helper',
|
||||
description: 'Exclude kana non-independent nouns merged with grammar helpers.',
|
||||
issueRef: '#56',
|
||||
data: { tailPos1: ['助詞', '助動詞'] },
|
||||
test: ({ token }, { tailPos1 }) =>
|
||||
isKanaOnlyNonIndependentNounHelperMerge(token, tailPos1) ? 'exclude' : 'pass',
|
||||
}),
|
||||
defineRule({
|
||||
id: 'standalone-auxiliary-inflection',
|
||||
description: 'Exclude standalone kana auxiliary inflection fragments.',
|
||||
issueRef: '#57',
|
||||
data: { trailingPos1: ['助動詞'] },
|
||||
test: ({ token }, { trailingPos1 }) =>
|
||||
isStandaloneAuxiliaryInflectionFragment(token, trailingPos1) ? 'exclude' : 'pass',
|
||||
}),
|
||||
defineRule({
|
||||
id: 'auxiliary-only-helper-span',
|
||||
description: 'Exclude kana helper spans without an independent lexical verb.',
|
||||
issueRef: '#57',
|
||||
data: { allowedPos1: ['助詞', '助動詞', '動詞'], lexicalVerbPos2: ['自立'] },
|
||||
test: ({ token }, { allowedPos1, lexicalVerbPos2 }) =>
|
||||
isAuxiliaryOnlyHelperSpan(token, allowedPos1, lexicalVerbPos2) ? 'exclude' : 'pass',
|
||||
}),
|
||||
defineRule({
|
||||
id: 'standalone-suru-te-helper',
|
||||
description: 'Exclude standalone して grammar-helper fragments.',
|
||||
issueRef: '#57',
|
||||
data: {},
|
||||
test: ({ token }) => (isStandaloneSuruTeGrammarHelper(token) ? 'exclude' : 'pass'),
|
||||
}),
|
||||
defineRule({
|
||||
id: 'standalone-grammar-particle',
|
||||
description: 'Exclude standalone particle surfaces and connective particle phrases.',
|
||||
issueRef: '#57',
|
||||
data: {
|
||||
surfaces: [
|
||||
'か',
|
||||
'が',
|
||||
'さ',
|
||||
'し',
|
||||
'ぞ',
|
||||
'ぜ',
|
||||
'と',
|
||||
'な',
|
||||
'に',
|
||||
'ね',
|
||||
'の',
|
||||
'は',
|
||||
'へ',
|
||||
'も',
|
||||
'や',
|
||||
'よ',
|
||||
'を',
|
||||
],
|
||||
phrases: ['たって', 'だって'],
|
||||
},
|
||||
test: ({ token }, { surfaces, phrases }) =>
|
||||
isStandaloneGrammarParticle(token, surfaces, phrases) ? 'exclude' : 'pass',
|
||||
}),
|
||||
defineRule({
|
||||
id: 'single-kana-fragment',
|
||||
description: 'Exclude isolated one-kana parser fragments.',
|
||||
issueRef: '#57',
|
||||
data: {},
|
||||
test: ({ token }) => (isSingleKanaSurfaceFragment(token) ? 'exclude' : 'pass'),
|
||||
}),
|
||||
defineRule({
|
||||
id: 'merged-trailing-quote-particle',
|
||||
description: 'Exclude lexical tokens merged only with a trailing quote-particle suffix.',
|
||||
issueRef: '#19',
|
||||
data: { suffixes: ['って', 'ってよ', 'ってね', 'ってな', 'ってさ', 'ってか', 'ってば'] },
|
||||
test: ({ token, pos1Exclusions }, { suffixes }) =>
|
||||
isExcludedTrailingParticleMergedToken(token, suffixes, pos1Exclusions) ? 'exclude' : 'pass',
|
||||
}),
|
||||
defineRule({
|
||||
id: 'lexical-kureru-keep',
|
||||
description: 'Keep lexical くれる before the legacy bare-くれ stop-term rule.',
|
||||
issueRef: '#57',
|
||||
data: {
|
||||
surfaces: ['くれ'],
|
||||
headwords: ['くれる'],
|
||||
pos1: ['動詞'],
|
||||
pos2: ['自立'],
|
||||
},
|
||||
test: ({ token }, data) => {
|
||||
const pos1 = splitPosTag(token.pos1);
|
||||
const pos2 = splitPosTag(token.pos2);
|
||||
return data.surfaces.includes(normalizeKana(token.surface)) &&
|
||||
data.headwords.includes(normalizeKana(token.headword)) &&
|
||||
pos1.length === 1 &&
|
||||
data.pos1.includes(pos1[0] ?? '') &&
|
||||
pos2.length === 1 &&
|
||||
data.pos2.includes(pos2[0] ?? '')
|
||||
? 'keep'
|
||||
: 'pass';
|
||||
},
|
||||
}),
|
||||
excludedTermRule,
|
||||
]);
|
||||
|
||||
export function evaluateSubtitleAnnotationRules(
|
||||
context: SubtitleAnnotationRuleContext,
|
||||
): SubtitleAnnotationRuleDecision {
|
||||
for (const rule of SUBTITLE_ANNOTATION_RULES) {
|
||||
const decision = rule.test(context);
|
||||
if (decision !== 'pass') {
|
||||
return decision;
|
||||
}
|
||||
}
|
||||
return 'pass';
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
import { MergedToken } from '../../../types';
|
||||
import { isKanaChar, isKanaOnlyText, normalizeKana, splitPosTag } from './token-classification';
|
||||
|
||||
export function isTrailingSmallTsuKanaSfx(text: string): boolean {
|
||||
const chars = [...normalizeKana(text)];
|
||||
return (
|
||||
chars.length >= 2 &&
|
||||
chars.length <= 4 &&
|
||||
chars.every(isKanaChar) &&
|
||||
chars[chars.length - 1] === 'っ'
|
||||
);
|
||||
}
|
||||
|
||||
function isReduplicatedKanaSfx(text: string): boolean {
|
||||
const chars = [...normalizeKana(text)];
|
||||
if (chars.length < 4 || chars.length % 2 !== 0 || !chars.every(isKanaChar)) {
|
||||
return false;
|
||||
}
|
||||
const half = chars.length / 2;
|
||||
return chars.slice(0, half).join('') === chars.slice(half).join('');
|
||||
}
|
||||
|
||||
export function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
|
||||
const normalized = normalizeKana(text);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
if (isReduplicatedKanaSfx(normalized)) {
|
||||
return true;
|
||||
}
|
||||
return normalized.length > 1 && normalized.endsWith('と')
|
||||
? isReduplicatedKanaSfx(normalized.slice(0, -1))
|
||||
: false;
|
||||
}
|
||||
|
||||
export function isExcludedTrailingParticleMergedToken(
|
||||
token: MergedToken,
|
||||
suffixes: readonly string[],
|
||||
leadingPos1Exclusions: ReadonlySet<string>,
|
||||
): boolean {
|
||||
const surface = normalizeKana(token.surface);
|
||||
const headword = normalizeKana(token.headword);
|
||||
if (!surface || !headword || !surface.startsWith(headword)) {
|
||||
return false;
|
||||
}
|
||||
if (!suffixes.includes(surface.slice(headword.length))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const [leadingPos1, ...trailingPos1] = splitPosTag(token.pos1);
|
||||
if (!leadingPos1 || leadingPos1Exclusions.has(leadingPos1)) {
|
||||
return false;
|
||||
}
|
||||
return trailingPos1.length > 0 && trailingPos1.every((part) => part === '助詞');
|
||||
}
|
||||
|
||||
export function isAuxiliaryStemGrammarTailToken(
|
||||
token: MergedToken,
|
||||
allowedPos1: readonly string[],
|
||||
): boolean {
|
||||
const pos1Parts = splitPosTag(token.pos1);
|
||||
if (pos1Parts.length === 0 || !pos1Parts.every((part) => allowedPos1.includes(part))) {
|
||||
return false;
|
||||
}
|
||||
return splitPosTag(token.pos3).includes('助動詞語幹');
|
||||
}
|
||||
|
||||
export function isKanaOnlyNonIndependentNounHelperMerge(
|
||||
token: MergedToken,
|
||||
tailPos1: readonly string[],
|
||||
): boolean {
|
||||
const surface = normalizeKana(token.surface);
|
||||
const headword = normalizeKana(token.headword);
|
||||
if (!surface || !headword || surface === headword || ![...surface].every(isKanaChar)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos1Parts = splitPosTag(token.pos1);
|
||||
if (pos1Parts.length < 2 || pos1Parts[0] !== '名詞') {
|
||||
return false;
|
||||
}
|
||||
const pos2Parts = splitPosTag(token.pos2);
|
||||
return pos2Parts[0] === '非自立' && pos1Parts.slice(1).every((part) => tailPos1.includes(part));
|
||||
}
|
||||
|
||||
export function isStandaloneAuxiliaryInflectionFragment(
|
||||
token: MergedToken,
|
||||
trailingPos1: readonly string[],
|
||||
): boolean {
|
||||
if (!isKanaOnlyText(token.surface)) {
|
||||
return false;
|
||||
}
|
||||
const pos1Parts = splitPosTag(token.pos1);
|
||||
if (pos1Parts.length === 0) {
|
||||
return false;
|
||||
}
|
||||
if (pos1Parts.every((part) => part === '助動詞')) {
|
||||
return true;
|
||||
}
|
||||
const pos2Parts = splitPosTag(token.pos2);
|
||||
return (
|
||||
pos1Parts[0] === '動詞' &&
|
||||
pos2Parts[0] === '接尾' &&
|
||||
pos1Parts.slice(1).every((part) => trailingPos1.includes(part))
|
||||
);
|
||||
}
|
||||
|
||||
export function isAuxiliaryOnlyHelperSpan(
|
||||
token: MergedToken,
|
||||
allowedPos1: readonly string[],
|
||||
lexicalVerbPos2: readonly string[],
|
||||
): boolean {
|
||||
if (!isKanaOnlyText(token.surface) || !isKanaOnlyText(token.headword)) {
|
||||
return false;
|
||||
}
|
||||
const pos1Parts = splitPosTag(token.pos1);
|
||||
if (
|
||||
pos1Parts.length === 0 ||
|
||||
!pos1Parts.every((part) => allowedPos1.includes(part)) ||
|
||||
!pos1Parts.includes('助詞') ||
|
||||
!pos1Parts.includes('動詞')
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return !splitPosTag(token.pos2).some((part) => lexicalVerbPos2.includes(part));
|
||||
}
|
||||
|
||||
export function isStandaloneSuruTeGrammarHelper(token: MergedToken): boolean {
|
||||
const surface = normalizeKana(token.surface);
|
||||
const headword = normalizeKana(token.headword);
|
||||
if (!surface.startsWith('して') || headword !== 'する') {
|
||||
return false;
|
||||
}
|
||||
const pos1Parts = splitPosTag(token.pos1);
|
||||
return isKanaOnlyText(surface) && (pos1Parts.length === 0 || pos1Parts.includes('動詞'));
|
||||
}
|
||||
|
||||
export function isStandaloneGrammarParticle(
|
||||
token: MergedToken,
|
||||
surfaces: readonly string[],
|
||||
phrases: readonly string[],
|
||||
): boolean {
|
||||
const surface = normalizeKana(token.surface);
|
||||
return (
|
||||
surface === normalizeKana(token.headword) &&
|
||||
(surfaces.includes(surface) || phrases.includes(surface))
|
||||
);
|
||||
}
|
||||
|
||||
export function isSingleKanaSurfaceFragment(token: MergedToken): boolean {
|
||||
const chars = [...normalizeKana(token.surface)];
|
||||
return chars.length === 1 && chars.every(isKanaChar);
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { MergedToken, PartOfSpeech } from '../../../types';
|
||||
import {
|
||||
createSubtitleAnnotationRuleContext,
|
||||
shouldExcludeTokenFromSubtitleAnnotations,
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS,
|
||||
SUBTITLE_ANNOTATION_RULES,
|
||||
} from './subtitle-annotation-filter';
|
||||
import { isKanaChar } from './token-classification';
|
||||
|
||||
function makeToken(overrides: Partial<MergedToken> = {}): MergedToken {
|
||||
return {
|
||||
surface: '猫',
|
||||
reading: 'ネコ',
|
||||
headword: '猫',
|
||||
startPos: 0,
|
||||
endPos: 1,
|
||||
partOfSpeech: PartOfSpeech.noun,
|
||||
isMerged: false,
|
||||
isKnown: false,
|
||||
isNPlusOneTarget: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('subtitle annotation rules expose stable ordered provenance', () => {
|
||||
assert.deepEqual(
|
||||
SUBTITLE_ANNOTATION_RULES.map(({ id, issueRef }) => ({ id, issueRef })),
|
||||
[
|
||||
{ id: 'unparsed-run', issueRef: '#153' },
|
||||
{ id: 'configured-pos1-exclusion', issueRef: '#19' },
|
||||
{ id: 'configured-pos2-exclusion', issueRef: '#150' },
|
||||
{ id: 'coarse-grammar-pos-fallback', issueRef: '#19' },
|
||||
{ id: 'auxiliary-stem-grammar-tail', issueRef: '#19' },
|
||||
{ id: 'kana-non-independent-noun-helper', issueRef: '#56' },
|
||||
{ id: 'standalone-auxiliary-inflection', issueRef: '#57' },
|
||||
{ id: 'auxiliary-only-helper-span', issueRef: '#57' },
|
||||
{ id: 'standalone-suru-te-helper', issueRef: '#57' },
|
||||
{ id: 'standalone-grammar-particle', issueRef: '#57' },
|
||||
{ id: 'single-kana-fragment', issueRef: '#57' },
|
||||
{ id: 'merged-trailing-quote-particle', issueRef: '#19' },
|
||||
{ id: 'lexical-kureru-keep', issueRef: '#57' },
|
||||
{ id: 'excluded-term-or-pattern', issueRef: '#19, #33, #57' },
|
||||
],
|
||||
);
|
||||
assert.equal(new Set(SUBTITLE_ANNOTATION_RULES.map(({ id }) => id)).size, 14);
|
||||
assert.ok(SUBTITLE_ANNOTATION_RULES.every(({ description }) => description.length > 0));
|
||||
assert.ok(
|
||||
SUBTITLE_ANNOTATION_RULES.every(
|
||||
({ data }) => Object.isFrozen(data) && Object.values(data).every(Object.isFrozen),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('lexical kureru keep rule precedes and overrides the excluded-term rule', () => {
|
||||
const token = makeToken({
|
||||
surface: 'くれ',
|
||||
headword: 'くれる',
|
||||
reading: 'クレ',
|
||||
partOfSpeech: PartOfSpeech.verb,
|
||||
pos1: '動詞',
|
||||
pos2: '自立',
|
||||
});
|
||||
const context = createSubtitleAnnotationRuleContext(token);
|
||||
const keepIndex = SUBTITLE_ANNOTATION_RULES.findIndex(({ id }) => id === 'lexical-kureru-keep');
|
||||
const termIndex = SUBTITLE_ANNOTATION_RULES.findIndex(
|
||||
({ id }) => id === 'excluded-term-or-pattern',
|
||||
);
|
||||
|
||||
assert.ok(keepIndex >= 0 && keepIndex < termIndex);
|
||||
assert.ok(SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has('くれ'));
|
||||
assert.equal(SUBTITLE_ANNOTATION_RULES[keepIndex]?.test(context), 'keep');
|
||||
assert.equal(SUBTITLE_ANNOTATION_RULES[termIndex]?.test(context), 'exclude');
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
|
||||
});
|
||||
|
||||
test('adding an exported excluded term changes annotation matching', () => {
|
||||
const term = '超語彙';
|
||||
const token = makeToken({
|
||||
surface: term,
|
||||
headword: term,
|
||||
reading: term,
|
||||
pos1: '名詞',
|
||||
pos2: '一般',
|
||||
});
|
||||
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term);
|
||||
try {
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS.add(term);
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
|
||||
} finally {
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term);
|
||||
}
|
||||
});
|
||||
|
||||
test('deleting an exported excluded term changes annotation matching', () => {
|
||||
const term = 'あなた';
|
||||
const token = makeToken({
|
||||
surface: term,
|
||||
headword: term,
|
||||
reading: term,
|
||||
pos1: '名詞',
|
||||
pos2: '一般',
|
||||
});
|
||||
|
||||
assert.ok(SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(term));
|
||||
try {
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term);
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
|
||||
} finally {
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS.add(term);
|
||||
}
|
||||
});
|
||||
|
||||
test('configured POS rules consume the context exclusion sets in table order', () => {
|
||||
const token = makeToken({ pos1: '名詞', pos2: '一般' });
|
||||
const context = createSubtitleAnnotationRuleContext(token, {
|
||||
pos1Exclusions: new Set(['名詞']),
|
||||
pos2Exclusions: new Set(['一般']),
|
||||
});
|
||||
const pos1Rule = SUBTITLE_ANNOTATION_RULES.find(({ id }) => id === 'configured-pos1-exclusion');
|
||||
const pos2Rule = SUBTITLE_ANNOTATION_RULES.find(({ id }) => id === 'configured-pos2-exclusion');
|
||||
|
||||
assert.equal(pos1Rule?.test(context), 'exclude');
|
||||
assert.equal(pos2Rule?.test(context), 'exclude');
|
||||
assert.equal(
|
||||
shouldExcludeTokenFromSubtitleAnnotations(token, {
|
||||
pos1Exclusions: context.pos1Exclusions,
|
||||
pos2Exclusions: context.pos2Exclusions,
|
||||
}),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('configured POS2 rule preserves the kanji non-independent noun exception', () => {
|
||||
const token = makeToken({ surface: '以外', headword: '以外', pos1: '名詞', pos2: '非自立' });
|
||||
const context = createSubtitleAnnotationRuleContext(token, {
|
||||
pos1Exclusions: new Set(),
|
||||
pos2Exclusions: new Set(['非自立']),
|
||||
});
|
||||
const pos2Rule = SUBTITLE_ANNOTATION_RULES.find(({ id }) => id === 'configured-pos2-exclusion');
|
||||
|
||||
assert.equal(pos2Rule?.test(context), 'pass');
|
||||
assert.equal(
|
||||
shouldExcludeTokenFromSubtitleAnnotations(token, {
|
||||
pos1Exclusions: context.pos1Exclusions,
|
||||
pos2Exclusions: context.pos2Exclusions,
|
||||
}),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('trailing quote-particle rule honors configured POS1 exclusions', () => {
|
||||
const token = makeToken({ surface: '猫って', headword: '猫', pos1: '名詞|助詞' });
|
||||
const context = createSubtitleAnnotationRuleContext(token, {
|
||||
pos1Exclusions: new Set(['名詞']),
|
||||
});
|
||||
const trailingParticleRule = SUBTITLE_ANNOTATION_RULES.find(
|
||||
({ id }) => id === 'merged-trailing-quote-particle',
|
||||
);
|
||||
|
||||
assert.equal(trailingParticleRule?.test(context), 'pass');
|
||||
});
|
||||
|
||||
test('configured POS2 rule preserves supplementary-plane kanji nouns', () => {
|
||||
const token = makeToken({ surface: '𠮟', headword: '𠮟', pos1: '名詞', pos2: '非自立' });
|
||||
|
||||
assert.equal(
|
||||
shouldExcludeTokenFromSubtitleAnnotations(token, {
|
||||
pos1Exclusions: new Set(),
|
||||
pos2Exclusions: new Set(['非自立']),
|
||||
}),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('kana detection excludes katakana punctuation boundaries', () => {
|
||||
assert.equal(isKanaChar('゠'), false);
|
||||
assert.equal(isKanaChar('・'), false);
|
||||
assert.equal(isKanaChar('ァ'), true);
|
||||
});
|
||||
@@ -1,424 +1,31 @@
|
||||
import {
|
||||
DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG,
|
||||
resolveAnnotationPos1ExclusionSet,
|
||||
} from '../../../token-pos1-exclusions';
|
||||
import {
|
||||
DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG,
|
||||
resolveAnnotationPos2ExclusionSet,
|
||||
} from '../../../token-pos2-exclusions';
|
||||
import { MergedToken } from '../../../types';
|
||||
import { shouldIgnoreJlptByTerm } from '../jlpt-token-filter';
|
||||
import { isSubtitleGrammarEndingText } from './grammar-ending';
|
||||
import {
|
||||
isContentTokenByPos,
|
||||
isKanaChar,
|
||||
isKanaOnlyText,
|
||||
normalizeKana,
|
||||
splitPosTag,
|
||||
} from './token-classification';
|
||||
createSubtitleAnnotationRuleContext,
|
||||
evaluateSubtitleAnnotationRules,
|
||||
} from './subtitle-annotation-filter-rules';
|
||||
import type { SubtitleAnnotationFilterOptions } from './subtitle-annotation-filter-rules';
|
||||
|
||||
const STANDALONE_GRAMMAR_PARTICLE_PHRASES = ['たって', 'だって'] as const;
|
||||
const STANDALONE_GRAMMAR_PARTICLE_PHRASES_SET: ReadonlySet<string> = new Set(
|
||||
STANDALONE_GRAMMAR_PARTICLE_PHRASES,
|
||||
);
|
||||
|
||||
export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
|
||||
'あ',
|
||||
'ああ',
|
||||
'ある',
|
||||
'あなた',
|
||||
'あんた',
|
||||
'ええ',
|
||||
'うう',
|
||||
'おお',
|
||||
'おい',
|
||||
'お前',
|
||||
'こいつ',
|
||||
'こっち',
|
||||
'くれ',
|
||||
'じゃない',
|
||||
'そうだ',
|
||||
'たち',
|
||||
'である',
|
||||
'どこか',
|
||||
'なんか',
|
||||
'べき',
|
||||
'って',
|
||||
'はあ',
|
||||
'はぁ',
|
||||
'はは',
|
||||
'へえ',
|
||||
'ふう',
|
||||
'ほう',
|
||||
'何か',
|
||||
'何だ',
|
||||
'何も',
|
||||
'如何した',
|
||||
'有る',
|
||||
'在る',
|
||||
'様',
|
||||
'誰も',
|
||||
'貴方',
|
||||
'もんか',
|
||||
'ものか',
|
||||
]);
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES = new Set([
|
||||
'って',
|
||||
'ってよ',
|
||||
'ってね',
|
||||
'ってな',
|
||||
'ってさ',
|
||||
'ってか',
|
||||
'ってば',
|
||||
]);
|
||||
const AUXILIARY_STEM_GRAMMAR_TAIL_POS1 = new Set(['名詞', '助動詞', '助詞']);
|
||||
const NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1 = new Set(['助詞', '助動詞']);
|
||||
const AUXILIARY_INFLECTION_TRAILING_POS1 = new Set(['助動詞']);
|
||||
const AUXILIARY_HELPER_SPAN_POS1 = new Set(['助詞', '助動詞', '動詞']);
|
||||
const LEXICAL_VERB_POS2 = new Set(['自立']);
|
||||
const STANDALONE_GRAMMAR_PARTICLE_SURFACES = new Set([
|
||||
'か',
|
||||
'が',
|
||||
'さ',
|
||||
'し',
|
||||
'ぞ',
|
||||
'ぜ',
|
||||
'と',
|
||||
'な',
|
||||
'に',
|
||||
'ね',
|
||||
'の',
|
||||
'は',
|
||||
'へ',
|
||||
'も',
|
||||
'や',
|
||||
'よ',
|
||||
'を',
|
||||
]);
|
||||
export interface SubtitleAnnotationFilterOptions {
|
||||
pos1Exclusions?: ReadonlySet<string>;
|
||||
pos2Exclusions?: ReadonlySet<string>;
|
||||
}
|
||||
|
||||
function resolvePos1Exclusions(options: SubtitleAnnotationFilterOptions = {}): ReadonlySet<string> {
|
||||
if (options.pos1Exclusions) {
|
||||
return options.pos1Exclusions;
|
||||
}
|
||||
|
||||
return resolveAnnotationPos1ExclusionSet(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG);
|
||||
}
|
||||
|
||||
function resolvePos2Exclusions(options: SubtitleAnnotationFilterOptions = {}): ReadonlySet<string> {
|
||||
if (options.pos2Exclusions) {
|
||||
return options.pos2Exclusions;
|
||||
}
|
||||
|
||||
return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
|
||||
}
|
||||
|
||||
function isTrailingSmallTsuKanaSfx(text: string): boolean {
|
||||
const normalized = normalizeKana(text);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const chars = [...normalized];
|
||||
if (chars.length < 2 || chars.length > 4) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!chars.every(isKanaChar)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return chars[chars.length - 1] === 'っ';
|
||||
}
|
||||
|
||||
function isReduplicatedKanaSfx(text: string): boolean {
|
||||
const normalized = normalizeKana(text);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const chars = [...normalized];
|
||||
if (chars.length < 4 || chars.length % 2 !== 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!chars.every(isKanaChar)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const half = chars.length / 2;
|
||||
return chars.slice(0, half).join('') === chars.slice(half).join('');
|
||||
}
|
||||
|
||||
function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
|
||||
const normalized = normalizeKana(text);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isReduplicatedKanaSfx(normalized)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (normalized.length <= 1 || !normalized.endsWith('と')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return isReduplicatedKanaSfx(normalized.slice(0, -1));
|
||||
}
|
||||
|
||||
function isExcludedTrailingParticleMergedToken(token: MergedToken): boolean {
|
||||
const normalizedSurface = normalizeKana(token.surface);
|
||||
const normalizedHeadword = normalizeKana(token.headword);
|
||||
if (
|
||||
!normalizedSurface ||
|
||||
!normalizedHeadword ||
|
||||
!normalizedSurface.startsWith(normalizedHeadword)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const suffix = normalizedSurface.slice(normalizedHeadword.length);
|
||||
if (!SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES.has(suffix)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos1Parts = splitPosTag(token.pos1);
|
||||
if (pos1Parts.length < 2) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const [leadingPos1, ...trailingPos1] = pos1Parts;
|
||||
if (!leadingPos1 || resolvePos1Exclusions().has(leadingPos1)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return trailingPos1.length > 0 && trailingPos1.every((part) => part === '助詞');
|
||||
}
|
||||
|
||||
function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean {
|
||||
const pos1Parts = splitPosTag(token.pos1);
|
||||
if (
|
||||
pos1Parts.length === 0 ||
|
||||
!pos1Parts.every((part) => AUXILIARY_STEM_GRAMMAR_TAIL_POS1.has(part))
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos3Parts = splitPosTag(token.pos3);
|
||||
return pos3Parts.includes('助動詞語幹');
|
||||
}
|
||||
|
||||
function isKanaOnlyNonIndependentNounHelperMerge(token: MergedToken): boolean {
|
||||
const normalizedSurface = normalizeKana(token.surface);
|
||||
const normalizedHeadword = normalizeKana(token.headword);
|
||||
if (
|
||||
!normalizedSurface ||
|
||||
!normalizedHeadword ||
|
||||
normalizedSurface === normalizedHeadword ||
|
||||
![...normalizedSurface].every(isKanaChar)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos1Parts = splitPosTag(token.pos1);
|
||||
if (pos1Parts.length < 2 || pos1Parts[0] !== '名詞') {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos2Parts = splitPosTag(token.pos2);
|
||||
if (pos2Parts[0] !== '非自立') {
|
||||
return false;
|
||||
}
|
||||
|
||||
return pos1Parts.slice(1).every((part) => NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1.has(part));
|
||||
}
|
||||
|
||||
function isLexicalKureruVerb(token: MergedToken): boolean {
|
||||
const normalizedSurface = normalizeKana(token.surface);
|
||||
const normalizedHeadword = normalizeKana(token.headword);
|
||||
const pos1Parts = splitPosTag(token.pos1);
|
||||
const pos2Parts = splitPosTag(token.pos2);
|
||||
return (
|
||||
normalizedSurface === 'くれ' &&
|
||||
normalizedHeadword === 'くれる' &&
|
||||
pos1Parts.length === 1 &&
|
||||
pos1Parts[0] === '動詞' &&
|
||||
pos2Parts.length === 1 &&
|
||||
pos2Parts[0] === '自立'
|
||||
);
|
||||
}
|
||||
|
||||
function isStandaloneAuxiliaryInflectionFragment(token: MergedToken): boolean {
|
||||
const normalizedSurface = normalizeKana(token.surface);
|
||||
if (!isKanaOnlyText(normalizedSurface)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos1Parts = splitPosTag(token.pos1);
|
||||
if (pos1Parts.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (pos1Parts.every((part) => part === '助動詞')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const pos2Parts = splitPosTag(token.pos2);
|
||||
return (
|
||||
pos1Parts[0] === '動詞' &&
|
||||
pos2Parts[0] === '接尾' &&
|
||||
pos1Parts.slice(1).every((part) => AUXILIARY_INFLECTION_TRAILING_POS1.has(part))
|
||||
);
|
||||
}
|
||||
|
||||
function isAuxiliaryOnlyHelperSpan(token: MergedToken): boolean {
|
||||
const normalizedSurface = normalizeKana(token.surface);
|
||||
const normalizedHeadword = normalizeKana(token.headword);
|
||||
if (!isKanaOnlyText(normalizedSurface) || !isKanaOnlyText(normalizedHeadword)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos1Parts = splitPosTag(token.pos1);
|
||||
if (
|
||||
pos1Parts.length === 0 ||
|
||||
!pos1Parts.every((part) => AUXILIARY_HELPER_SPAN_POS1.has(part)) ||
|
||||
!pos1Parts.includes('助詞') ||
|
||||
!pos1Parts.includes('動詞')
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos2Parts = splitPosTag(token.pos2);
|
||||
return !pos2Parts.some((part) => LEXICAL_VERB_POS2.has(part));
|
||||
}
|
||||
|
||||
function isStandaloneSuruTeGrammarHelper(token: MergedToken): boolean {
|
||||
const normalizedSurface = normalizeKana(token.surface);
|
||||
const normalizedHeadword = normalizeKana(token.headword);
|
||||
if (!normalizedSurface.startsWith('して') || normalizedHeadword !== 'する') {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos1Parts = splitPosTag(token.pos1);
|
||||
return (
|
||||
isKanaOnlyText(normalizedSurface) && (pos1Parts.length === 0 || pos1Parts.includes('動詞'))
|
||||
);
|
||||
}
|
||||
|
||||
function isStandaloneGrammarParticle(token: MergedToken): boolean {
|
||||
const normalizedSurface = normalizeKana(token.surface);
|
||||
const normalizedHeadword = normalizeKana(token.headword);
|
||||
return (
|
||||
normalizedSurface === normalizedHeadword &&
|
||||
(STANDALONE_GRAMMAR_PARTICLE_SURFACES.has(normalizedSurface) ||
|
||||
STANDALONE_GRAMMAR_PARTICLE_PHRASES_SET.has(normalizedSurface))
|
||||
);
|
||||
}
|
||||
|
||||
function isSingleKanaSurfaceFragment(token: MergedToken): boolean {
|
||||
const normalizedSurface = normalizeKana(token.surface);
|
||||
const chars = [...normalizedSurface];
|
||||
return chars.length === 1 && chars.every(isKanaChar);
|
||||
}
|
||||
|
||||
function isExcludedByTerm(token: MergedToken): boolean {
|
||||
const candidates = [token.surface, token.reading, token.headword].filter(
|
||||
(candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0,
|
||||
);
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const trimmed = candidate.trim();
|
||||
if (!trimmed) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const normalized = normalizeKana(trimmed);
|
||||
if (!normalized) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(trimmed) ||
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(normalized) ||
|
||||
isSubtitleGrammarEndingText(trimmed) ||
|
||||
isSubtitleGrammarEndingText(normalized) ||
|
||||
shouldIgnoreJlptByTerm(trimmed) ||
|
||||
shouldIgnoreJlptByTerm(normalized)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
isTrailingSmallTsuKanaSfx(trimmed) ||
|
||||
isTrailingSmallTsuKanaSfx(normalized) ||
|
||||
isReduplicatedKanaSfxWithOptionalTrailingTo(trimmed) ||
|
||||
isReduplicatedKanaSfxWithOptionalTrailingTo(normalized)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
export {
|
||||
createSubtitleAnnotationRuleContext,
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS,
|
||||
SUBTITLE_ANNOTATION_RULES,
|
||||
} from './subtitle-annotation-filter-rules';
|
||||
export type {
|
||||
SubtitleAnnotationFilterOptions,
|
||||
SubtitleAnnotationRule,
|
||||
SubtitleAnnotationRuleContext,
|
||||
SubtitleAnnotationRuleDecision,
|
||||
} from './subtitle-annotation-filter-rules';
|
||||
export { isKanjiNonIndependentNounToken } from './token-classification';
|
||||
|
||||
export function shouldExcludeTokenFromSubtitleAnnotations(
|
||||
token: MergedToken,
|
||||
options: SubtitleAnnotationFilterOptions = {},
|
||||
): boolean {
|
||||
// No Yomitan dictionary entry backs this token (ぅ～ elongations, truncated
|
||||
// inflections) — it exists only to stay hoverable and must never receive
|
||||
// annotations or count in the N+1 math.
|
||||
if (token.isUnparsedRun === true) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const pos1Exclusions = resolvePos1Exclusions(options);
|
||||
const pos2Exclusions = resolvePos2Exclusions(options);
|
||||
if (!isContentTokenByPos(token, pos1Exclusions, pos2Exclusions)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isAuxiliaryStemGrammarTailToken(token)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isKanaOnlyNonIndependentNounHelperMerge(token)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isStandaloneAuxiliaryInflectionFragment(token)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isAuxiliaryOnlyHelperSpan(token)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isStandaloneSuruTeGrammarHelper(token)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isStandaloneGrammarParticle(token)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isSingleKanaSurfaceFragment(token)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isExcludedTrailingParticleMergedToken(token)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isLexicalKureruVerb(token)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return isExcludedByTerm(token);
|
||||
return (
|
||||
evaluateSubtitleAnnotationRules(createSubtitleAnnotationRuleContext(token, options)) ===
|
||||
'exclude'
|
||||
);
|
||||
}
|
||||
|
||||
export function stripSubtitleAnnotationMetadata(
|
||||
|
||||
@@ -105,7 +105,9 @@ function hasKanjiChar(text: string): boolean {
|
||||
code !== undefined &&
|
||||
((code >= 0x3400 && code <= 0x4dbf) ||
|
||||
(code >= 0x4e00 && code <= 0x9fff) ||
|
||||
(code >= 0xf900 && code <= 0xfaff))
|
||||
(code >= 0xf900 && code <= 0xfaff) ||
|
||||
(code >= 0x20000 && code <= 0x2fa1f) ||
|
||||
(code >= 0x30000 && code <= 0x323af))
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
@@ -115,7 +117,7 @@ function hasKanjiChar(text: string): boolean {
|
||||
|
||||
// MeCab's 非自立 tag suppresses kana grammar nouns (こと, もの, とき), but
|
||||
// Yomitan can segment kanji-bearing nouns (日, 方, 上, …) as real vocabulary.
|
||||
function isKanjiNonIndependentNounToken(
|
||||
export function isKanjiNonIndependentNounToken(
|
||||
token: MergedToken,
|
||||
pos1Exclusions: ReadonlySet<string>,
|
||||
): boolean {
|
||||
|
||||
Reference in New Issue
Block a user