diff --git a/changes/kana-pos-content-classification.md b/changes/kana-pos-content-classification.md new file mode 100644 index 00000000..149304c7 --- /dev/null +++ b/changes/kana-pos-content-classification.md @@ -0,0 +1,4 @@ +type: fixed +area: overlay + +- Kept kanji vocabulary tagged `名詞/非自立` eligible for N+1 highlighting, consistent with frequency, JLPT, and vocabulary persistence. diff --git a/docs-site/subtitle-annotations.md b/docs-site/subtitle-annotations.md index 9b90d567..6a6ee6fc 100644 --- a/docs-site/subtitle-annotations.md +++ b/docs-site/subtitle-annotations.md @@ -10,6 +10,8 @@ SubMiner's primary tokenizer is Yomitan itself - subtitle text is tokenized base Before any of those layers render, SubMiner strips annotation metadata from tokens that are usually just subtitle glue or annotation noise. Standalone particles, auxiliaries, adnominals, common explanatory endings like `んです` / `のだ`, merged trailing quote-particle forms like `...って`, auxiliary-stem grammar tails like `そうだ` (MeCab POS3 `助動詞語幹`), repeated kana interjections, and similar non-lexical helper tokens remain hoverable in the subtitle text, but they render as plain tokens without known-word, N+1, frequency, JLPT, or name-match annotation styling. +Kanji vocabulary that MeCab labels `名詞/非自立`, such as `日` or `以外`, remains content for every annotation layer. The `非自立` exclusion only suppresses kana grammar nouns such as `こと` and `もの`. + ## N+1 Word Highlighting N+1 highlighting identifies sentences where you know every word except one, making them ideal mining targets. When enabled, SubMiner builds a local cache of your known vocabulary from Anki and highlights tokens accordingly. @@ -24,16 +26,16 @@ N+1 highlighting identifies sentences where you know every word except one, maki **Key settings:** -| Option | Default | Description | -| ----------------------------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ankiConnect.knownWords.highlightEnabled` | `false` | Enable known-word cache lookups used by N+1 highlighting | -| `ankiConnect.knownWords.refreshMinutes` | `1440` | Minutes between Anki cache refreshes | -| `ankiConnect.knownWords.decks` | `{}` | Deck→fields map for known-word cache queries | -| `ankiConnect.knownWords.matchMode` | `"headword"` | `"headword"` (dictionary form) or `"surface"` (raw text) | -| `ankiConnect.nPlusOne.enabled` | `false` | Enable N+1 target highlighting | -| `ankiConnect.nPlusOne.minSentenceWords` | `3` | Minimum tokens in a sentence for N+1 to trigger | -| `subtitleStyle.nPlusOneColor` | `#c6a0f6` | Color for the single unknown target word | -| `subtitleStyle.knownWordColor` | `#a6da95` | Color for already-known tokens | +| Option | Default | Description | +| ----------------------------------------- | ------------ | -------------------------------------------------------- | +| `ankiConnect.knownWords.highlightEnabled` | `false` | Enable known-word cache lookups used by N+1 highlighting | +| `ankiConnect.knownWords.refreshMinutes` | `1440` | Minutes between Anki cache refreshes | +| `ankiConnect.knownWords.decks` | `{}` | Deck→fields map for known-word cache queries | +| `ankiConnect.knownWords.matchMode` | `"headword"` | `"headword"` (dictionary form) or `"surface"` (raw text) | +| `ankiConnect.nPlusOne.enabled` | `false` | Enable N+1 target highlighting | +| `ankiConnect.nPlusOne.minSentenceWords` | `3` | Minimum tokens in a sentence for N+1 to trigger | +| `subtitleStyle.nPlusOneColor` | `#c6a0f6` | Color for the single unknown target word | +| `subtitleStyle.knownWordColor` | `#a6da95` | Color for already-known tokens | Prefer expression/word fields for `ankiConnect.knownWords.decks`. Reading-only fields can mark unrelated homophones as known, so only include them when that tradeoff is intentional. diff --git a/src/core/services/tokenizer.ts b/src/core/services/tokenizer.ts index c9b450f6..34430d79 100644 --- a/src/core/services/tokenizer.ts +++ b/src/core/services/tokenizer.ts @@ -25,6 +25,7 @@ import { requestYomitanTermFrequencies, } from './tokenizer/yomitan-parser-runtime'; import type { YomitanTermFrequency } from './tokenizer/yomitan-parser-runtime'; +import { isKanaChar } from './tokenizer/token-classification'; const logger = createLogger('main:tokenizer'); @@ -322,20 +323,6 @@ function normalizeFrequencyLookupText(rawText: string): string { return rawText.trim().toLowerCase(); } -function isKanaChar(char: string): boolean { - const code = char.codePointAt(0); - if (code === undefined) { - return false; - } - return ( - (code >= 0x3041 && code <= 0x3096) || - (code >= 0x309b && code <= 0x309f) || - code === 0x30fc || - (code >= 0x30a0 && code <= 0x30fa) || - (code >= 0x30fd && code <= 0x30ff) - ); -} - function getTrailingKanaSuffix(surface: string): string { const chars = Array.from(surface); let splitIndex = chars.length; diff --git a/src/core/services/tokenizer/annotation-stage.test.ts b/src/core/services/tokenizer/annotation-stage.test.ts index d0d19b95..3313ca81 100644 --- a/src/core/services/tokenizer/annotation-stage.test.ts +++ b/src/core/services/tokenizer/annotation-stage.test.ts @@ -1563,6 +1563,7 @@ test('annotateTokens keeps frequency for unknown non-independent kanji noun toke ); assert.equal(result[0]?.isKnown, false); + assert.equal(result[0]?.isNPlusOneTarget, true); assert.equal(result[0]?.frequencyRank, 718); assert.equal(result[0]?.jlptLevel, 'N4'); }); diff --git a/src/core/services/tokenizer/annotation-stage.ts b/src/core/services/tokenizer/annotation-stage.ts index 9fd347a8..5a208531 100644 --- a/src/core/services/tokenizer/annotation-stage.ts +++ b/src/core/services/tokenizer/annotation-stage.ts @@ -10,14 +10,19 @@ import { import { JlptLevel, MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types'; import { shouldIgnoreJlptByTerm, shouldIgnoreJlptForMecabPos1 } from '../jlpt-token-filter'; import { - isKanjiNonIndependentNounToken, shouldExcludeTokenFromSubtitleAnnotations as sharedShouldExcludeTokenFromSubtitleAnnotations, stripSubtitleAnnotationMetadata as sharedStripSubtitleAnnotationMetadata, } from './subtitle-annotation-filter'; +import { + isKanaChar, + isKanaOnlyText, + isPosTagExcluded, + isTokenPos2Excluded, + normalizeKana, + normalizePosTag, + splitPosTag, +} from './token-classification'; -const KATAKANA_TO_HIRAGANA_OFFSET = 0x60; -const KATAKANA_CODEPOINT_START = 0x30a1; -const KATAKANA_CODEPOINT_END = 0x30f6; const JLPT_LEVEL_LOOKUP_CACHE_LIMIT = 2048; const jlptLevelLookupCaches = new WeakMap< @@ -55,30 +60,6 @@ function resolveKnownWordText( return matchMode === 'surface' ? surface : headword; } -function normalizePos1Tag(pos1: string | undefined): string { - return typeof pos1 === 'string' ? pos1.trim() : ''; -} - -function splitNormalizedTagParts(normalizedTag: string): string[] { - if (!normalizedTag) { - return []; - } - - return normalizedTag - .split('|') - .map((part) => part.trim()) - .filter((part) => part.length > 0); -} - -function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet): boolean { - const parts = splitNormalizedTagParts(normalizedTag); - if (parts.length === 0) { - return false; - } - - return parts.every((part) => exclusions.has(part)); -} - function resolvePos1Exclusions(options: AnnotationStageOptions): ReadonlySet { if (options.pos1Exclusions) { return options.pos1Exclusions; @@ -95,10 +76,6 @@ function resolvePos2Exclusions(options: AnnotationStageOptions): ReadonlySet, pos2Exclusions: ReadonlySet, ): boolean { - const pos1Parts = splitNormalizedTagParts(normalizedPos1); + const pos1Parts = splitPosTag(normalizedPos1); if (pos1Parts.length < 2) { return false; } - const pos2Parts = splitNormalizedTagParts(normalizedPos2); + const pos2Parts = splitPosTag(normalizedPos2); if (isExcludedComponent(pos1Parts[0], pos2Parts[0], pos1Exclusions, pos2Exclusions)) { return false; } @@ -144,8 +121,8 @@ function shouldAllowOrdinalPrefixNounFrequency(token: MergedToken): boolean { return false; } - const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1)); - const pos2Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos2)); + const pos1Parts = splitPosTag(token.pos1); + const pos2Parts = splitPosTag(token.pos2); return ( pos1Parts.length >= 2 && pos1Parts[0] === '接頭詞' && @@ -166,8 +143,8 @@ function shouldAllowHonorificPrefixNounFrequency(token: MergedToken): boolean { return false; } - const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1)); - const pos2Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos2)); + const pos1Parts = splitPosTag(token.pos1); + const pos2Parts = splitPosTag(token.pos2); return ( pos1Parts.length >= 2 && pos1Parts[0] === '接頭詞' && @@ -182,12 +159,12 @@ function shouldAllowDeterminerLedNounFrequency( pos1Exclusions: ReadonlySet, pos2Exclusions: ReadonlySet, ): boolean { - const pos1Parts = splitNormalizedTagParts(normalizedPos1); + const pos1Parts = splitPosTag(normalizedPos1); if (pos1Parts.length < 2 || pos1Parts[0] !== '連体詞') { return false; } - const pos2Parts = splitNormalizedTagParts(normalizedPos2); + const pos2Parts = splitPosTag(normalizedPos2); if (!isExcludedComponent(pos1Parts[0], pos2Parts[0], pos1Exclusions, pos2Exclusions)) { return false; } @@ -218,9 +195,9 @@ function isFrequencyExcludedByPos( return true; } - const normalizedPos1 = normalizePos1Tag(token.pos1); + const normalizedPos1 = normalizePosTag(token.pos1); const hasPos1 = normalizedPos1.length > 0; - const normalizedPos2 = normalizePos2Tag(token.pos2); + const normalizedPos2 = normalizePosTag(token.pos2); const hasPos2 = normalizedPos2.length > 0; const allowContentLedMergedToken = shouldAllowContentLedMergedTokenFrequency( normalizedPos1, @@ -238,7 +215,7 @@ function isFrequencyExcludedByPos( const allowHonorificPrefixNounToken = shouldAllowHonorificPrefixNounFrequency(token); if ( - isExcludedByTagSet(normalizedPos1, pos1Exclusions) && + isPosTagExcluded(normalizedPos1, pos1Exclusions) && !allowContentLedMergedToken && !allowDeterminerLedNounToken && !allowOrdinalPrefixNounToken && @@ -248,7 +225,7 @@ function isFrequencyExcludedByPos( } if ( - isExcludedByTagSet(normalizedPos2, pos2Exclusions) && + isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions) && !allowContentLedMergedToken && !allowDeterminerLedNounToken && !allowOrdinalPrefixNounToken && @@ -280,8 +257,7 @@ export function shouldExcludeTokenFromVocabularyPersistence( return ( sharedShouldExcludeTokenFromSubtitleAnnotations(token, { pos1Exclusions, pos2Exclusions }) || - (isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) && - !isKanjiNonIndependentNounToken(token, pos1Exclusions)) + isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) ); } @@ -332,45 +308,6 @@ function resolveJlptLookupText(token: MergedToken): string { return token.surface; } -function normalizeJlptTextForExclusion(text: string): string { - const raw = text.trim(); - if (!raw) { - return ''; - } - - let normalized = ''; - for (const char of raw) { - const code = char.codePointAt(0); - if (code === undefined) { - continue; - } - - if (code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END) { - normalized += String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET); - continue; - } - - normalized += char; - } - - return normalized; -} - -function isKanaChar(char: string): boolean { - const code = char.codePointAt(0); - if (code === undefined) { - return false; - } - - return ( - (code >= 0x3041 && code <= 0x3096) || - (code >= 0x309b && code <= 0x309f) || - code === 0x30fc || - (code >= 0x30a0 && code <= 0x30fa) || - (code >= 0x30fd && code <= 0x30ff) - ); -} - function isRepeatedKanaSfx(text: string): boolean { const normalized = text.trim(); if (!normalized) { @@ -406,7 +343,7 @@ function isRepeatedKanaSfx(text: string): boolean { } function isTrailingSmallTsuKanaSfx(text: string): boolean { - const normalized = normalizeJlptTextForExclusion(text); + const normalized = normalizeKana(text); if (!normalized) { return false; } @@ -424,7 +361,7 @@ function isTrailingSmallTsuKanaSfx(text: string): boolean { } function isReduplicatedKanaSfx(text: string): boolean { - const normalized = normalizeJlptTextForExclusion(text); + const normalized = normalizeKana(text); if (!normalized) { return false; } @@ -443,7 +380,7 @@ function isReduplicatedKanaSfx(text: string): boolean { } function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean { - const normalized = normalizeJlptTextForExclusion(text); + const normalized = normalizeKana(text); if (!normalized) { return false; } @@ -460,7 +397,7 @@ function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean { } function hasAdjacentKanaRepeat(text: string): boolean { - const normalized = normalizeJlptTextForExclusion(text); + const normalized = normalizeKana(text); if (!normalized) { return false; } @@ -490,7 +427,7 @@ function isLikelyFrequencyNoiseToken(token: MergedToken): boolean { continue; } - const normalizedCandidate = normalizeJlptTextForExclusion(trimmedCandidate); + const normalizedCandidate = normalizeKana(trimmedCandidate); if (!normalizedCandidate) { continue; } @@ -528,19 +465,6 @@ function isSingleKanaFrequencyNoiseToken(text: string | undefined): boolean { return chars.length === 1 && isKanaChar(chars[0]!); } -function isKanaOnlyText(text: string | undefined): boolean { - if (typeof text !== 'string') { - return false; - } - - const normalized = text.trim(); - if (!normalized) { - return false; - } - - return [...normalized].every(isKanaChar); -} - function isKanaOnlyMixedFunctionContentToken( token: MergedToken, pos1Exclusions: ReadonlySet, @@ -549,7 +473,7 @@ function isKanaOnlyMixedFunctionContentToken( return false; } - const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1)); + const pos1Parts = splitPosTag(token.pos1); const hasMixedFunctionContentParts = pos1Parts.length >= 2 && pos1Parts.some((part) => pos1Exclusions.has(part)) && @@ -558,8 +482,8 @@ function isKanaOnlyMixedFunctionContentToken( return false; } - const normalizedReading = normalizeJlptTextForExclusion(token.reading); - const normalizedHeadwordReading = normalizeJlptTextForExclusion(token.headwordReading ?? ''); + const normalizedReading = normalizeKana(token.reading); + const normalizedHeadwordReading = normalizeKana(token.headwordReading ?? ''); return ( !normalizedReading || !normalizedHeadwordReading || @@ -577,7 +501,7 @@ function isJlptEligibleToken(token: MergedToken): boolean { ); for (const candidate of candidates) { - const normalizedCandidate = normalizeJlptTextForExclusion(candidate); + const normalizedCandidate = normalizeKana(candidate); if (!normalizedCandidate) { continue; } @@ -612,8 +536,8 @@ export function stripSubtitleAnnotationMetadata( // at least as many characters as the surface, with the surface's kana appearing // in order within the reading. function isCompleteReadingForSurface(surface: string, reading: string): boolean { - const surfaceChars = [...normalizeJlptTextForExclusion(surface)]; - const readingChars = [...normalizeJlptTextForExclusion(reading)]; + const surfaceChars = [...normalizeKana(surface)]; + const readingChars = [...normalizeKana(reading)]; if (readingChars.length < surfaceChars.length) { return false; } @@ -697,10 +621,7 @@ function filterTokenFrequencyRank( pos1Exclusions: ReadonlySet, pos2Exclusions: ReadonlySet, ): number | undefined { - if ( - isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) && - !isKanjiNonIndependentNounToken(token, pos1Exclusions) - ) { + if (isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions)) { return undefined; } diff --git a/src/core/services/tokenizer/parser-selection-stage.ts b/src/core/services/tokenizer/parser-selection-stage.ts index 5d508de7..ae290c80 100644 --- a/src/core/services/tokenizer/parser-selection-stage.ts +++ b/src/core/services/tokenizer/parser-selection-stage.ts @@ -1,5 +1,6 @@ import { MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types'; import { isStandaloneGrammarEndingText } from './grammar-ending'; +import { isKanaChar, isKanaOnlyText } from './token-classification'; interface YomitanParseHeadword { term?: unknown; @@ -41,21 +42,6 @@ function resolveKnownWordText( return matchMode === 'surface' ? surface : headword; } -function isKanaChar(char: string): boolean { - const code = char.codePointAt(0); - if (code === undefined) { - return false; - } - - return ( - (code >= 0x3041 && code <= 0x3096) || - (code >= 0x309b && code <= 0x309f) || - code === 0x30fc || - (code >= 0x30a0 && code <= 0x30fa) || - (code >= 0x30fd && code <= 0x30ff) - ); -} - function isYomitanParseLine(value: unknown): value is YomitanParseLine { if (!Array.isArray(value)) { return false; @@ -138,10 +124,6 @@ function selectMergedHeadword( return firstHeadword; } -function isKanaOnlyText(text: string): boolean { - return text.length > 0 && Array.from(text).every((char) => isKanaChar(char)); -} - function isStandaloneGrammarEndingSegment(segment: YomitanParseSegment): boolean { const surface = segment.text?.trim() ?? ''; const headword = extractYomitanHeadword(segment).trim(); diff --git a/src/core/services/tokenizer/part-of-speech.ts b/src/core/services/tokenizer/part-of-speech.ts index b396e2a1..bab17742 100644 --- a/src/core/services/tokenizer/part-of-speech.ts +++ b/src/core/services/tokenizer/part-of-speech.ts @@ -1,8 +1,5 @@ import { PartOfSpeech } from '../../../types'; - -function normalizePosTag(value: string | null | undefined): string { - return typeof value === 'string' ? value.trim() : ''; -} +import { normalizePosTag, splitPosTag } from './token-classification'; export function isPartOfSpeechValue(value: unknown): value is PartOfSpeech { return typeof value === 'string' && Object.values(PartOfSpeech).includes(value as PartOfSpeech); @@ -35,10 +32,7 @@ export function deriveStoredPartOfSpeech(input: { partOfSpeech?: string | null; pos1?: string | null; }): PartOfSpeech { - const pos1Parts = normalizePosTag(input.pos1) - .split('|') - .map((part) => part.trim()) - .filter((part) => part.length > 0); + const pos1Parts = splitPosTag(input.pos1); if (pos1Parts.length > 0) { const derivedParts = [...new Set(pos1Parts.map((part) => mapMecabPos1ToPartOfSpeech(part)))]; diff --git a/src/core/services/tokenizer/subtitle-annotation-filter.ts b/src/core/services/tokenizer/subtitle-annotation-filter.ts index 7141ca9f..ab06afc2 100644 --- a/src/core/services/tokenizer/subtitle-annotation-filter.ts +++ b/src/core/services/tokenizer/subtitle-annotation-filter.ts @@ -6,13 +6,16 @@ import { DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG, resolveAnnotationPos2ExclusionSet, } from '../../../token-pos2-exclusions'; -import { MergedToken, PartOfSpeech } from '../../../types'; +import { MergedToken } from '../../../types'; import { shouldIgnoreJlptByTerm } from '../jlpt-token-filter'; import { isSubtitleGrammarEndingText } from './grammar-ending'; - -const KATAKANA_TO_HIRAGANA_OFFSET = 0x60; -const KATAKANA_CODEPOINT_START = 0x30a1; -const KATAKANA_CODEPOINT_END = 0x30f6; +import { + isContentTokenByPos, + isKanaChar, + isKanaOnlyText, + normalizeKana, + splitPosTag, +} from './token-classification'; const STANDALONE_GRAMMAR_PARTICLE_PHRASES = ['たって', 'だって'] as const; const STANDALONE_GRAMMAR_PARTICLE_PHRASES_SET: ReadonlySet = new Set( @@ -97,30 +100,6 @@ export interface SubtitleAnnotationFilterOptions { pos2Exclusions?: ReadonlySet; } -function normalizePosTag(pos: string | undefined): string { - return typeof pos === 'string' ? pos.trim() : ''; -} - -function splitNormalizedTagParts(normalizedTag: string): string[] { - if (!normalizedTag) { - return []; - } - - return normalizedTag - .split('|') - .map((part) => part.trim()) - .filter((part) => part.length > 0); -} - -function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet): boolean { - const parts = splitNormalizedTagParts(normalizedTag); - if (parts.length === 0) { - return false; - } - - return parts.every((part) => exclusions.has(part)); -} - function resolvePos1Exclusions(options: SubtitleAnnotationFilterOptions = {}): ReadonlySet { if (options.pos1Exclusions) { return options.pos1Exclusions; @@ -137,85 +116,6 @@ function resolvePos2Exclusions(options: SubtitleAnnotationFilterOptions = {}): R return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG); } -function hasKanjiChar(text: string): boolean { - for (const char of text) { - const code = char.codePointAt(0); - if (code === undefined) { - continue; - } - if ( - (code >= 0x3400 && code <= 0x4dbf) || - (code >= 0x4e00 && code <= 0x9fff) || - (code >= 0xf900 && code <= 0xfaff) - ) { - return true; - } - } - return false; -} - -// Kanji-bearing non-independent nouns (日, 方, 上, …) are real vocabulary that -// Yomitan segments as standalone tokens; MeCab's 非自立 tag exists to suppress -// kana grammar nouns (こと, もの, とき) and must not hide these. -export function isKanjiNonIndependentNounToken( - token: MergedToken, - pos1Exclusions: ReadonlySet, -): boolean { - if (pos1Exclusions.has('名詞')) { - return false; - } - - const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1)); - const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2)); - if (pos1Parts.length !== 1 || pos2Parts.length !== 1) { - return false; - } - if (pos1Parts[0] !== '名詞' || pos2Parts[0] !== '非自立') { - return false; - } - - return hasKanjiChar(token.surface) || hasKanjiChar(token.headword); -} - -function normalizeKana(text: string): string { - const raw = text.trim(); - if (!raw) { - return ''; - } - - let normalized = ''; - for (const char of raw) { - const code = char.codePointAt(0); - if (code === undefined) { - continue; - } - - if (code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END) { - normalized += String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET); - continue; - } - - normalized += char; - } - - return normalized; -} - -function isKanaChar(char: string): boolean { - const code = char.codePointAt(0); - if (code === undefined) { - return false; - } - - return ( - (code >= 0x3041 && code <= 0x3096) || - (code >= 0x309b && code <= 0x309f) || - code === 0x30fc || - (code >= 0x30a0 && code <= 0x30fa) || - (code >= 0x30fd && code <= 0x30ff) - ); -} - function isTrailingSmallTsuKanaSfx(text: string): boolean { const normalized = normalizeKana(text); if (!normalized) { @@ -286,7 +186,7 @@ function isExcludedTrailingParticleMergedToken(token: MergedToken): boolean { return false; } - const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1)); + const pos1Parts = splitPosTag(token.pos1); if (pos1Parts.length < 2) { return false; } @@ -300,7 +200,7 @@ function isExcludedTrailingParticleMergedToken(token: MergedToken): boolean { } function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean { - const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1)); + const pos1Parts = splitPosTag(token.pos1); if ( pos1Parts.length === 0 || !pos1Parts.every((part) => AUXILIARY_STEM_GRAMMAR_TAIL_POS1.has(part)) @@ -308,7 +208,7 @@ function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean { return false; } - const pos3Parts = splitNormalizedTagParts(normalizePosTag(token.pos3)); + const pos3Parts = splitPosTag(token.pos3); return pos3Parts.includes('助動詞語幹'); } @@ -324,12 +224,12 @@ function isKanaOnlyNonIndependentNounHelperMerge(token: MergedToken): boolean { return false; } - const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1)); + const pos1Parts = splitPosTag(token.pos1); if (pos1Parts.length < 2 || pos1Parts[0] !== '名詞') { return false; } - const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2)); + const pos2Parts = splitPosTag(token.pos2); if (pos2Parts[0] !== '非自立') { return false; } @@ -337,16 +237,11 @@ function isKanaOnlyNonIndependentNounHelperMerge(token: MergedToken): boolean { return pos1Parts.slice(1).every((part) => NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1.has(part)); } -function isKanaOnlyText(text: string): boolean { - const normalized = normalizeKana(text); - return normalized.length > 0 && [...normalized].every(isKanaChar); -} - function isLexicalKureruVerb(token: MergedToken): boolean { const normalizedSurface = normalizeKana(token.surface); const normalizedHeadword = normalizeKana(token.headword); - const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1)); - const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2)); + const pos1Parts = splitPosTag(token.pos1); + const pos2Parts = splitPosTag(token.pos2); return ( normalizedSurface === 'くれ' && normalizedHeadword === 'くれる' && @@ -363,7 +258,7 @@ function isStandaloneAuxiliaryInflectionFragment(token: MergedToken): boolean { return false; } - const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1)); + const pos1Parts = splitPosTag(token.pos1); if (pos1Parts.length === 0) { return false; } @@ -372,7 +267,7 @@ function isStandaloneAuxiliaryInflectionFragment(token: MergedToken): boolean { return true; } - const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2)); + const pos2Parts = splitPosTag(token.pos2); return ( pos1Parts[0] === '動詞' && pos2Parts[0] === '接尾' && @@ -387,7 +282,7 @@ function isAuxiliaryOnlyHelperSpan(token: MergedToken): boolean { return false; } - const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1)); + const pos1Parts = splitPosTag(token.pos1); if ( pos1Parts.length === 0 || !pos1Parts.every((part) => AUXILIARY_HELPER_SPAN_POS1.has(part)) || @@ -397,7 +292,7 @@ function isAuxiliaryOnlyHelperSpan(token: MergedToken): boolean { return false; } - const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2)); + const pos2Parts = splitPosTag(token.pos2); return !pos2Parts.some((part) => LEXICAL_VERB_POS2.has(part)); } @@ -408,7 +303,7 @@ function isStandaloneSuruTeGrammarHelper(token: MergedToken): boolean { return false; } - const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1)); + const pos1Parts = splitPosTag(token.pos1); return ( isKanaOnlyText(normalizedSurface) && (pos1Parts.length === 0 || pos1Parts.includes('動詞')) ); @@ -483,29 +378,7 @@ export function shouldExcludeTokenFromSubtitleAnnotations( const pos1Exclusions = resolvePos1Exclusions(options); const pos2Exclusions = resolvePos2Exclusions(options); - const normalizedPos1 = normalizePosTag(token.pos1); - const normalizedPos2 = normalizePosTag(token.pos2); - const hasPos1 = normalizedPos1.length > 0; - const hasPos2 = normalizedPos2.length > 0; - - if (isExcludedByTagSet(normalizedPos1, pos1Exclusions)) { - return true; - } - - if ( - isExcludedByTagSet(normalizedPos2, pos2Exclusions) && - !isKanjiNonIndependentNounToken(token, pos1Exclusions) - ) { - return true; - } - - if ( - !hasPos1 && - !hasPos2 && - (token.partOfSpeech === PartOfSpeech.particle || - token.partOfSpeech === PartOfSpeech.bound_auxiliary || - token.partOfSpeech === PartOfSpeech.symbol) - ) { + if (!isContentTokenByPos(token, pos1Exclusions, pos2Exclusions)) { return true; } diff --git a/src/core/services/tokenizer/token-classification.test.ts b/src/core/services/tokenizer/token-classification.test.ts new file mode 100644 index 00000000..6d5c699b --- /dev/null +++ b/src/core/services/tokenizer/token-classification.test.ts @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { MergedToken, PartOfSpeech } from '../../../types'; +import { + isContentTokenByPos, + isKanaCandidateIgnorableChar, + isKanaCandidateText, + isKanaChar, + isKanaOnlyText, + isTokenPos2Excluded, +} from './token-classification'; + +const POS1_EXCLUSIONS = new Set(['助詞']); +const POS2_EXCLUSIONS = new Set(['非自立']); + +function makeNoun(surface: string): MergedToken { + return { + surface, + reading: surface, + headword: surface, + startPos: 0, + endPos: surface.length, + partOfSpeech: PartOfSpeech.noun, + pos1: '名詞', + pos2: '非自立', + isMerged: false, + isKnown: false, + isNPlusOneTarget: false, + }; +} + +test('kana classification excludes the katakana-hiragana double hyphen', () => { + assert.equal(isKanaChar('゠'), false); + assert.equal(isKanaOnlyText('゠'), false); +}); + +test('POS classification keeps kanji non-independent nouns as content', () => { + const token = makeNoun('日'); + + assert.equal(isTokenPos2Excluded(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), false); + assert.equal(isContentTokenByPos(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), true); +}); + +test('POS classification excludes kana non-independent nouns', () => { + const token = makeNoun('こと'); + + assert.equal(isTokenPos2Excluded(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), true); + assert.equal(isContentTokenByPos(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), false); +}); + +test('kana candidate classification allows punctuation around kana only', () => { + assert.equal(isKanaCandidateIgnorableChar('！'), true); + assert.equal(isKanaCandidateIgnorableChar('猫'), false); + assert.equal(isKanaCandidateText('「かな！？」'), true); + assert.equal(isKanaCandidateText('「！？」'), false); + assert.equal(isKanaCandidateText('かな猫'), false); +}); diff --git a/src/core/services/tokenizer/token-classification.ts b/src/core/services/tokenizer/token-classification.ts new file mode 100644 index 00000000..259b5e58 --- /dev/null +++ b/src/core/services/tokenizer/token-classification.ts @@ -0,0 +1,167 @@ +import { MergedToken, PartOfSpeech } from '../../../types'; + +const KATAKANA_TO_HIRAGANA_OFFSET = 0x60; +const KATAKANA_CODEPOINT_START = 0x30a1; +const KATAKANA_CODEPOINT_END = 0x30f6; + +export function normalizeKana(text: string): string { + const raw = text.trim(); + if (!raw) { + return ''; + } + + let normalized = ''; + for (const char of raw) { + const code = char.codePointAt(0); + if (code === undefined) { + continue; + } + + normalized += + code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END + ? String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET) + : char; + } + + return normalized; +} + +export function isKanaChar(char: string): boolean { + const code = char.codePointAt(0); + if (code === undefined) { + return false; + } + + return ( + (code >= 0x3041 && code <= 0x3096) || + (code >= 0x309b && code <= 0x309f) || + code === 0x30fc || + (code >= 0x30a1 && code <= 0x30fa) || + (code >= 0x30fd && code <= 0x30ff) + ); +} + +export function isKanaOnlyText(text: string | null | undefined): boolean { + if (typeof text !== 'string') { + return false; + } + + const normalized = normalizeKana(text); + return normalized.length > 0 && [...normalized].every(isKanaChar); +} + +export function isKanaCandidateIgnorableChar(char: string): boolean { + return /^[\s.,!?;:()[\]{}"'`、。！？…‥・「」『』（）［］｛｝〈〉《》【】―-]$/u.test(char); +} + +export function isKanaCandidateText(text: string): boolean { + const normalized = text.trim(); + if (normalized.length === 0) { + return false; + } + + let hasKana = false; + for (const char of normalized) { + if (isKanaChar(char)) { + hasKana = true; + continue; + } + if (!isKanaCandidateIgnorableChar(char)) { + return false; + } + } + + return hasKana; +} + +export function normalizePosTag(value: string | null | undefined): string { + return typeof value === 'string' ? value.trim() : ''; +} + +export function splitPosTag(value: string | null | undefined): string[] { + const normalized = normalizePosTag(value); + if (!normalized) { + return []; + } + + return normalized + .split('|') + .map((part) => part.trim()) + .filter((part) => part.length > 0); +} + +export function isPosTagExcluded( + value: string | null | undefined, + exclusions: ReadonlySet, +): boolean { + const parts = splitPosTag(value); + return parts.length > 0 && parts.every((part) => exclusions.has(part)); +} + +function hasKanjiChar(text: string): boolean { + for (const char of text) { + const code = char.codePointAt(0); + if ( + code !== undefined && + ((code >= 0x3400 && code <= 0x4dbf) || + (code >= 0x4e00 && code <= 0x9fff) || + (code >= 0xf900 && code <= 0xfaff)) + ) { + return true; + } + } + return false; +} + +// MeCab's 非自立 tag suppresses kana grammar nouns (こと, もの, とき), but +// Yomitan can segment kanji-bearing nouns (日, 方, 上, …) as real vocabulary. +function isKanjiNonIndependentNounToken( + token: MergedToken, + pos1Exclusions: ReadonlySet, +): boolean { + if (pos1Exclusions.has('名詞')) { + return false; + } + + const pos1Parts = splitPosTag(token.pos1); + const pos2Parts = splitPosTag(token.pos2); + return ( + pos1Parts.length === 1 && + pos1Parts[0] === '名詞' && + pos2Parts.length === 1 && + pos2Parts[0] === '非自立' && + (hasKanjiChar(token.surface) || hasKanjiChar(token.headword)) + ); +} + +export function isTokenPos2Excluded( + token: MergedToken, + pos1Exclusions: ReadonlySet, + pos2Exclusions: ReadonlySet, +): boolean { + return ( + isPosTagExcluded(token.pos2, pos2Exclusions) && + !isKanjiNonIndependentNounToken(token, pos1Exclusions) + ); +} + +export function isContentTokenByPos( + token: MergedToken, + pos1Exclusions: ReadonlySet, + pos2Exclusions: ReadonlySet, +): boolean { + if ( + isPosTagExcluded(token.pos1, pos1Exclusions) || + isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions) + ) { + return false; + } + + if (splitPosTag(token.pos1).length > 0 || splitPosTag(token.pos2).length > 0) { + return true; + } + + return ![PartOfSpeech.particle, PartOfSpeech.bound_auxiliary, PartOfSpeech.symbol].includes( + token.partOfSpeech, + ); +} diff --git a/src/token-merger.ts b/src/token-merger.ts index dd69866c..1e0ecf47 100644 --- a/src/token-merger.ts +++ b/src/token-merger.ts @@ -20,6 +20,7 @@ import { PartOfSpeech, Token, MergedToken } from './types'; import { DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG } from './token-pos1-exclusions'; import { DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG } from './token-pos2-exclusions'; import { shouldExcludeTokenFromSubtitleAnnotations } from './core/services/tokenizer/subtitle-annotation-filter'; +import { isKanaCandidateText } from './core/services/tokenizer/token-classification'; export function isNoun(tok: Token): boolean { return tok.partOfSpeech === PartOfSpeech.noun; @@ -261,67 +262,6 @@ const SENTENCE_BOUNDARY_SURFACES = new Set(['。', '？', '！', '?', '!', '…' const N_PLUS_ONE_IGNORED_POS1 = new Set(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG.defaults); const N_PLUS_ONE_IGNORED_POS2 = new Set(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG.defaults); -function normalizePos1Tag(pos1: string | undefined): string { - return typeof pos1 === 'string' ? pos1.trim() : ''; -} - -function normalizePos2Tag(pos2: string | undefined): string { - return typeof pos2 === 'string' ? pos2.trim() : ''; -} - -function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet): boolean { - if (!normalizedTag) { - return false; - } - const parts = normalizedTag - .split('|') - .map((part) => part.trim()) - .filter((part) => part.length > 0); - if (parts.length === 0) { - return false; - } - return parts.every((part) => exclusions.has(part)); -} - -function isKanaChar(char: string): boolean { - const code = char.codePointAt(0); - if (code === undefined) { - return false; - } - - return ( - (code >= 0x3041 && code <= 0x3096) || - (code >= 0x309b && code <= 0x309f) || - code === 0x30fc || - (code >= 0x30a0 && code <= 0x30fa) || - (code >= 0x30fd && code <= 0x30ff) - ); -} - -function isKanaCandidateIgnorableChar(char: string): boolean { - return /^[\s.,!?;:()[\]{}"'`、。！？…‥・「」『』（）［］｛｝〈〉《》【】―-]$/u.test(char); -} - -function isKanaOnlyText(text: string): boolean { - const normalized = text.trim(); - if (normalized.length === 0) { - return false; - } - - let hasKana = false; - for (const char of normalized) { - if (isKanaChar(char)) { - hasKana = true; - continue; - } - if (!isKanaCandidateIgnorableChar(char)) { - return false; - } - } - - return hasKana; -} - function normalizeSourceTextForTokenOffsets(sourceText: string | undefined): string | undefined { return typeof sourceText === 'string' ? sourceText.replace(/\r?\n/g, ' ').trim() : undefined; } @@ -334,7 +274,7 @@ export function isNPlusOneCandidateToken( if (token.isKnown) { return false; } - if (isKanaOnlyText(token.surface)) { + if (isKanaCandidateText(token.surface)) { return false; } return isNPlusOneWordCountToken(token, pos1Exclusions, pos2Exclusions); @@ -349,28 +289,6 @@ function isNPlusOneWordCountToken( return false; } - const normalizedPos1 = normalizePos1Tag(token.pos1); - const hasPos1 = normalizedPos1.length > 0; - if (isExcludedByTagSet(normalizedPos1, pos1Exclusions)) { - return false; - } - - const normalizedPos2 = normalizePos2Tag(token.pos2); - const hasPos2 = normalizedPos2.length > 0; - if (isExcludedByTagSet(normalizedPos2, pos2Exclusions)) { - return false; - } - - if ( - !hasPos1 && - !hasPos2 && - (token.partOfSpeech === PartOfSpeech.particle || - token.partOfSpeech === PartOfSpeech.bound_auxiliary || - token.partOfSpeech === PartOfSpeech.symbol) - ) { - return false; - } - if (token.partOfSpeech === PartOfSpeech.noun && token.pos2 === '固有名詞') { return false; }