diff --git a/changes/kanji-non-independent-noun-annotations.md b/changes/kanji-non-independent-noun-annotations.md new file mode 100644 index 00000000..f6b7e695 --- /dev/null +++ b/changes/kanji-non-independent-noun-annotations.md @@ -0,0 +1,4 @@ +type: fixed +area: overlay + +- Kanji-bearing nouns that MeCab tags as non-independent (非自立) — e.g. 日 in いい日だったな, 点, 以外 — now keep frequency/JLPT highlighting and count toward vocabulary stats. Yomitan segments them as standalone vocabulary tokens, so the MeCab POS filter only suppresses kana grammar nouns (こと, もの, とき) it was meant for. diff --git a/src/core/services/tokenizer.test.ts b/src/core/services/tokenizer.test.ts index 7e01a54c..007a88e3 100644 --- a/src/core/services/tokenizer.test.ts +++ b/src/core/services/tokenizer.test.ts @@ -3486,8 +3486,8 @@ test('tokenizeSubtitle keeps known-word highlight for exact non-independent kanj assert.equal(result.tokens?.[1]?.surface, '点'); assert.equal(result.tokens?.[1]?.isKnown, true); assert.equal(result.tokens?.[1]?.isNPlusOneTarget, false); - assert.equal(result.tokens?.[1]?.frequencyRank, undefined); - assert.equal(result.tokens?.[1]?.jlptLevel, undefined); + assert.equal(result.tokens?.[1]?.frequencyRank, 1384); + assert.equal(result.tokens?.[1]?.jlptLevel, 'N3'); }); test('tokenizeSubtitle keeps mecab-tagged interjections tokenized while clearing annotation metadata', async () => { diff --git a/src/core/services/tokenizer/annotation-stage.test.ts b/src/core/services/tokenizer/annotation-stage.test.ts index ccd58bd3..b18fd259 100644 --- a/src/core/services/tokenizer/annotation-stage.test.ts +++ b/src/core/services/tokenizer/annotation-stage.test.ts @@ -575,7 +575,9 @@ test('shouldExcludeTokenFromSubtitleAnnotations keeps lexical tokens outside exp assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false); }); -test('shouldExcludeTokenFromSubtitleAnnotations still excludes lexical non-independent kanji nouns from non-known annotations', () => { +test('shouldExcludeTokenFromSubtitleAnnotations keeps lexical non-independent kanji nouns', () => { + // Yomitan segments 以外/日/方 as standalone vocabulary tokens; MeCab's + // 非自立 tag must only suppress kana grammar nouns (こと, もの, とき). const token = makeToken({ surface: '以外', headword: '以外', @@ -586,6 +588,21 @@ test('shouldExcludeTokenFromSubtitleAnnotations still excludes lexical non-indep pos3: '副詞可能', }); + assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false); + assert.equal(shouldExcludeTokenFromVocabularyPersistence(token), false); +}); + +test('shouldExcludeTokenFromSubtitleAnnotations still excludes kana non-independent nouns', () => { + const token = makeToken({ + surface: 'こと', + headword: 'こと', + reading: 'コト', + partOfSpeech: PartOfSpeech.noun, + pos1: '名詞', + pos2: '非自立', + pos3: '一般', + }); + assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true); assert.equal(shouldExcludeTokenFromVocabularyPersistence(token), true); }); @@ -1421,8 +1438,8 @@ test('annotateTokens keeps known-word status for non-independent kanji noun toke assert.equal(result[0]?.isKnown, true); assert.equal(result[0]?.isNPlusOneTarget, false); - assert.equal(result[0]?.frequencyRank, undefined); - assert.equal(result[0]?.jlptLevel, undefined); + assert.equal(result[0]?.frequencyRank, 1384); + assert.equal(result[0]?.jlptLevel, 'N3'); }); test('annotateTokens keeps known-word status for lexical non-independent kanji nouns', () => { @@ -1450,23 +1467,54 @@ test('annotateTokens keeps known-word status for lexical non-independent kanji n ); assert.equal(result[0]?.isKnown, true); - assert.equal(result[0]?.frequencyRank, undefined); + assert.equal(result[0]?.frequencyRank, 437); assert.equal(result[0]?.isNPlusOneTarget, false); }); -test('annotateTokens clears all annotations for non-independent kanji noun tokens under unified gate', () => { +test('annotateTokens keeps frequency for unknown non-independent kanji noun tokens', () => { + // 日 in いい日だったな: MeCab tags it 名詞/非自立 but Yomitan segments it as + // a standalone vocabulary token, so frequency highlighting must survive. const tokens = [ makeToken({ - surface: '者', - reading: 'もの', - headword: '者', + surface: '日', + reading: 'ひ', + headword: '日', + partOfSpeech: PartOfSpeech.noun, + pos1: '名詞', + pos2: '非自立', + pos3: '副詞可能', + startPos: 2, + endPos: 3, + frequencyRank: 718, + }), + ]; + + const result = annotateTokens( + tokens, + makeDeps({ + getJlptLevel: (text) => (text === '日' ? 'N4' : null), + }), + { minSentenceWordsForNPlusOne: 1 }, + ); + + assert.equal(result[0]?.isKnown, false); + assert.equal(result[0]?.frequencyRank, 718); + assert.equal(result[0]?.jlptLevel, 'N4'); +}); + +test('annotateTokens still clears annotations for kana non-independent noun tokens', () => { + const tokens = [ + makeToken({ + surface: 'こと', + reading: 'こと', + headword: 'こと', partOfSpeech: PartOfSpeech.other, pos1: '名詞', pos2: '非自立', pos3: '一般', startPos: 0, - endPos: 1, - frequencyRank: 475, + endPos: 2, + frequencyRank: 96, }), ]; diff --git a/src/core/services/tokenizer/annotation-stage.ts b/src/core/services/tokenizer/annotation-stage.ts index 7369e9f2..d72c4ac0 100644 --- a/src/core/services/tokenizer/annotation-stage.ts +++ b/src/core/services/tokenizer/annotation-stage.ts @@ -10,6 +10,7 @@ import { import { JlptLevel, MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types'; import { shouldIgnoreJlptByTerm, shouldIgnoreJlptForMecabPos1 } from '../jlpt-token-filter'; import { + isKanjiNonIndependentNounToken, shouldExcludeTokenFromSubtitleAnnotations as sharedShouldExcludeTokenFromSubtitleAnnotations, stripSubtitleAnnotationMetadata as sharedStripSubtitleAnnotationMetadata, } from './subtitle-annotation-filter'; @@ -94,23 +95,6 @@ function normalizePos2Tag(pos2: string | undefined): string { return typeof pos2 === 'string' ? pos2.trim() : ''; } -function hasKanjiChar(text: string): boolean { - for (const char of text) { - const code = char.codePointAt(0); - if (code === undefined) { - continue; - } - if ( - (code >= 0x3400 && code <= 0x4dbf) || - (code >= 0x4e00 && code <= 0x9fff) || - (code >= 0xf900 && code <= 0xfaff) - ) { - return true; - } - } - return false; -} - function isExcludedComponent( pos1: string | undefined, pos2: string | undefined, @@ -283,34 +267,6 @@ function isFrequencyExcludedByPos( ); } -function shouldKeepFrequencyForNonIndependentKanjiNoun( - token: MergedToken, - pos1Exclusions: ReadonlySet, -): boolean { - if (pos1Exclusions.has('名詞')) { - return false; - } - - const rank = - typeof token.frequencyRank === 'number' && Number.isFinite(token.frequencyRank) - ? Math.max(1, Math.floor(token.frequencyRank)) - : null; - if (rank === null) { - return false; - } - - const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1)); - const pos2Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos2)); - if (pos1Parts.length !== 1 || pos2Parts.length !== 1) { - return false; - } - if (pos1Parts[0] !== '名詞' || pos2Parts[0] !== '非自立') { - return false; - } - - return hasKanjiChar(token.surface) || hasKanjiChar(token.headword); -} - export function shouldExcludeTokenFromVocabularyPersistence( token: MergedToken, options: Pick = {}, @@ -320,7 +276,8 @@ export function shouldExcludeTokenFromVocabularyPersistence( return ( sharedShouldExcludeTokenFromSubtitleAnnotations(token, { pos1Exclusions, pos2Exclusions }) || - isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) + (isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) && + !isKanjiNonIndependentNounToken(token, pos1Exclusions)) ); } @@ -721,7 +678,7 @@ function filterTokenFrequencyRank( ): number | undefined { if ( isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) && - !shouldKeepFrequencyForNonIndependentKanjiNoun(token, pos1Exclusions) + !isKanjiNonIndependentNounToken(token, pos1Exclusions) ) { return undefined; } diff --git a/src/core/services/tokenizer/subtitle-annotation-filter.ts b/src/core/services/tokenizer/subtitle-annotation-filter.ts index 153ab17e..67adb0b6 100644 --- a/src/core/services/tokenizer/subtitle-annotation-filter.ts +++ b/src/core/services/tokenizer/subtitle-annotation-filter.ts @@ -137,6 +137,46 @@ function resolvePos2Exclusions(options: SubtitleAnnotationFilterOptions = {}): R return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG); } +function hasKanjiChar(text: string): boolean { + for (const char of text) { + const code = char.codePointAt(0); + if (code === undefined) { + continue; + } + if ( + (code >= 0x3400 && code <= 0x4dbf) || + (code >= 0x4e00 && code <= 0x9fff) || + (code >= 0xf900 && code <= 0xfaff) + ) { + return true; + } + } + return false; +} + +// Kanji-bearing non-independent nouns (日, 方, 上, …) are real vocabulary that +// Yomitan segments as standalone tokens; MeCab's 非自立 tag exists to suppress +// kana grammar nouns (こと, もの, とき) and must not hide these. +export function isKanjiNonIndependentNounToken( + token: MergedToken, + pos1Exclusions: ReadonlySet, +): boolean { + if (pos1Exclusions.has('名詞')) { + return false; + } + + const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1)); + const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2)); + if (pos1Parts.length !== 1 || pos2Parts.length !== 1) { + return false; + } + if (pos1Parts[0] !== '名詞' || pos2Parts[0] !== '非自立') { + return false; + } + + return hasKanjiChar(token.surface) || hasKanjiChar(token.headword); +} + function normalizeKana(text: string): string { const raw = text.trim(); if (!raw) { @@ -445,7 +485,10 @@ export function shouldExcludeTokenFromSubtitleAnnotations( return true; } - if (isExcludedByTagSet(normalizedPos2, pos2Exclusions)) { + if ( + isExcludedByTagSet(normalizedPos2, pos2Exclusions) && + !isKanjiNonIndependentNounToken(token, pos1Exclusions) + ) { return true; }