sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-07-27 04:49:49 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix(overlay): keep frequency/JLPT highlight for kanji non-independent nouns (#150)

Browse Source
This commit is contained in:
Kyle Yasuda
2026-07-08 22:25:32 -07:00
committed by GitHub
parent 8b21a2bca8
commit 7f13aed50a
5 changed files with 112 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changes/kanji-non-independent-noun-annotations.md
+4
View File
@@ -0,0 +1,4 @@
type: fixed
area: overlay
- Kanji-bearing nouns that MeCab tags as non-independent (非自立) — e.g. 日 in いい日だったな, 点, 以外 — now keep frequency/JLPT highlighting and count toward vocabulary stats. Yomitan segments them as standalone vocabulary tokens, so the MeCab POS filter only suppresses kana grammar nouns (こと, もの, とき) it was meant for.
src/core/services/tokenizer.test.ts
+2 -2
View File
@@ -3486,8 +3486,8 @@ test('tokenizeSubtitle keeps known-word highlight for exact non-independent kanj
assert.equal(result.tokens?.[1]?.surface, '点');
assert.equal(result.tokens?.[1]?.isKnown, true);
assert.equal(result.tokens?.[1]?.isNPlusOneTarget, false);
assert.equal(result.tokens?.[1]?.frequencyRank, undefined);
assert.equal(result.tokens?.[1]?.jlptLevel, undefined);
assert.equal(result.tokens?.[1]?.frequencyRank, 1384);
assert.equal(result.tokens?.[1]?.jlptLevel, 'N3');
});
test('tokenizeSubtitle keeps mecab-tagged interjections tokenized while clearing annotation metadata', async () => {
src/core/services/tokenizer/annotation-stage.test.ts
+58 -10
View File
@@ -575,7 +575,9 @@ test('shouldExcludeTokenFromSubtitleAnnotations keeps lexical tokens outside exp
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
});
test('shouldExcludeTokenFromSubtitleAnnotations still excludes lexical non-independent kanji nouns from non-known annotations', () => {
test('shouldExcludeTokenFromSubtitleAnnotations keeps lexical non-independent kanji nouns', () => {
// Yomitan segments 以外/日/方 as standalone vocabulary tokens; MeCab's
// 非自立 tag must only suppress kana grammar nouns (こと, もの, とき).
const token = makeToken({
surface: '以外',
headword: '以外',
@@ -586,6 +588,21 @@ test('shouldExcludeTokenFromSubtitleAnnotations still excludes lexical non-indep
pos3: '副詞可能',
});
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
assert.equal(shouldExcludeTokenFromVocabularyPersistence(token), false);
});
test('shouldExcludeTokenFromSubtitleAnnotations still excludes kana non-independent nouns', () => {
const token = makeToken({
surface: 'こと',
headword: 'こと',
reading: 'コト',
partOfSpeech: PartOfSpeech.noun,
pos1: '名詞',
pos2: '非自立',
pos3: '一般',
});
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
assert.equal(shouldExcludeTokenFromVocabularyPersistence(token), true);
});
@@ -1421,8 +1438,8 @@ test('annotateTokens keeps known-word status for non-independent kanji noun toke
assert.equal(result[0]?.isKnown, true);
assert.equal(result[0]?.isNPlusOneTarget, false);
assert.equal(result[0]?.frequencyRank, undefined);
assert.equal(result[0]?.jlptLevel, undefined);
assert.equal(result[0]?.frequencyRank, 1384);
assert.equal(result[0]?.jlptLevel, 'N3');
});
test('annotateTokens keeps known-word status for lexical non-independent kanji nouns', () => {
@@ -1450,23 +1467,54 @@ test('annotateTokens keeps known-word status for lexical non-independent kanji n
);
assert.equal(result[0]?.isKnown, true);
assert.equal(result[0]?.frequencyRank, undefined);
assert.equal(result[0]?.frequencyRank, 437);
assert.equal(result[0]?.isNPlusOneTarget, false);
});
test('annotateTokens clears all annotations for non-independent kanji noun tokens under unified gate', () => {
test('annotateTokens keeps frequency for unknown non-independent kanji noun tokens', () => {
// 日 in いい日だったな: MeCab tags it 名詞/非自立 but Yomitan segments it as
// a standalone vocabulary token, so frequency highlighting must survive.
const tokens = [
makeToken({
surface: '者',
reading: 'もの',
headword: '者',
surface: '日',
reading: 'ひ',
headword: '日',
partOfSpeech: PartOfSpeech.noun,
pos1: '名詞',
pos2: '非自立',
pos3: '副詞可能',
startPos: 2,
endPos: 3,
frequencyRank: 718,
}),
];
const result = annotateTokens(
tokens,
makeDeps({
getJlptLevel: (text) => (text === '日' ? 'N4' : null),
}),
{ minSentenceWordsForNPlusOne: 1 },
);
assert.equal(result[0]?.isKnown, false);
assert.equal(result[0]?.frequencyRank, 718);
assert.equal(result[0]?.jlptLevel, 'N4');
});
test('annotateTokens still clears annotations for kana non-independent noun tokens', () => {
const tokens = [
makeToken({
surface: 'こと',
reading: 'こと',
headword: 'こと',
partOfSpeech: PartOfSpeech.other,
pos1: '名詞',
pos2: '非自立',
pos3: '一般',
startPos: 0,
endPos: 1,
frequencyRank: 475,
endPos: 2,
frequencyRank: 96,
}),
];
src/core/services/tokenizer/annotation-stage.ts
+4 -47
View File
@@ -10,6 +10,7 @@ import {
import { JlptLevel, MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types';
import { shouldIgnoreJlptByTerm, shouldIgnoreJlptForMecabPos1 } from '../jlpt-token-filter';
import {
isKanjiNonIndependentNounToken,
shouldExcludeTokenFromSubtitleAnnotations as sharedShouldExcludeTokenFromSubtitleAnnotations,
stripSubtitleAnnotationMetadata as sharedStripSubtitleAnnotationMetadata,
} from './subtitle-annotation-filter';
@@ -94,23 +95,6 @@ function normalizePos2Tag(pos2: string | undefined): string {
return typeof pos2 === 'string' ? pos2.trim() : '';
}
function hasKanjiChar(text: string): boolean {
for (const char of text) {
const code = char.codePointAt(0);
if (code === undefined) {
continue;
}
if (
(code >= 0x3400 && code <= 0x4dbf) ||
(code >= 0x4e00 && code <= 0x9fff) ||
(code >= 0xf900 && code <= 0xfaff)
) {
return true;
}
}
return false;
}
function isExcludedComponent(
pos1: string | undefined,
pos2: string | undefined,
@@ -283,34 +267,6 @@ function isFrequencyExcludedByPos(
);
}
function shouldKeepFrequencyForNonIndependentKanjiNoun(
token: MergedToken,
pos1Exclusions: ReadonlySet<string>,
): boolean {
if (pos1Exclusions.has('名詞')) {
return false;
}
const rank =
typeof token.frequencyRank === 'number' && Number.isFinite(token.frequencyRank)
? Math.max(1, Math.floor(token.frequencyRank))
: null;
if (rank === null) {
return false;
}
const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
const pos2Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos2));
if (pos1Parts.length !== 1 || pos2Parts.length !== 1) {
return false;
}
if (pos1Parts[0] !== '名詞' || pos2Parts[0] !== '非自立') {
return false;
}
return hasKanjiChar(token.surface) || hasKanjiChar(token.headword);
}
export function shouldExcludeTokenFromVocabularyPersistence(
token: MergedToken,
options: Pick<AnnotationStageOptions, 'pos1Exclusions' | 'pos2Exclusions'> = {},
@@ -320,7 +276,8 @@ export function shouldExcludeTokenFromVocabularyPersistence(
return (
sharedShouldExcludeTokenFromSubtitleAnnotations(token, { pos1Exclusions, pos2Exclusions }) ||
isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions)
(isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) &&
!isKanjiNonIndependentNounToken(token, pos1Exclusions))
);
}
@@ -721,7 +678,7 @@ function filterTokenFrequencyRank(
): number | undefined {
if (
isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) &&
!shouldKeepFrequencyForNonIndependentKanjiNoun(token, pos1Exclusions)
!isKanjiNonIndependentNounToken(token, pos1Exclusions)
) {
return undefined;
}
src/core/services/tokenizer/subtitle-annotation-filter.ts
+44 -1
View File
@@ -137,6 +137,46 @@ function resolvePos2Exclusions(options: SubtitleAnnotationFilterOptions = {}): R
return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
}
function hasKanjiChar(text: string): boolean {
for (const char of text) {
const code = char.codePointAt(0);
if (code === undefined) {
continue;
}
if (
(code >= 0x3400 && code <= 0x4dbf) ||
(code >= 0x4e00 && code <= 0x9fff) ||
(code >= 0xf900 && code <= 0xfaff)
) {
return true;
}
}
return false;
}
// Kanji-bearing non-independent nouns (日, 方, 上, …) are real vocabulary that
// Yomitan segments as standalone tokens; MeCab's 非自立 tag exists to suppress
// kana grammar nouns (こと, もの, とき) and must not hide these.
export function isKanjiNonIndependentNounToken(
token: MergedToken,
pos1Exclusions: ReadonlySet<string>,
): boolean {
if (pos1Exclusions.has('名詞')) {
return false;
}
const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
if (pos1Parts.length !== 1 || pos2Parts.length !== 1) {
return false;
}
if (pos1Parts[0] !== '名詞' || pos2Parts[0] !== '非自立') {
return false;
}
return hasKanjiChar(token.surface) || hasKanjiChar(token.headword);
}
function normalizeKana(text: string): string {
const raw = text.trim();
if (!raw) {
@@ -445,7 +485,10 @@ export function shouldExcludeTokenFromSubtitleAnnotations(
return true;
}
if (isExcludedByTagSet(normalizedPos2, pos2Exclusions)) {
if (
isExcludedByTagSet(normalizedPos2, pos2Exclusions) &&
!isKanjiNonIndependentNounToken(token, pos1Exclusions)
) {
return true;
}