mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-08 19:21:32 -07:00
feat(stats): add v1 immersion stats dashboard (#19)
This commit is contained in:
GitHub
@@ -1,7 +1,12 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { MergedToken, PartOfSpeech } from '../../../types';
|
||||
import { annotateTokens, AnnotationStageDeps } from './annotation-stage';
|
||||
import {
|
||||
annotateTokens,
|
||||
AnnotationStageDeps,
|
||||
shouldExcludeTokenFromSubtitleAnnotations,
|
||||
stripSubtitleAnnotationMetadata,
|
||||
} from './annotation-stage';
|
||||
|
||||
function makeToken(overrides: Partial<MergedToken> = {}): MergedToken {
|
||||
return {
|
||||
@@ -50,6 +55,29 @@ test('annotateTokens known-word match mode uses headword vs surface', () => {
|
||||
assert.equal(surfaceResult[0]?.isKnown, false);
|
||||
});
|
||||
|
||||
test('annotateTokens falls back to reading for known-word matches when headword lookup misses', () => {
|
||||
const tokens = [
|
||||
makeToken({
|
||||
surface: '大体',
|
||||
headword: '大体',
|
||||
reading: 'だいたい',
|
||||
frequencyRank: 1895,
|
||||
}),
|
||||
];
|
||||
|
||||
const result = annotateTokens(
|
||||
tokens,
|
||||
makeDeps({
|
||||
isKnownWord: (text) => text === 'だいたい',
|
||||
getJlptLevel: (text) => (text === '大体' ? 'N4' : null),
|
||||
}),
|
||||
);
|
||||
|
||||
assert.equal(result[0]?.isKnown, true);
|
||||
assert.equal(result[0]?.jlptLevel, 'N4');
|
||||
assert.equal(result[0]?.frequencyRank, 1895);
|
||||
});
|
||||
|
||||
test('annotateTokens excludes frequency for particle/bound_auxiliary and pos1 exclusions', () => {
|
||||
const tokens = [
|
||||
makeToken({
|
||||
@@ -150,6 +178,278 @@ test('annotateTokens handles JLPT disabled and eligibility exclusion paths', ()
|
||||
assert.equal(excludedLookupCalls, 0);
|
||||
});
|
||||
|
||||
test('shouldExcludeTokenFromSubtitleAnnotations excludes explanatory ending variants', () => {
|
||||
const tokens = [
|
||||
makeToken({
|
||||
surface: 'んです',
|
||||
headword: 'ん',
|
||||
reading: 'ンデス',
|
||||
pos1: '名詞|助動詞',
|
||||
pos2: '非自立',
|
||||
}),
|
||||
makeToken({
|
||||
surface: 'のだ',
|
||||
headword: 'の',
|
||||
reading: 'ノダ',
|
||||
pos1: '名詞|助動詞',
|
||||
pos2: '非自立',
|
||||
}),
|
||||
makeToken({
|
||||
surface: 'んだ',
|
||||
headword: 'ん',
|
||||
reading: 'ンダ',
|
||||
pos1: '名詞|助動詞',
|
||||
pos2: '非自立',
|
||||
}),
|
||||
makeToken({
|
||||
surface: 'のです',
|
||||
headword: 'の',
|
||||
reading: 'ノデス',
|
||||
pos1: '名詞|助動詞',
|
||||
pos2: '非自立',
|
||||
}),
|
||||
makeToken({
|
||||
surface: 'なんです',
|
||||
headword: 'だ',
|
||||
reading: 'ナンデス',
|
||||
pos1: '助動詞|名詞|助動詞',
|
||||
pos2: '|非自立',
|
||||
}),
|
||||
makeToken({
|
||||
surface: 'んでした',
|
||||
headword: 'ん',
|
||||
reading: 'ンデシタ',
|
||||
pos1: '助動詞|助動詞|助動詞',
|
||||
}),
|
||||
makeToken({
|
||||
surface: 'のでは',
|
||||
headword: 'の',
|
||||
reading: 'ノデハ',
|
||||
pos1: '助詞|接続詞',
|
||||
}),
|
||||
];
|
||||
|
||||
for (const token of tokens) {
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true, token.surface);
|
||||
}
|
||||
});
|
||||
|
||||
test('shouldExcludeTokenFromSubtitleAnnotations excludes explanatory pondering endings', () => {
|
||||
const token = makeToken({
|
||||
surface: 'のかな',
|
||||
headword: 'の',
|
||||
reading: 'ノカナ',
|
||||
pos1: '名詞|助動詞',
|
||||
pos2: '非自立',
|
||||
});
|
||||
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
|
||||
});
|
||||
|
||||
test('shouldExcludeTokenFromSubtitleAnnotations excludes auxiliary-stem そうだ grammar tails', () => {
|
||||
const token = makeToken({
|
||||
surface: 'そうだ',
|
||||
headword: 'そうだ',
|
||||
reading: 'ソウダ',
|
||||
pos1: '名詞|助動詞',
|
||||
pos2: '特殊',
|
||||
pos3: '助動詞語幹',
|
||||
});
|
||||
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
|
||||
});
|
||||
|
||||
test('shouldExcludeTokenFromSubtitleAnnotations keeps lexical tokens outside explanatory ending family', () => {
|
||||
const token = makeToken({
|
||||
surface: '問題',
|
||||
headword: '問題',
|
||||
reading: 'モンダイ',
|
||||
partOfSpeech: PartOfSpeech.noun,
|
||||
pos1: '名詞',
|
||||
pos2: '一般',
|
||||
});
|
||||
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
|
||||
});
|
||||
|
||||
test('shouldExcludeTokenFromSubtitleAnnotations excludes standalone particles auxiliaries and adnominals', () => {
|
||||
const tokens = [
|
||||
makeToken({
|
||||
surface: 'は',
|
||||
headword: 'は',
|
||||
reading: 'ハ',
|
||||
partOfSpeech: PartOfSpeech.particle,
|
||||
pos1: '助詞',
|
||||
}),
|
||||
makeToken({
|
||||
surface: 'です',
|
||||
headword: 'です',
|
||||
reading: 'デス',
|
||||
partOfSpeech: PartOfSpeech.bound_auxiliary,
|
||||
pos1: '助動詞',
|
||||
}),
|
||||
makeToken({
|
||||
surface: 'この',
|
||||
headword: 'この',
|
||||
reading: 'コノ',
|
||||
partOfSpeech: PartOfSpeech.other,
|
||||
pos1: '連体詞',
|
||||
}),
|
||||
];
|
||||
|
||||
for (const token of tokens) {
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true, token.surface);
|
||||
}
|
||||
});
|
||||
|
||||
test('shouldExcludeTokenFromSubtitleAnnotations keeps mixed content tokens with trailing helpers', () => {
|
||||
const token = makeToken({
|
||||
surface: '行きます',
|
||||
headword: '行く',
|
||||
reading: 'イキマス',
|
||||
partOfSpeech: PartOfSpeech.verb,
|
||||
pos1: '動詞|助動詞',
|
||||
pos2: '自立',
|
||||
});
|
||||
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
|
||||
});
|
||||
|
||||
test('shouldExcludeTokenFromSubtitleAnnotations excludes merged lexical tokens with trailing quote particles', () => {
|
||||
const token = makeToken({
|
||||
surface: 'どうしてもって',
|
||||
headword: 'どうしても',
|
||||
reading: 'ドウシテモッテ',
|
||||
partOfSpeech: PartOfSpeech.other,
|
||||
pos1: '副詞|助詞',
|
||||
pos2: '一般|格助詞',
|
||||
});
|
||||
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
|
||||
});
|
||||
|
||||
test('shouldExcludeTokenFromSubtitleAnnotations excludes kana-only demonstrative helper merges', () => {
|
||||
const token = makeToken({
|
||||
surface: 'これで',
|
||||
headword: 'これ',
|
||||
reading: 'コレデ',
|
||||
partOfSpeech: PartOfSpeech.noun,
|
||||
pos1: '名詞|助詞',
|
||||
pos2: '代名詞|格助詞',
|
||||
});
|
||||
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
|
||||
});
|
||||
|
||||
test('stripSubtitleAnnotationMetadata keeps token hover data while clearing annotation fields', () => {
|
||||
const token = makeToken({
|
||||
surface: 'は',
|
||||
headword: 'は',
|
||||
reading: 'ハ',
|
||||
partOfSpeech: PartOfSpeech.particle,
|
||||
pos1: '助詞',
|
||||
isKnown: true,
|
||||
isNPlusOneTarget: true,
|
||||
isNameMatch: true,
|
||||
jlptLevel: 'N5',
|
||||
frequencyRank: 12,
|
||||
});
|
||||
|
||||
assert.deepEqual(stripSubtitleAnnotationMetadata(token), {
|
||||
...token,
|
||||
isKnown: false,
|
||||
isNPlusOneTarget: false,
|
||||
isNameMatch: false,
|
||||
jlptLevel: undefined,
|
||||
frequencyRank: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
test('stripSubtitleAnnotationMetadata leaves content tokens unchanged', () => {
|
||||
const token = makeToken({
|
||||
surface: '猫',
|
||||
headword: '猫',
|
||||
reading: 'ネコ',
|
||||
partOfSpeech: PartOfSpeech.noun,
|
||||
pos1: '名詞',
|
||||
isKnown: true,
|
||||
jlptLevel: 'N5',
|
||||
frequencyRank: 42,
|
||||
});
|
||||
|
||||
assert.strictEqual(stripSubtitleAnnotationMetadata(token), token);
|
||||
});
|
||||
|
||||
test('annotateTokens prioritizes name matches over n+1, frequency, and JLPT when enabled', () => {
|
||||
let jlptLookupCalls = 0;
|
||||
const tokens = [
|
||||
makeToken({
|
||||
surface: 'オリヴィア',
|
||||
reading: 'オリヴィア',
|
||||
headword: 'オリヴィア',
|
||||
isNameMatch: true,
|
||||
frequencyRank: 42,
|
||||
startPos: 0,
|
||||
endPos: 5,
|
||||
}),
|
||||
];
|
||||
|
||||
const result = annotateTokens(
|
||||
tokens,
|
||||
makeDeps({
|
||||
getJlptLevel: () => {
|
||||
jlptLookupCalls += 1;
|
||||
return 'N2';
|
||||
},
|
||||
}),
|
||||
{
|
||||
nameMatchEnabled: true,
|
||||
minSentenceWordsForNPlusOne: 1,
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(result[0]?.isNameMatch, true);
|
||||
assert.equal(result[0]?.isNPlusOneTarget, false);
|
||||
assert.equal(result[0]?.frequencyRank, undefined);
|
||||
assert.equal(result[0]?.jlptLevel, undefined);
|
||||
assert.equal(jlptLookupCalls, 0);
|
||||
});
|
||||
|
||||
test('annotateTokens keeps other annotations for name matches when name highlighting is disabled', () => {
|
||||
let jlptLookupCalls = 0;
|
||||
const tokens = [
|
||||
makeToken({
|
||||
surface: 'オリヴィア',
|
||||
reading: 'オリヴィア',
|
||||
headword: 'オリヴィア',
|
||||
isNameMatch: true,
|
||||
frequencyRank: 42,
|
||||
startPos: 0,
|
||||
endPos: 5,
|
||||
}),
|
||||
];
|
||||
|
||||
const result = annotateTokens(
|
||||
tokens,
|
||||
makeDeps({
|
||||
getJlptLevel: () => {
|
||||
jlptLookupCalls += 1;
|
||||
return 'N2';
|
||||
},
|
||||
}),
|
||||
{
|
||||
nameMatchEnabled: false,
|
||||
minSentenceWordsForNPlusOne: 1,
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(result[0]?.isNameMatch, true);
|
||||
assert.equal(result[0]?.isNPlusOneTarget, true);
|
||||
assert.equal(result[0]?.frequencyRank, 42);
|
||||
assert.equal(result[0]?.jlptLevel, 'N2');
|
||||
assert.equal(jlptLookupCalls, 1);
|
||||
});
|
||||
|
||||
test('annotateTokens N+1 handoff marks expected target when threshold is satisfied', () => {
|
||||
const tokens = [
|
||||
makeToken({ surface: '私', headword: '私', startPos: 0, endPos: 1 }),
|
||||
@@ -206,8 +506,8 @@ test('annotateTokens N+1 minimum sentence words counts only eligible word tokens
|
||||
);
|
||||
|
||||
assert.equal(result[0]?.isKnown, false);
|
||||
assert.equal(result[1]?.isKnown, true);
|
||||
assert.equal(result[2]?.isKnown, true);
|
||||
assert.equal(result[1]?.isKnown, false);
|
||||
assert.equal(result[2]?.isKnown, false);
|
||||
assert.equal(result[0]?.isNPlusOneTarget, false);
|
||||
});
|
||||
|
||||
@@ -293,6 +593,32 @@ test('annotateTokens excludes default non-independent pos2 from frequency and N+
|
||||
assert.equal(result[0]?.isNPlusOneTarget, false);
|
||||
});
|
||||
|
||||
test('annotateTokens clears all annotations for non-independent kanji noun tokens under unified gate', () => {
|
||||
const tokens = [
|
||||
makeToken({
|
||||
surface: '者',
|
||||
reading: 'もの',
|
||||
headword: '者',
|
||||
partOfSpeech: PartOfSpeech.other,
|
||||
pos1: '名詞',
|
||||
pos2: '非自立',
|
||||
pos3: '一般',
|
||||
startPos: 0,
|
||||
endPos: 1,
|
||||
frequencyRank: 475,
|
||||
}),
|
||||
];
|
||||
|
||||
const result = annotateTokens(tokens, makeDeps(), {
|
||||
minSentenceWordsForNPlusOne: 1,
|
||||
});
|
||||
|
||||
assert.equal(result[0]?.isKnown, false);
|
||||
assert.equal(result[0]?.isNPlusOneTarget, false);
|
||||
assert.equal(result[0]?.frequencyRank, undefined);
|
||||
assert.equal(result[0]?.jlptLevel, undefined);
|
||||
});
|
||||
|
||||
test('annotateTokens excludes likely kana SFX tokens from frequency when POS tags are missing', () => {
|
||||
const tokens = [
|
||||
makeToken({
|
||||
@@ -444,3 +770,33 @@ test('annotateTokens excludes composite tokens when all component pos tags are e
|
||||
assert.equal(result[0]?.frequencyRank, undefined);
|
||||
assert.equal(result[0]?.isNPlusOneTarget, false);
|
||||
});
|
||||
|
||||
test('annotateTokens applies one shared exclusion gate across known N+1 frequency and JLPT', () => {
|
||||
const tokens = [
|
||||
makeToken({
|
||||
surface: 'これで',
|
||||
headword: 'これ',
|
||||
reading: 'コレデ',
|
||||
partOfSpeech: PartOfSpeech.noun,
|
||||
pos1: '名詞|助詞',
|
||||
pos2: '代名詞|格助詞',
|
||||
startPos: 0,
|
||||
endPos: 3,
|
||||
frequencyRank: 9,
|
||||
}),
|
||||
];
|
||||
|
||||
const result = annotateTokens(
|
||||
tokens,
|
||||
makeDeps({
|
||||
isKnownWord: (text) => text === 'これ',
|
||||
getJlptLevel: (text) => (text === 'これ' ? 'N5' : null),
|
||||
}),
|
||||
{ minSentenceWordsForNPlusOne: 1 },
|
||||
);
|
||||
|
||||
assert.equal(result[0]?.isKnown, false);
|
||||
assert.equal(result[0]?.isNPlusOneTarget, false);
|
||||
assert.equal(result[0]?.frequencyRank, undefined);
|
||||
assert.equal(result[0]?.jlptLevel, undefined);
|
||||
});
|
||||
|
||||
@@ -9,11 +9,65 @@ import {
|
||||
} from '../../../token-pos2-exclusions';
|
||||
import { JlptLevel, MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types';
|
||||
import { shouldIgnoreJlptByTerm, shouldIgnoreJlptForMecabPos1 } from '../jlpt-token-filter';
|
||||
import {
|
||||
shouldExcludeTokenFromSubtitleAnnotations as sharedShouldExcludeTokenFromSubtitleAnnotations,
|
||||
stripSubtitleAnnotationMetadata as sharedStripSubtitleAnnotationMetadata,
|
||||
} from './subtitle-annotation-filter';
|
||||
|
||||
const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
|
||||
const KATAKANA_CODEPOINT_START = 0x30a1;
|
||||
const KATAKANA_CODEPOINT_END = 0x30f6;
|
||||
const JLPT_LEVEL_LOOKUP_CACHE_LIMIT = 2048;
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
|
||||
'ああ',
|
||||
'ええ',
|
||||
'うう',
|
||||
'おお',
|
||||
'はあ',
|
||||
'はは',
|
||||
'へえ',
|
||||
'ふう',
|
||||
'ほう',
|
||||
]);
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_PREFIXES = ['ん', 'の', 'なん', 'なの'];
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_CORES = [
|
||||
'だ',
|
||||
'です',
|
||||
'でした',
|
||||
'だった',
|
||||
'では',
|
||||
'じゃ',
|
||||
'でしょう',
|
||||
'だろう',
|
||||
] as const;
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_TRAILING_PARTICLES = [
|
||||
'',
|
||||
'か',
|
||||
'ね',
|
||||
'よ',
|
||||
'な',
|
||||
'よね',
|
||||
'かな',
|
||||
'かね',
|
||||
] as const;
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDINGS = new Set(
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_PREFIXES.flatMap((prefix) =>
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_CORES.flatMap((core) =>
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_TRAILING_PARTICLES.map(
|
||||
(particle) => `${prefix}${core}${particle}`,
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES = new Set([
|
||||
'って',
|
||||
'ってよ',
|
||||
'ってね',
|
||||
'ってな',
|
||||
'ってさ',
|
||||
'ってか',
|
||||
'ってば',
|
||||
]);
|
||||
|
||||
const jlptLevelLookupCaches = new WeakMap<
|
||||
(text: string) => JlptLevel | null,
|
||||
@@ -28,6 +82,7 @@ export interface AnnotationStageDeps {
|
||||
|
||||
export interface AnnotationStageOptions {
|
||||
nPlusOneEnabled?: boolean;
|
||||
nameMatchEnabled?: boolean;
|
||||
jlptEnabled?: boolean;
|
||||
frequencyEnabled?: boolean;
|
||||
minSentenceWordsForNPlusOne?: number;
|
||||
@@ -43,33 +98,27 @@ function resolveKnownWordText(
|
||||
return matchMode === 'surface' ? surface : headword;
|
||||
}
|
||||
|
||||
function applyKnownWordMarking(
|
||||
tokens: MergedToken[],
|
||||
isKnownWord: (text: string) => boolean,
|
||||
knownWordMatchMode: NPlusOneMatchMode,
|
||||
): MergedToken[] {
|
||||
return tokens.map((token) => {
|
||||
const matchText = resolveKnownWordText(token.surface, token.headword, knownWordMatchMode);
|
||||
|
||||
return {
|
||||
...token,
|
||||
isKnown: token.isKnown || (matchText ? isKnownWord(matchText) : false),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function normalizePos1Tag(pos1: string | undefined): string {
|
||||
return typeof pos1 === 'string' ? pos1.trim() : '';
|
||||
}
|
||||
|
||||
function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<string>): boolean {
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_POS1 = new Set(['感動詞']);
|
||||
const SUBTITLE_ANNOTATION_GRAMMAR_ONLY_POS1 = new Set(['助詞', '助動詞', '連体詞']);
|
||||
const AUXILIARY_STEM_GRAMMAR_TAIL_POS1 = new Set(['名詞', '助動詞', '助詞']);
|
||||
|
||||
function splitNormalizedTagParts(normalizedTag: string): string[] {
|
||||
if (!normalizedTag) {
|
||||
return false;
|
||||
return [];
|
||||
}
|
||||
const parts = normalizedTag
|
||||
|
||||
return normalizedTag
|
||||
.split('|')
|
||||
.map((part) => part.trim())
|
||||
.filter((part) => part.length > 0);
|
||||
}
|
||||
|
||||
function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<string>): boolean {
|
||||
const parts = splitNormalizedTagParts(normalizedTag);
|
||||
if (parts.length === 0) {
|
||||
return false;
|
||||
}
|
||||
@@ -78,6 +127,50 @@ function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<strin
|
||||
return parts.some((part) => exclusions.has(part));
|
||||
}
|
||||
|
||||
function isExcludedFromSubtitleAnnotationsByPos1(normalizedPos1: string): boolean {
|
||||
const parts = splitNormalizedTagParts(normalizedPos1);
|
||||
if (parts.some((part) => SUBTITLE_ANNOTATION_EXCLUDED_POS1.has(part))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return parts.length > 0 && parts.every((part) => SUBTITLE_ANNOTATION_GRAMMAR_ONLY_POS1.has(part));
|
||||
}
|
||||
|
||||
function isExcludedTrailingParticleMergedToken(token: MergedToken): boolean {
|
||||
const normalizedSurface = normalizeJlptTextForExclusion(token.surface);
|
||||
const normalizedHeadword = normalizeJlptTextForExclusion(token.headword);
|
||||
if (!normalizedSurface || !normalizedHeadword || !normalizedSurface.startsWith(normalizedHeadword)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const suffix = normalizedSurface.slice(normalizedHeadword.length);
|
||||
if (!SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES.has(suffix)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
|
||||
if (pos1Parts.length < 2) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const [leadingPos1, ...trailingPos1] = pos1Parts;
|
||||
if (!leadingPos1 || SUBTITLE_ANNOTATION_GRAMMAR_ONLY_POS1.has(leadingPos1)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return trailingPos1.length > 0 && trailingPos1.every((part) => part === '助詞');
|
||||
}
|
||||
|
||||
function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean {
|
||||
const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
|
||||
if (pos1Parts.length === 0 || !pos1Parts.every((part) => AUXILIARY_STEM_GRAMMAR_TAIL_POS1.has(part))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos3Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos3));
|
||||
return pos3Parts.includes('助動詞語幹');
|
||||
}
|
||||
|
||||
function resolvePos1Exclusions(options: AnnotationStageOptions): ReadonlySet<string> {
|
||||
if (options.pos1Exclusions) {
|
||||
return options.pos1Exclusions;
|
||||
@@ -98,6 +191,61 @@ function normalizePos2Tag(pos2: string | undefined): string {
|
||||
return typeof pos2 === 'string' ? pos2.trim() : '';
|
||||
}
|
||||
|
||||
function hasKanjiChar(text: string): boolean {
|
||||
for (const char of text) {
|
||||
const code = char.codePointAt(0);
|
||||
if (code === undefined) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
(code >= 0x3400 && code <= 0x4dbf) ||
|
||||
(code >= 0x4e00 && code <= 0x9fff) ||
|
||||
(code >= 0xf900 && code <= 0xfaff)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isExcludedComponent(
|
||||
pos1: string | undefined,
|
||||
pos2: string | undefined,
|
||||
pos1Exclusions: ReadonlySet<string>,
|
||||
pos2Exclusions: ReadonlySet<string>,
|
||||
): boolean {
|
||||
return (
|
||||
(typeof pos1 === 'string' && pos1Exclusions.has(pos1)) ||
|
||||
(typeof pos2 === 'string' && pos2Exclusions.has(pos2))
|
||||
);
|
||||
}
|
||||
|
||||
function shouldAllowContentLedMergedTokenFrequency(
|
||||
normalizedPos1: string,
|
||||
normalizedPos2: string,
|
||||
pos1Exclusions: ReadonlySet<string>,
|
||||
pos2Exclusions: ReadonlySet<string>,
|
||||
): boolean {
|
||||
const pos1Parts = splitNormalizedTagParts(normalizedPos1);
|
||||
if (pos1Parts.length < 2) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos2Parts = splitNormalizedTagParts(normalizedPos2);
|
||||
if (isExcludedComponent(pos1Parts[0], pos2Parts[0], pos1Exclusions, pos2Exclusions)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const componentCount = Math.max(pos1Parts.length, pos2Parts.length);
|
||||
for (let index = 1; index < componentCount; index += 1) {
|
||||
if (!isExcludedComponent(pos1Parts[index], pos2Parts[index], pos1Exclusions, pos2Exclusions)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
function isFrequencyExcludedByPos(
|
||||
token: MergedToken,
|
||||
pos1Exclusions: ReadonlySet<string>,
|
||||
@@ -109,13 +257,20 @@ function isFrequencyExcludedByPos(
|
||||
|
||||
const normalizedPos1 = normalizePos1Tag(token.pos1);
|
||||
const hasPos1 = normalizedPos1.length > 0;
|
||||
if (isExcludedByTagSet(normalizedPos1, pos1Exclusions)) {
|
||||
const normalizedPos2 = normalizePos2Tag(token.pos2);
|
||||
const hasPos2 = normalizedPos2.length > 0;
|
||||
const allowContentLedMergedToken = shouldAllowContentLedMergedTokenFrequency(
|
||||
normalizedPos1,
|
||||
normalizedPos2,
|
||||
pos1Exclusions,
|
||||
pos2Exclusions,
|
||||
);
|
||||
|
||||
if (isExcludedByTagSet(normalizedPos1, pos1Exclusions) && !allowContentLedMergedToken) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const normalizedPos2 = normalizePos2Tag(token.pos2);
|
||||
const hasPos2 = normalizedPos2.length > 0;
|
||||
if (isExcludedByTagSet(normalizedPos2, pos2Exclusions)) {
|
||||
if (isExcludedByTagSet(normalizedPos2, pos2Exclusions) && !allowContentLedMergedToken) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -133,26 +288,43 @@ function isFrequencyExcludedByPos(
|
||||
);
|
||||
}
|
||||
|
||||
function applyFrequencyMarking(
|
||||
tokens: MergedToken[],
|
||||
function shouldKeepFrequencyForNonIndependentKanjiNoun(
|
||||
token: MergedToken,
|
||||
pos1Exclusions: ReadonlySet<string>,
|
||||
pos2Exclusions: ReadonlySet<string>,
|
||||
): MergedToken[] {
|
||||
return tokens.map((token) => {
|
||||
if (isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions)) {
|
||||
return { ...token, frequencyRank: undefined };
|
||||
}
|
||||
): boolean {
|
||||
if (pos1Exclusions.has('名詞')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (typeof token.frequencyRank === 'number' && Number.isFinite(token.frequencyRank)) {
|
||||
const rank = Math.max(1, Math.floor(token.frequencyRank));
|
||||
return { ...token, frequencyRank: rank };
|
||||
}
|
||||
const rank =
|
||||
typeof token.frequencyRank === 'number' && Number.isFinite(token.frequencyRank)
|
||||
? Math.max(1, Math.floor(token.frequencyRank))
|
||||
: null;
|
||||
if (rank === null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return {
|
||||
...token,
|
||||
frequencyRank: undefined,
|
||||
};
|
||||
});
|
||||
const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
|
||||
const pos2Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos2));
|
||||
if (pos1Parts.length !== 1 || pos2Parts.length !== 1) {
|
||||
return false;
|
||||
}
|
||||
if (pos1Parts[0] !== '名詞' || pos2Parts[0] !== '非自立') {
|
||||
return false;
|
||||
}
|
||||
|
||||
return hasKanjiChar(token.surface) || hasKanjiChar(token.headword);
|
||||
}
|
||||
|
||||
export function shouldExcludeTokenFromVocabularyPersistence(
|
||||
token: MergedToken,
|
||||
options: Pick<AnnotationStageOptions, 'pos1Exclusions' | 'pos2Exclusions'> = {},
|
||||
): boolean {
|
||||
return isFrequencyExcludedByPos(
|
||||
token,
|
||||
resolvePos1Exclusions(options),
|
||||
resolvePos2Exclusions(options),
|
||||
);
|
||||
}
|
||||
|
||||
function getCachedJlptLevel(
|
||||
@@ -312,6 +484,23 @@ function isReduplicatedKanaSfx(text: string): boolean {
|
||||
return chars.slice(0, half).join('') === chars.slice(half).join('');
|
||||
}
|
||||
|
||||
function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
|
||||
const normalized = normalizeJlptTextForExclusion(text);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isReduplicatedKanaSfx(normalized)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (normalized.length <= 1 || !normalized.endsWith('と')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return isReduplicatedKanaSfx(normalized.slice(0, -1));
|
||||
}
|
||||
|
||||
function hasAdjacentKanaRepeat(text: string): boolean {
|
||||
const normalized = normalizeJlptTextForExclusion(text);
|
||||
if (!normalized) {
|
||||
@@ -386,12 +575,7 @@ function isJlptEligibleToken(token: MergedToken): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
const candidates = [
|
||||
resolveJlptLookupText(token),
|
||||
token.surface,
|
||||
token.reading,
|
||||
token.headword,
|
||||
].filter(
|
||||
const candidates = [resolveJlptLookupText(token), token.surface, token.headword].filter(
|
||||
(candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0,
|
||||
);
|
||||
|
||||
@@ -414,24 +598,110 @@ function isJlptEligibleToken(token: MergedToken): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
function applyJlptMarking(
|
||||
tokens: MergedToken[],
|
||||
getJlptLevel: (text: string) => JlptLevel | null,
|
||||
): MergedToken[] {
|
||||
return tokens.map((token) => {
|
||||
if (!isJlptEligibleToken(token)) {
|
||||
return { ...token, jlptLevel: undefined };
|
||||
function isExcludedFromSubtitleAnnotationsByTerm(token: MergedToken): boolean {
|
||||
const candidates = [token.surface, token.reading, resolveJlptLookupText(token)].filter(
|
||||
(candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0,
|
||||
);
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const trimmedCandidate = candidate.trim();
|
||||
if (!trimmedCandidate) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const primaryLevel = getCachedJlptLevel(resolveJlptLookupText(token), getJlptLevel);
|
||||
const fallbackLevel =
|
||||
primaryLevel === null ? getCachedJlptLevel(token.surface, getJlptLevel) : null;
|
||||
const normalizedCandidate = normalizeJlptTextForExclusion(trimmedCandidate);
|
||||
if (!normalizedCandidate) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return {
|
||||
...token,
|
||||
jlptLevel: primaryLevel ?? fallbackLevel ?? token.jlptLevel,
|
||||
};
|
||||
});
|
||||
if (
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(trimmedCandidate) ||
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(normalizedCandidate) ||
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDINGS.has(trimmedCandidate) ||
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDINGS.has(normalizedCandidate)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
isTrailingSmallTsuKanaSfx(trimmedCandidate) ||
|
||||
isTrailingSmallTsuKanaSfx(normalizedCandidate) ||
|
||||
isReduplicatedKanaSfxWithOptionalTrailingTo(trimmedCandidate) ||
|
||||
isReduplicatedKanaSfxWithOptionalTrailingTo(normalizedCandidate)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export function shouldExcludeTokenFromSubtitleAnnotations(token: MergedToken): boolean {
|
||||
return sharedShouldExcludeTokenFromSubtitleAnnotations(token);
|
||||
}
|
||||
|
||||
export function stripSubtitleAnnotationMetadata(token: MergedToken): MergedToken {
|
||||
return sharedStripSubtitleAnnotationMetadata(token);
|
||||
}
|
||||
|
||||
function computeTokenKnownStatus(
|
||||
token: MergedToken,
|
||||
isKnownWord: (text: string) => boolean,
|
||||
knownWordMatchMode: NPlusOneMatchMode,
|
||||
): boolean {
|
||||
const matchText = resolveKnownWordText(token.surface, token.headword, knownWordMatchMode);
|
||||
if (token.isKnown || (matchText ? isKnownWord(matchText) : false)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const normalizedReading = token.reading.trim();
|
||||
if (!normalizedReading) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return normalizedReading !== matchText.trim() && isKnownWord(normalizedReading);
|
||||
}
|
||||
|
||||
function filterTokenFrequencyRank(
|
||||
token: MergedToken,
|
||||
pos1Exclusions: ReadonlySet<string>,
|
||||
pos2Exclusions: ReadonlySet<string>,
|
||||
): number | undefined {
|
||||
if (
|
||||
isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) &&
|
||||
!shouldKeepFrequencyForNonIndependentKanjiNoun(token, pos1Exclusions)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (typeof token.frequencyRank === 'number' && Number.isFinite(token.frequencyRank)) {
|
||||
return Math.max(1, Math.floor(token.frequencyRank));
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function computeTokenJlptLevel(
|
||||
token: MergedToken,
|
||||
getJlptLevel: (text: string) => JlptLevel | null,
|
||||
): JlptLevel | undefined {
|
||||
if (!isJlptEligibleToken(token)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const primaryLevel = getCachedJlptLevel(resolveJlptLookupText(token), getJlptLevel);
|
||||
const fallbackLevel =
|
||||
primaryLevel === null ? getCachedJlptLevel(token.surface, getJlptLevel) : null;
|
||||
|
||||
const level = primaryLevel ?? fallbackLevel ?? token.jlptLevel;
|
||||
return level ?? undefined;
|
||||
}
|
||||
|
||||
function hasPrioritizedNameMatch(
|
||||
token: MergedToken,
|
||||
options: Pick<AnnotationStageOptions, 'nameMatchEnabled'>,
|
||||
): boolean {
|
||||
return options.nameMatchEnabled !== false && token.isNameMatch === true;
|
||||
}
|
||||
|
||||
export function annotateTokens(
|
||||
@@ -442,36 +712,50 @@ export function annotateTokens(
|
||||
const pos1Exclusions = resolvePos1Exclusions(options);
|
||||
const pos2Exclusions = resolvePos2Exclusions(options);
|
||||
const nPlusOneEnabled = options.nPlusOneEnabled !== false;
|
||||
const knownMarkedTokens = nPlusOneEnabled
|
||||
? applyKnownWordMarking(tokens, deps.isKnownWord, deps.knownWordMatchMode)
|
||||
: tokens.map((token) => ({
|
||||
...token,
|
||||
isKnown: false,
|
||||
isNPlusOneTarget: false,
|
||||
}));
|
||||
|
||||
const nameMatchEnabled = options.nameMatchEnabled !== false;
|
||||
const frequencyEnabled = options.frequencyEnabled !== false;
|
||||
const frequencyMarkedTokens = frequencyEnabled
|
||||
? applyFrequencyMarking(knownMarkedTokens, pos1Exclusions, pos2Exclusions)
|
||||
: knownMarkedTokens.map((token) => ({
|
||||
...token,
|
||||
frequencyRank: undefined,
|
||||
}));
|
||||
|
||||
const jlptEnabled = options.jlptEnabled !== false;
|
||||
const jlptMarkedTokens = jlptEnabled
|
||||
? applyJlptMarking(frequencyMarkedTokens, deps.getJlptLevel)
|
||||
: frequencyMarkedTokens.map((token) => ({
|
||||
...token,
|
||||
jlptLevel: undefined,
|
||||
}));
|
||||
|
||||
// Single pass: compute known word status, frequency filtering, and JLPT level together
|
||||
const annotated = tokens.map((token) => {
|
||||
if (
|
||||
sharedShouldExcludeTokenFromSubtitleAnnotations(token, {
|
||||
pos1Exclusions,
|
||||
pos2Exclusions,
|
||||
})
|
||||
) {
|
||||
return sharedStripSubtitleAnnotationMetadata(token, {
|
||||
pos1Exclusions,
|
||||
pos2Exclusions,
|
||||
});
|
||||
}
|
||||
|
||||
const prioritizedNameMatch = nameMatchEnabled && token.isNameMatch === true;
|
||||
const isKnown = nPlusOneEnabled
|
||||
? computeTokenKnownStatus(token, deps.isKnownWord, deps.knownWordMatchMode)
|
||||
: false;
|
||||
|
||||
const frequencyRank =
|
||||
frequencyEnabled && !prioritizedNameMatch
|
||||
? filterTokenFrequencyRank(token, pos1Exclusions, pos2Exclusions)
|
||||
: undefined;
|
||||
|
||||
const jlptLevel =
|
||||
jlptEnabled && !prioritizedNameMatch
|
||||
? computeTokenJlptLevel(token, deps.getJlptLevel)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
...token,
|
||||
isKnown,
|
||||
isNPlusOneTarget: nPlusOneEnabled && !prioritizedNameMatch ? token.isNPlusOneTarget : false,
|
||||
frequencyRank,
|
||||
jlptLevel,
|
||||
};
|
||||
});
|
||||
|
||||
if (!nPlusOneEnabled) {
|
||||
return jlptMarkedTokens.map((token) => ({
|
||||
...token,
|
||||
isKnown: false,
|
||||
isNPlusOneTarget: false,
|
||||
}));
|
||||
return annotated;
|
||||
}
|
||||
|
||||
const minSentenceWordsForNPlusOne = options.minSentenceWordsForNPlusOne;
|
||||
@@ -482,10 +766,25 @@ export function annotateTokens(
|
||||
? minSentenceWordsForNPlusOne
|
||||
: 3;
|
||||
|
||||
return markNPlusOneTargets(
|
||||
jlptMarkedTokens,
|
||||
const nPlusOneMarked = markNPlusOneTargets(
|
||||
annotated,
|
||||
sanitizedMinSentenceWordsForNPlusOne,
|
||||
pos1Exclusions,
|
||||
pos2Exclusions,
|
||||
);
|
||||
|
||||
if (!nameMatchEnabled) {
|
||||
return nPlusOneMarked;
|
||||
}
|
||||
|
||||
return nPlusOneMarked.map((token) =>
|
||||
hasPrioritizedNameMatch(token, options)
|
||||
? {
|
||||
...token,
|
||||
isNPlusOneTarget: false,
|
||||
frequencyRank: undefined,
|
||||
jlptLevel: undefined,
|
||||
}
|
||||
: token,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -212,3 +212,57 @@ test('merges trailing katakana continuation without headword into previous token
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
// Regression: merged content+function token candidate must not beat a multi-token split
|
||||
// candidate that preserves the content token as a standalone frequency-eligible unit.
|
||||
// Background: Yomitan scanning can produce a single-token candidate where a content word
|
||||
// is merged with trailing function particles (e.g. かかってこいよ → headword かかってくる).
|
||||
// When a competing multi-token candidate splits content and function separately, the
|
||||
// multi-token candidate should win so the content token remains frequency-highlightable.
|
||||
test('multi-token candidate beats single merged content+function token candidate (frequency regression)', () => {
|
||||
// Candidate A: single merged token — content verb fused with trailing sentence-final particle
|
||||
// This is the "bad" candidate: downstream annotation would exclude frequency for the whole
|
||||
// token because the merged pos1 would contain a function-word component.
|
||||
const mergedCandidate = makeParseItem('scanning-parser', [
|
||||
[{ text: 'かかってこいよ', reading: 'かかってこいよ', headword: 'かかってくる' }],
|
||||
]);
|
||||
|
||||
// Candidate B: two tokens — content verb surface + particle separately.
|
||||
// The content token is frequency-eligible on its own.
|
||||
const splitCandidate = makeParseItem('scanning-parser', [
|
||||
[{ text: 'かかってこい', reading: 'かかってこい', headword: 'かかってくる' }],
|
||||
[{ text: 'よ', reading: 'よ', headword: 'よ' }],
|
||||
]);
|
||||
|
||||
// When merged candidate comes first in the array, multi-token split still wins.
|
||||
const tokens = selectYomitanParseTokens(
|
||||
[mergedCandidate, splitCandidate],
|
||||
() => false,
|
||||
'headword',
|
||||
);
|
||||
assert.equal(tokens?.length, 2);
|
||||
assert.equal(tokens?.[0]?.surface, 'かかってこい');
|
||||
assert.equal(tokens?.[0]?.headword, 'かかってくる');
|
||||
assert.equal(tokens?.[1]?.surface, 'よ');
|
||||
});
|
||||
|
||||
test('multi-token candidate beats single merged content+function token regardless of input order', () => {
|
||||
const mergedCandidate = makeParseItem('scanning-parser', [
|
||||
[{ text: 'かかってこいよ', reading: 'かかってこいよ', headword: 'かかってくる' }],
|
||||
]);
|
||||
|
||||
const splitCandidate = makeParseItem('scanning-parser', [
|
||||
[{ text: 'かかってこい', reading: 'かかってこい', headword: 'かかってくる' }],
|
||||
[{ text: 'よ', reading: 'よ', headword: 'よ' }],
|
||||
]);
|
||||
|
||||
// Split candidate comes first — should still win over merged.
|
||||
const tokens = selectYomitanParseTokens(
|
||||
[splitCandidate, mergedCandidate],
|
||||
() => false,
|
||||
'headword',
|
||||
);
|
||||
assert.equal(tokens?.length, 2);
|
||||
assert.equal(tokens?.[0]?.surface, 'かかってこい');
|
||||
assert.equal(tokens?.[1]?.surface, 'よ');
|
||||
});
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { PartOfSpeech } from '../../../types';
|
||||
|
||||
function normalizePosTag(value: string | null | undefined): string {
|
||||
return typeof value === 'string' ? value.trim() : '';
|
||||
}
|
||||
|
||||
export function isPartOfSpeechValue(value: unknown): value is PartOfSpeech {
|
||||
return typeof value === 'string' && Object.values(PartOfSpeech).includes(value as PartOfSpeech);
|
||||
}
|
||||
|
||||
export function mapMecabPos1ToPartOfSpeech(pos1: string | null | undefined): PartOfSpeech {
|
||||
switch (normalizePosTag(pos1)) {
|
||||
case '名詞':
|
||||
return PartOfSpeech.noun;
|
||||
case '動詞':
|
||||
return PartOfSpeech.verb;
|
||||
case '形容詞':
|
||||
return PartOfSpeech.i_adjective;
|
||||
case '形状詞':
|
||||
case '形容動詞':
|
||||
return PartOfSpeech.na_adjective;
|
||||
case '助詞':
|
||||
return PartOfSpeech.particle;
|
||||
case '助動詞':
|
||||
return PartOfSpeech.bound_auxiliary;
|
||||
case '記号':
|
||||
case '補助記号':
|
||||
return PartOfSpeech.symbol;
|
||||
default:
|
||||
return PartOfSpeech.other;
|
||||
}
|
||||
}
|
||||
|
||||
export function deriveStoredPartOfSpeech(input: {
|
||||
partOfSpeech?: string | null;
|
||||
pos1?: string | null;
|
||||
}): PartOfSpeech {
|
||||
const pos1Parts = normalizePosTag(input.pos1)
|
||||
.split('|')
|
||||
.map((part) => part.trim())
|
||||
.filter((part) => part.length > 0);
|
||||
|
||||
if (pos1Parts.length > 0) {
|
||||
const derivedParts = [...new Set(pos1Parts.map((part) => mapMecabPos1ToPartOfSpeech(part)))];
|
||||
if (derivedParts.length === 1) {
|
||||
return derivedParts[0]!;
|
||||
}
|
||||
return PartOfSpeech.other;
|
||||
}
|
||||
|
||||
if (isPartOfSpeechValue(input.partOfSpeech)) {
|
||||
return input.partOfSpeech;
|
||||
}
|
||||
|
||||
return PartOfSpeech.other;
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
import {
|
||||
DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG,
|
||||
resolveAnnotationPos1ExclusionSet,
|
||||
} from '../../../token-pos1-exclusions';
|
||||
import {
|
||||
DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG,
|
||||
resolveAnnotationPos2ExclusionSet,
|
||||
} from '../../../token-pos2-exclusions';
|
||||
import { MergedToken, PartOfSpeech } from '../../../types';
|
||||
import { shouldIgnoreJlptByTerm } from '../jlpt-token-filter';
|
||||
|
||||
const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
|
||||
const KATAKANA_CODEPOINT_START = 0x30a1;
|
||||
const KATAKANA_CODEPOINT_END = 0x30f6;
|
||||
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
|
||||
'ああ',
|
||||
'ええ',
|
||||
'うう',
|
||||
'おお',
|
||||
'はあ',
|
||||
'はは',
|
||||
'へえ',
|
||||
'ふう',
|
||||
'ほう',
|
||||
]);
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_PREFIXES = ['ん', 'の', 'なん', 'なの'];
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_CORES = [
|
||||
'だ',
|
||||
'です',
|
||||
'でした',
|
||||
'だった',
|
||||
'では',
|
||||
'じゃ',
|
||||
'でしょう',
|
||||
'だろう',
|
||||
] as const;
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_TRAILING_PARTICLES = [
|
||||
'',
|
||||
'か',
|
||||
'ね',
|
||||
'よ',
|
||||
'な',
|
||||
'よね',
|
||||
'かな',
|
||||
'かね',
|
||||
] as const;
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_THOUGHT_SUFFIXES = ['か', 'かな', 'かね'] as const;
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDINGS = new Set(
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_PREFIXES.flatMap((prefix) =>
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_CORES.flatMap((core) =>
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_TRAILING_PARTICLES.map(
|
||||
(particle) => `${prefix}${core}${particle}`,
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
const SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES = new Set([
|
||||
'って',
|
||||
'ってよ',
|
||||
'ってね',
|
||||
'ってな',
|
||||
'ってさ',
|
||||
'ってか',
|
||||
'ってば',
|
||||
]);
|
||||
const AUXILIARY_STEM_GRAMMAR_TAIL_POS1 = new Set(['名詞', '助動詞', '助詞']);
|
||||
|
||||
export interface SubtitleAnnotationFilterOptions {
|
||||
pos1Exclusions?: ReadonlySet<string>;
|
||||
pos2Exclusions?: ReadonlySet<string>;
|
||||
}
|
||||
|
||||
function normalizePosTag(pos: string | undefined): string {
|
||||
return typeof pos === 'string' ? pos.trim() : '';
|
||||
}
|
||||
|
||||
function splitNormalizedTagParts(normalizedTag: string): string[] {
|
||||
if (!normalizedTag) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return normalizedTag
|
||||
.split('|')
|
||||
.map((part) => part.trim())
|
||||
.filter((part) => part.length > 0);
|
||||
}
|
||||
|
||||
function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<string>): boolean {
|
||||
const parts = splitNormalizedTagParts(normalizedTag);
|
||||
if (parts.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return parts.every((part) => exclusions.has(part));
|
||||
}
|
||||
|
||||
function resolvePos1Exclusions(
|
||||
options: SubtitleAnnotationFilterOptions = {},
|
||||
): ReadonlySet<string> {
|
||||
if (options.pos1Exclusions) {
|
||||
return options.pos1Exclusions;
|
||||
}
|
||||
|
||||
return resolveAnnotationPos1ExclusionSet(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG);
|
||||
}
|
||||
|
||||
function resolvePos2Exclusions(
|
||||
options: SubtitleAnnotationFilterOptions = {},
|
||||
): ReadonlySet<string> {
|
||||
if (options.pos2Exclusions) {
|
||||
return options.pos2Exclusions;
|
||||
}
|
||||
|
||||
return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
|
||||
}
|
||||
|
||||
function normalizeKana(text: string): string {
|
||||
const raw = text.trim();
|
||||
if (!raw) {
|
||||
return '';
|
||||
}
|
||||
|
||||
let normalized = '';
|
||||
for (const char of raw) {
|
||||
const code = char.codePointAt(0);
|
||||
if (code === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END) {
|
||||
normalized += String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET);
|
||||
continue;
|
||||
}
|
||||
|
||||
normalized += char;
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function isKanaChar(char: string): boolean {
|
||||
const code = char.codePointAt(0);
|
||||
if (code === undefined) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
(code >= 0x3041 && code <= 0x3096) ||
|
||||
(code >= 0x309b && code <= 0x309f) ||
|
||||
code === 0x30fc ||
|
||||
(code >= 0x30a0 && code <= 0x30fa) ||
|
||||
(code >= 0x30fd && code <= 0x30ff)
|
||||
);
|
||||
}
|
||||
|
||||
function isTrailingSmallTsuKanaSfx(text: string): boolean {
|
||||
const normalized = normalizeKana(text);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const chars = [...normalized];
|
||||
if (chars.length < 2 || chars.length > 4) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!chars.every(isKanaChar)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return chars[chars.length - 1] === 'っ';
|
||||
}
|
||||
|
||||
function isReduplicatedKanaSfx(text: string): boolean {
|
||||
const normalized = normalizeKana(text);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const chars = [...normalized];
|
||||
if (chars.length < 4 || chars.length % 2 !== 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!chars.every(isKanaChar)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const half = chars.length / 2;
|
||||
return chars.slice(0, half).join('') === chars.slice(half).join('');
|
||||
}
|
||||
|
||||
function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
|
||||
const normalized = normalizeKana(text);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isReduplicatedKanaSfx(normalized)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (normalized.length <= 1 || !normalized.endsWith('と')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return isReduplicatedKanaSfx(normalized.slice(0, -1));
|
||||
}
|
||||
|
||||
function isExcludedTrailingParticleMergedToken(token: MergedToken): boolean {
|
||||
const normalizedSurface = normalizeKana(token.surface);
|
||||
const normalizedHeadword = normalizeKana(token.headword);
|
||||
if (!normalizedSurface || !normalizedHeadword || !normalizedSurface.startsWith(normalizedHeadword)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const suffix = normalizedSurface.slice(normalizedHeadword.length);
|
||||
if (!SUBTITLE_ANNOTATION_EXCLUDED_TRAILING_PARTICLE_SUFFIXES.has(suffix)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
|
||||
if (pos1Parts.length < 2) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const [leadingPos1, ...trailingPos1] = pos1Parts;
|
||||
if (!leadingPos1 || resolvePos1Exclusions().has(leadingPos1)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return trailingPos1.length > 0 && trailingPos1.every((part) => part === '助詞');
|
||||
}
|
||||
|
||||
function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean {
|
||||
const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
|
||||
if (pos1Parts.length === 0 || !pos1Parts.every((part) => AUXILIARY_STEM_GRAMMAR_TAIL_POS1.has(part))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const pos3Parts = splitNormalizedTagParts(normalizePosTag(token.pos3));
|
||||
return pos3Parts.includes('助動詞語幹');
|
||||
}
|
||||
|
||||
function isExcludedByTerm(token: MergedToken): boolean {
|
||||
const candidates = [token.surface, token.reading, token.headword].filter(
|
||||
(candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0,
|
||||
);
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const trimmed = candidate.trim();
|
||||
if (!trimmed) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const normalized = normalizeKana(trimmed);
|
||||
if (!normalized) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_PREFIXES.some((prefix) =>
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_THOUGHT_SUFFIXES.some(
|
||||
(suffix) => normalized === `${prefix}${suffix}`,
|
||||
),
|
||||
)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(trimmed) ||
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(normalized) ||
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDINGS.has(trimmed) ||
|
||||
SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDINGS.has(normalized) ||
|
||||
shouldIgnoreJlptByTerm(trimmed) ||
|
||||
shouldIgnoreJlptByTerm(normalized)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
isTrailingSmallTsuKanaSfx(trimmed) ||
|
||||
isTrailingSmallTsuKanaSfx(normalized) ||
|
||||
isReduplicatedKanaSfxWithOptionalTrailingTo(trimmed) ||
|
||||
isReduplicatedKanaSfxWithOptionalTrailingTo(normalized)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export function shouldExcludeTokenFromSubtitleAnnotations(
|
||||
token: MergedToken,
|
||||
options: SubtitleAnnotationFilterOptions = {},
|
||||
): boolean {
|
||||
const pos1Exclusions = resolvePos1Exclusions(options);
|
||||
const pos2Exclusions = resolvePos2Exclusions(options);
|
||||
const normalizedPos1 = normalizePosTag(token.pos1);
|
||||
const normalizedPos2 = normalizePosTag(token.pos2);
|
||||
const hasPos1 = normalizedPos1.length > 0;
|
||||
const hasPos2 = normalizedPos2.length > 0;
|
||||
|
||||
if (isExcludedByTagSet(normalizedPos1, pos1Exclusions)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isExcludedByTagSet(normalizedPos2, pos2Exclusions)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
!hasPos1 &&
|
||||
!hasPos2 &&
|
||||
(token.partOfSpeech === PartOfSpeech.particle ||
|
||||
token.partOfSpeech === PartOfSpeech.bound_auxiliary ||
|
||||
token.partOfSpeech === PartOfSpeech.symbol)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isAuxiliaryStemGrammarTailToken(token)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isExcludedTrailingParticleMergedToken(token)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return isExcludedByTerm(token);
|
||||
}
|
||||
|
||||
export function stripSubtitleAnnotationMetadata(
|
||||
token: MergedToken,
|
||||
options: SubtitleAnnotationFilterOptions = {},
|
||||
): MergedToken {
|
||||
if (!shouldExcludeTokenFromSubtitleAnnotations(token, options)) {
|
||||
return token;
|
||||
}
|
||||
|
||||
return {
|
||||
...token,
|
||||
isKnown: false,
|
||||
isNPlusOneTarget: false,
|
||||
isNameMatch: false,
|
||||
jlptLevel: undefined,
|
||||
frequencyRank: undefined,
|
||||
};
|
||||
}
|
||||
@@ -188,6 +188,7 @@ test('requestYomitanTermFrequencies returns normalized frequency entries', async
|
||||
{
|
||||
term: '猫',
|
||||
reading: 'ねこ',
|
||||
hasReading: true,
|
||||
dictionary: 'freq-dict',
|
||||
dictionaryPriority: 0,
|
||||
frequency: 77,
|
||||
@@ -197,6 +198,7 @@ test('requestYomitanTermFrequencies returns normalized frequency entries', async
|
||||
{
|
||||
term: '鍛える',
|
||||
reading: 'きたえる',
|
||||
hasReading: false,
|
||||
dictionary: 'freq-dict',
|
||||
dictionaryPriority: 1,
|
||||
frequency: 46961,
|
||||
@@ -217,9 +219,11 @@ test('requestYomitanTermFrequencies returns normalized frequency entries', async
|
||||
|
||||
assert.equal(result.length, 2);
|
||||
assert.equal(result[0]?.term, '猫');
|
||||
assert.equal(result[0]?.hasReading, true);
|
||||
assert.equal(result[0]?.frequency, 77);
|
||||
assert.equal(result[0]?.dictionaryPriority, 0);
|
||||
assert.equal(result[1]?.term, '鍛える');
|
||||
assert.equal(result[1]?.hasReading, false);
|
||||
assert.equal(result[1]?.frequency, 2847);
|
||||
assert.match(scriptValue, /getTermFrequencies/);
|
||||
assert.match(scriptValue, /optionsGetFull/);
|
||||
@@ -247,6 +251,96 @@ test('requestYomitanTermFrequencies prefers primary rank from displayValue array
|
||||
assert.equal(result[0]?.frequency, 7141);
|
||||
});
|
||||
|
||||
test('requestYomitanTermFrequencies prefers primary rank from displayValue string pair when raw frequency matches trailing count', async () => {
|
||||
const deps = createDeps(async () => [
|
||||
{
|
||||
term: '潜む',
|
||||
reading: 'ひそむ',
|
||||
dictionary: 'freq-dict',
|
||||
dictionaryPriority: 0,
|
||||
frequency: 121,
|
||||
displayValue: '118,121',
|
||||
displayValueParsed: false,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await requestYomitanTermFrequencies([{ term: '潜む', reading: 'ひそむ' }], deps, {
|
||||
error: () => undefined,
|
||||
});
|
||||
|
||||
assert.equal(result.length, 1);
|
||||
assert.equal(result[0]?.term, '潜む');
|
||||
assert.equal(result[0]?.frequency, 118);
|
||||
});
|
||||
|
||||
test('requestYomitanTermFrequencies uses leading display digits for displayValue strings', async () => {
|
||||
const deps = createDeps(async () => [
|
||||
{
|
||||
term: '例',
|
||||
reading: 'れい',
|
||||
dictionary: 'freq-dict',
|
||||
dictionaryPriority: 0,
|
||||
frequency: 1234,
|
||||
displayValue: '1,234',
|
||||
displayValueParsed: false,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await requestYomitanTermFrequencies([{ term: '例', reading: 'れい' }], deps, {
|
||||
error: () => undefined,
|
||||
});
|
||||
|
||||
assert.equal(result.length, 1);
|
||||
assert.equal(result[0]?.term, '例');
|
||||
assert.equal(result[0]?.frequency, 1);
|
||||
});
|
||||
|
||||
test('requestYomitanTermFrequencies ignores occurrence-based dictionaries for rank tagging', async () => {
|
||||
let metadataScript = '';
|
||||
const deps = createDeps(async (script) => {
|
||||
if (script.includes('getTermFrequencies')) {
|
||||
return [
|
||||
{
|
||||
term: '潜む',
|
||||
reading: 'ひそむ',
|
||||
dictionary: 'CC100',
|
||||
frequency: 118121,
|
||||
displayValue: null,
|
||||
displayValueParsed: false,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (script.includes('optionsGetFull')) {
|
||||
metadataScript = script;
|
||||
return {
|
||||
profileCurrent: 0,
|
||||
profileIndex: 0,
|
||||
scanLength: 40,
|
||||
dictionaries: ['CC100'],
|
||||
dictionaryPriorityByName: { CC100: 0 },
|
||||
dictionaryFrequencyModeByName: { CC100: 'occurrence-based' },
|
||||
profiles: [
|
||||
{
|
||||
options: {
|
||||
scanning: { length: 40 },
|
||||
dictionaries: [{ name: 'CC100', enabled: true, id: 0 }],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return [];
|
||||
});
|
||||
|
||||
const result = await requestYomitanTermFrequencies([{ term: '潜む', reading: 'ひそむ' }], deps, {
|
||||
error: () => undefined,
|
||||
});
|
||||
|
||||
assert.deepEqual(result, []);
|
||||
assert.match(metadataScript, /getDictionaryInfo/);
|
||||
});
|
||||
|
||||
test('requestYomitanTermFrequencies requests term-only fallback only after reading miss', async () => {
|
||||
const frequencyScripts: string[] = [];
|
||||
const deps = createDeps(async (script) => {
|
||||
@@ -485,6 +579,317 @@ test('requestYomitanScanTokens uses left-to-right termsFind scanning instead of
|
||||
assert.match(scannerScript ?? '', /deinflect:\s*true/);
|
||||
});
|
||||
|
||||
test('requestYomitanScanTokens extracts best frequency rank from selected termsFind entry', async () => {
|
||||
let scannerScript = '';
|
||||
const deps = createDeps(async (script) => {
|
||||
if (script.includes('termsFind')) {
|
||||
scannerScript = script;
|
||||
return [];
|
||||
}
|
||||
if (script.includes('optionsGetFull')) {
|
||||
return {
|
||||
profileCurrent: 0,
|
||||
profileIndex: 0,
|
||||
scanLength: 40,
|
||||
dictionaries: ['JPDBv2㋕', 'Jiten', 'CC100'],
|
||||
dictionaryPriorityByName: {
|
||||
'JPDBv2㋕': 0,
|
||||
Jiten: 1,
|
||||
CC100: 2,
|
||||
},
|
||||
dictionaryFrequencyModeByName: {
|
||||
'JPDBv2㋕': 'rank-based',
|
||||
Jiten: 'rank-based',
|
||||
CC100: 'rank-based',
|
||||
},
|
||||
profiles: [
|
||||
{
|
||||
options: {
|
||||
scanning: { length: 40 },
|
||||
dictionaries: [
|
||||
{ name: 'JPDBv2㋕', enabled: true, id: 0 },
|
||||
{ name: 'Jiten', enabled: true, id: 1 },
|
||||
{ name: 'CC100', enabled: true, id: 2 },
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return null;
|
||||
});
|
||||
|
||||
await requestYomitanScanTokens('潜み', deps, {
|
||||
error: () => undefined,
|
||||
});
|
||||
|
||||
const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
|
||||
if (action !== 'termsFind') {
|
||||
throw new Error(`unexpected action: ${action}`);
|
||||
}
|
||||
|
||||
const text = (params as { text?: string } | undefined)?.text ?? '';
|
||||
if (!text.startsWith('潜み')) {
|
||||
return { originalTextLength: 0, dictionaryEntries: [] };
|
||||
}
|
||||
|
||||
return {
|
||||
originalTextLength: 2,
|
||||
dictionaryEntries: [
|
||||
{
|
||||
headwords: [
|
||||
{
|
||||
term: '潜む',
|
||||
reading: 'ひそむ',
|
||||
sources: [{ originalText: '潜み', isPrimary: true, matchType: 'exact' }],
|
||||
},
|
||||
],
|
||||
frequencies: [
|
||||
{
|
||||
headwordIndex: 0,
|
||||
dictionary: 'JPDBv2㋕',
|
||||
frequency: 20181,
|
||||
displayValue: '4073,20181句',
|
||||
},
|
||||
{
|
||||
headwordIndex: 0,
|
||||
dictionary: 'Jiten',
|
||||
frequency: 28594,
|
||||
displayValue: '4592,28594句',
|
||||
},
|
||||
{
|
||||
headwordIndex: 0,
|
||||
dictionary: 'CC100',
|
||||
frequency: 118121,
|
||||
displayValue: null,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
|
||||
assert.deepEqual(result, [
|
||||
{
|
||||
surface: '潜み',
|
||||
reading: 'ひそ',
|
||||
headword: '潜む',
|
||||
startPos: 0,
|
||||
endPos: 2,
|
||||
isNameMatch: false,
|
||||
frequencyRank: 4073,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('requestYomitanScanTokens uses frequency from later exact-match entry when first exact entry has none', async () => {
|
||||
let scannerScript = '';
|
||||
const deps = createDeps(async (script) => {
|
||||
if (script.includes('termsFind')) {
|
||||
scannerScript = script;
|
||||
return [];
|
||||
}
|
||||
if (script.includes('optionsGetFull')) {
|
||||
return {
|
||||
profileCurrent: 0,
|
||||
profileIndex: 0,
|
||||
scanLength: 40,
|
||||
dictionaries: ['JPDBv2㋕', 'Jiten', 'CC100'],
|
||||
dictionaryPriorityByName: {
|
||||
'JPDBv2㋕': 0,
|
||||
Jiten: 1,
|
||||
CC100: 2,
|
||||
},
|
||||
dictionaryFrequencyModeByName: {
|
||||
'JPDBv2㋕': 'rank-based',
|
||||
Jiten: 'rank-based',
|
||||
CC100: 'rank-based',
|
||||
},
|
||||
profiles: [
|
||||
{
|
||||
options: {
|
||||
scanning: { length: 40 },
|
||||
dictionaries: [
|
||||
{ name: 'JPDBv2㋕', enabled: true, id: 0 },
|
||||
{ name: 'Jiten', enabled: true, id: 1 },
|
||||
{ name: 'CC100', enabled: true, id: 2 },
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return null;
|
||||
});
|
||||
|
||||
await requestYomitanScanTokens('者', deps, {
|
||||
error: () => undefined,
|
||||
});
|
||||
|
||||
const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
|
||||
if (action !== 'termsFind') {
|
||||
throw new Error(`unexpected action: ${action}`);
|
||||
}
|
||||
|
||||
const text = (params as { text?: string } | undefined)?.text ?? '';
|
||||
if (!text.startsWith('者')) {
|
||||
return { originalTextLength: 0, dictionaryEntries: [] };
|
||||
}
|
||||
|
||||
return {
|
||||
originalTextLength: 1,
|
||||
dictionaryEntries: [
|
||||
{
|
||||
headwords: [
|
||||
{
|
||||
term: '者',
|
||||
reading: 'もの',
|
||||
sources: [{ originalText: '者', isPrimary: true, matchType: 'exact' }],
|
||||
},
|
||||
],
|
||||
frequencies: [],
|
||||
},
|
||||
{
|
||||
headwords: [
|
||||
{
|
||||
term: '者',
|
||||
reading: 'もの',
|
||||
sources: [{ originalText: '者', isPrimary: true, matchType: 'exact' }],
|
||||
},
|
||||
],
|
||||
frequencies: [
|
||||
{
|
||||
headwordIndex: 0,
|
||||
dictionary: 'JPDBv2㋕',
|
||||
frequency: 79601,
|
||||
displayValue: '475,79601句',
|
||||
},
|
||||
{
|
||||
headwordIndex: 0,
|
||||
dictionary: 'Jiten',
|
||||
frequency: 338,
|
||||
displayValue: '338',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
|
||||
assert.deepEqual(result, [
|
||||
{
|
||||
surface: '者',
|
||||
reading: 'もの',
|
||||
headword: '者',
|
||||
startPos: 0,
|
||||
endPos: 1,
|
||||
isNameMatch: false,
|
||||
frequencyRank: 475,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('requestYomitanScanTokens can use frequency from later exact secondary-match entry', async () => {
|
||||
let scannerScript = '';
|
||||
const deps = createDeps(async (script) => {
|
||||
if (script.includes('termsFind')) {
|
||||
scannerScript = script;
|
||||
return [];
|
||||
}
|
||||
if (script.includes('optionsGetFull')) {
|
||||
return {
|
||||
profileCurrent: 0,
|
||||
profileIndex: 0,
|
||||
scanLength: 40,
|
||||
dictionaries: ['JPDBv2㋕', 'Jiten', 'CC100'],
|
||||
dictionaryPriorityByName: {
|
||||
'JPDBv2㋕': 0,
|
||||
Jiten: 1,
|
||||
CC100: 2,
|
||||
},
|
||||
dictionaryFrequencyModeByName: {
|
||||
'JPDBv2㋕': 'rank-based',
|
||||
Jiten: 'rank-based',
|
||||
CC100: 'rank-based',
|
||||
},
|
||||
profiles: [
|
||||
{
|
||||
options: {
|
||||
scanning: { length: 40 },
|
||||
dictionaries: [
|
||||
{ name: 'JPDBv2㋕', enabled: true, id: 0 },
|
||||
{ name: 'Jiten', enabled: true, id: 1 },
|
||||
{ name: 'CC100', enabled: true, id: 2 },
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return null;
|
||||
});
|
||||
|
||||
await requestYomitanScanTokens('者', deps, {
|
||||
error: () => undefined,
|
||||
});
|
||||
|
||||
const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
|
||||
if (action !== 'termsFind') {
|
||||
throw new Error(`unexpected action: ${action}`);
|
||||
}
|
||||
|
||||
const text = (params as { text?: string } | undefined)?.text ?? '';
|
||||
if (!text.startsWith('者')) {
|
||||
return { originalTextLength: 0, dictionaryEntries: [] };
|
||||
}
|
||||
|
||||
return {
|
||||
originalTextLength: 1,
|
||||
dictionaryEntries: [
|
||||
{
|
||||
headwords: [
|
||||
{
|
||||
term: '者',
|
||||
reading: 'もの',
|
||||
sources: [{ originalText: '者', isPrimary: true, matchType: 'exact' }],
|
||||
},
|
||||
],
|
||||
frequencies: [],
|
||||
},
|
||||
{
|
||||
headwords: [
|
||||
{
|
||||
term: '者',
|
||||
reading: 'もの',
|
||||
sources: [{ originalText: '者', isPrimary: false, matchType: 'exact' }],
|
||||
},
|
||||
],
|
||||
frequencies: [
|
||||
{
|
||||
headwordIndex: 0,
|
||||
dictionary: 'JPDBv2㋕',
|
||||
frequency: 79601,
|
||||
displayValue: '475,79601句',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
|
||||
assert.deepEqual(result, [
|
||||
{
|
||||
surface: '者',
|
||||
reading: 'もの',
|
||||
headword: '者',
|
||||
startPos: 0,
|
||||
endPos: 1,
|
||||
isNameMatch: false,
|
||||
frequencyRank: 475,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('requestYomitanScanTokens marks tokens backed by SubMiner character dictionary entries', async () => {
|
||||
const deps = createDeps(async (script) => {
|
||||
if (script.includes('optionsGetFull')) {
|
||||
|
||||
@@ -20,19 +20,24 @@ interface YomitanParserRuntimeDeps {
|
||||
createYomitanExtensionWindow?: (pageName: string) => Promise<BrowserWindow | null>;
|
||||
}
|
||||
|
||||
type YomitanFrequencyMode = 'occurrence-based' | 'rank-based';
|
||||
|
||||
export interface YomitanDictionaryInfo {
|
||||
title: string;
|
||||
revision?: string | number;
|
||||
frequencyMode?: YomitanFrequencyMode;
|
||||
}
|
||||
|
||||
export interface YomitanTermFrequency {
|
||||
term: string;
|
||||
reading: string | null;
|
||||
hasReading: boolean;
|
||||
dictionary: string;
|
||||
dictionaryPriority: number;
|
||||
frequency: number;
|
||||
displayValue: string | null;
|
||||
displayValueParsed: boolean;
|
||||
frequencyDerivedFromDisplayValue: boolean;
|
||||
}
|
||||
|
||||
export interface YomitanTermReadingPair {
|
||||
@@ -47,6 +52,7 @@ export interface YomitanScanToken {
|
||||
startPos: number;
|
||||
endPos: number;
|
||||
isNameMatch?: boolean;
|
||||
frequencyRank?: number;
|
||||
}
|
||||
|
||||
interface YomitanProfileMetadata {
|
||||
@@ -54,6 +60,7 @@ interface YomitanProfileMetadata {
|
||||
scanLength: number;
|
||||
dictionaries: string[];
|
||||
dictionaryPriorityByName: Record<string, number>;
|
||||
dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>;
|
||||
}
|
||||
|
||||
const DEFAULT_YOMITAN_SCAN_LENGTH = 40;
|
||||
@@ -78,7 +85,8 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
|
||||
typeof entry.headword === 'string' &&
|
||||
typeof entry.startPos === 'number' &&
|
||||
typeof entry.endPos === 'number' &&
|
||||
(entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean'),
|
||||
(entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean') &&
|
||||
(entry.frequencyRank === undefined || typeof entry.frequencyRank === 'number'),
|
||||
)
|
||||
);
|
||||
}
|
||||
@@ -117,24 +125,22 @@ function parsePositiveFrequencyString(value: string): number | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
const numericPrefix = trimmed.match(/^\d[\d,]*/)?.[0];
|
||||
if (!numericPrefix) {
|
||||
const numericMatch = trimmed.match(/[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?/)?.[0];
|
||||
if (!numericMatch) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const chunks = numericPrefix.split(',');
|
||||
const normalizedNumber =
|
||||
chunks.length <= 1
|
||||
? (chunks[0] ?? '')
|
||||
: chunks.slice(1).every((chunk) => /^\d{3}$/.test(chunk))
|
||||
? chunks.join('')
|
||||
: (chunks[0] ?? '');
|
||||
const parsed = Number.parseInt(normalizedNumber, 10);
|
||||
const parsed = Number.parseFloat(numericMatch);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return parsed;
|
||||
const normalized = Math.floor(parsed);
|
||||
if (!Number.isFinite(normalized) || normalized <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function parsePositiveFrequencyValue(value: unknown): number | null {
|
||||
@@ -159,6 +165,19 @@ function parsePositiveFrequencyValue(value: unknown): number | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseDisplayFrequencyValue(value: unknown): number | null {
|
||||
if (typeof value === 'string') {
|
||||
const leadingDigits = value.trim().match(/^\d+/)?.[0];
|
||||
if (!leadingDigits) {
|
||||
return null;
|
||||
}
|
||||
const parsed = Number.parseInt(leadingDigits, 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
|
||||
}
|
||||
|
||||
return parsePositiveFrequencyValue(value);
|
||||
}
|
||||
|
||||
function toYomitanTermFrequency(value: unknown): YomitanTermFrequency | null {
|
||||
if (!isObject(value)) {
|
||||
return null;
|
||||
@@ -170,7 +189,7 @@ function toYomitanTermFrequency(value: unknown): YomitanTermFrequency | null {
|
||||
const displayValueRaw = value.displayValue;
|
||||
const parsedDisplayFrequency =
|
||||
displayValueRaw !== null && displayValueRaw !== undefined
|
||||
? parsePositiveFrequencyValue(displayValueRaw)
|
||||
? parseDisplayFrequencyValue(displayValueRaw)
|
||||
: null;
|
||||
const frequency = parsedDisplayFrequency ?? rawFrequency;
|
||||
if (!term || !dictionary || frequency === null) {
|
||||
@@ -184,17 +203,20 @@ function toYomitanTermFrequency(value: unknown): YomitanTermFrequency | null {
|
||||
|
||||
const reading =
|
||||
value.reading === null ? null : typeof value.reading === 'string' ? value.reading : null;
|
||||
const hasReading = value.hasReading === false ? false : reading !== null;
|
||||
const displayValue = typeof displayValueRaw === 'string' ? displayValueRaw : null;
|
||||
const displayValueParsed = value.displayValueParsed === true;
|
||||
|
||||
return {
|
||||
term,
|
||||
reading,
|
||||
hasReading,
|
||||
dictionary,
|
||||
dictionaryPriority,
|
||||
frequency,
|
||||
displayValue,
|
||||
displayValueParsed,
|
||||
frequencyDerivedFromDisplayValue: parsedDisplayFrequency !== null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -300,17 +322,34 @@ function toYomitanProfileMetadata(value: unknown): YomitanProfileMetadata | null
|
||||
}
|
||||
}
|
||||
|
||||
const dictionaryFrequencyModeByNameRaw = value.dictionaryFrequencyModeByName;
|
||||
const dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>> = {};
|
||||
if (isObject(dictionaryFrequencyModeByNameRaw)) {
|
||||
for (const [name, frequencyModeRaw] of Object.entries(dictionaryFrequencyModeByNameRaw)) {
|
||||
const normalizedName = name.trim();
|
||||
if (!normalizedName) {
|
||||
continue;
|
||||
}
|
||||
if (frequencyModeRaw !== 'occurrence-based' && frequencyModeRaw !== 'rank-based') {
|
||||
continue;
|
||||
}
|
||||
dictionaryFrequencyModeByName[normalizedName] = frequencyModeRaw;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
profileIndex,
|
||||
scanLength,
|
||||
dictionaries,
|
||||
dictionaryPriorityByName,
|
||||
dictionaryFrequencyModeByName,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeFrequencyEntriesWithPriority(
|
||||
rawResult: unknown[],
|
||||
dictionaryPriorityByName: Record<string, number>,
|
||||
dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>,
|
||||
): YomitanTermFrequency[] {
|
||||
const normalized: YomitanTermFrequency[] = [];
|
||||
for (const entry of rawResult) {
|
||||
@@ -319,6 +358,10 @@ function normalizeFrequencyEntriesWithPriority(
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dictionaryFrequencyModeByName[frequency.dictionary] === 'occurrence-based') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const dictionaryPriority = dictionaryPriorityByName[frequency.dictionary];
|
||||
normalized.push({
|
||||
...frequency,
|
||||
@@ -425,8 +468,34 @@ async function requestYomitanProfileMetadata(
|
||||
acc[entry.name] = index;
|
||||
return acc;
|
||||
}, {});
|
||||
let dictionaryFrequencyModeByName = {};
|
||||
try {
|
||||
const dictionaryInfo = await invoke("getDictionaryInfo", undefined);
|
||||
dictionaryFrequencyModeByName = Array.isArray(dictionaryInfo)
|
||||
? dictionaryInfo.reduce((acc, entry) => {
|
||||
if (!entry || typeof entry !== "object" || typeof entry.title !== "string") {
|
||||
return acc;
|
||||
}
|
||||
if (
|
||||
entry.frequencyMode === "occurrence-based" ||
|
||||
entry.frequencyMode === "rank-based"
|
||||
) {
|
||||
acc[entry.title] = entry.frequencyMode;
|
||||
}
|
||||
return acc;
|
||||
}, {})
|
||||
: {};
|
||||
} catch {
|
||||
dictionaryFrequencyModeByName = {};
|
||||
}
|
||||
|
||||
return { profileIndex, scanLength, dictionaries, dictionaryPriorityByName };
|
||||
return {
|
||||
profileIndex,
|
||||
scanLength,
|
||||
dictionaries,
|
||||
dictionaryPriorityByName,
|
||||
dictionaryFrequencyModeByName
|
||||
};
|
||||
})();
|
||||
`;
|
||||
|
||||
@@ -774,7 +843,133 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
|
||||
}
|
||||
return segments;
|
||||
}
|
||||
function getPreferredHeadword(dictionaryEntries, token) {
|
||||
function parsePositiveFrequencyNumber(value) {
|
||||
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
|
||||
return Math.max(1, Math.floor(value));
|
||||
}
|
||||
if (typeof value === 'string') {
|
||||
const numericMatch = value.trim().match(/[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?/)?.[0];
|
||||
if (!numericMatch) { return null; }
|
||||
const parsed = Number.parseFloat(numericMatch);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) { return null; }
|
||||
return Math.max(1, Math.floor(parsed));
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
const parsed = parsePositiveFrequencyNumber(item);
|
||||
if (parsed !== null) { return parsed; }
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function parseDisplayFrequencyNumber(value) {
|
||||
if (typeof value === 'string') {
|
||||
const leadingDigits = value.trim().match(/^\d+/)?.[0];
|
||||
if (!leadingDigits) { return null; }
|
||||
const parsed = Number.parseInt(leadingDigits, 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
|
||||
}
|
||||
return parsePositiveFrequencyNumber(value);
|
||||
}
|
||||
function getFrequencyDictionaryName(frequency) {
|
||||
const candidates = [
|
||||
frequency?.dictionary,
|
||||
frequency?.dictionaryName,
|
||||
frequency?.name,
|
||||
frequency?.title,
|
||||
frequency?.dictionaryTitle,
|
||||
frequency?.dictionaryAlias
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (typeof candidate === 'string' && candidate.trim().length > 0) {
|
||||
return candidate.trim();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function getBestFrequencyRank(dictionaryEntry, headwordIndex, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
|
||||
let best = null;
|
||||
const headwordCount = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords.length : 0;
|
||||
for (const frequency of dictionaryEntry?.frequencies || []) {
|
||||
if (!frequency || typeof frequency !== 'object') { continue; }
|
||||
const frequencyHeadwordIndex = frequency.headwordIndex;
|
||||
if (typeof frequencyHeadwordIndex === 'number') {
|
||||
if (frequencyHeadwordIndex !== headwordIndex) { continue; }
|
||||
} else if (headwordCount > 1) {
|
||||
continue;
|
||||
}
|
||||
const dictionary = getFrequencyDictionaryName(frequency);
|
||||
if (!dictionary) { continue; }
|
||||
if (dictionaryFrequencyModeByName[dictionary] === 'occurrence-based') { continue; }
|
||||
const rank =
|
||||
parseDisplayFrequencyNumber(frequency.displayValue) ??
|
||||
parsePositiveFrequencyNumber(frequency.frequency);
|
||||
if (rank === null) { continue; }
|
||||
const priorityRaw = dictionaryPriorityByName[dictionary];
|
||||
const fallbackPriority =
|
||||
typeof frequency.dictionaryIndex === 'number' && Number.isFinite(frequency.dictionaryIndex)
|
||||
? Math.max(0, Math.floor(frequency.dictionaryIndex))
|
||||
: Number.MAX_SAFE_INTEGER;
|
||||
const priority =
|
||||
typeof priorityRaw === 'number' && Number.isFinite(priorityRaw)
|
||||
? Math.max(0, Math.floor(priorityRaw))
|
||||
: fallbackPriority;
|
||||
if (best === null || priority < best.priority || (priority === best.priority && rank < best.rank)) {
|
||||
best = { priority, rank };
|
||||
}
|
||||
}
|
||||
return best?.rank ?? null;
|
||||
}
|
||||
function hasExactSource(headword, token, requirePrimary) {
|
||||
for (const src of headword.sources || []) {
|
||||
if (src.originalText !== token) { continue; }
|
||||
if (requirePrimary && !src.isPrimary) { continue; }
|
||||
if (src.matchType !== 'exact') { continue; }
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
function collectExactHeadwordMatches(dictionaryEntries, token, requirePrimary) {
|
||||
const matches = [];
|
||||
for (const dictionaryEntry of dictionaryEntries || []) {
|
||||
const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
|
||||
for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
|
||||
const headword = headwords[headwordIndex];
|
||||
if (!hasExactSource(headword, token, requirePrimary)) { continue; }
|
||||
matches.push({ dictionaryEntry, headword, headwordIndex });
|
||||
}
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
function sameHeadword(match, preferredMatch) {
|
||||
if (!match || !preferredMatch) {
|
||||
return false;
|
||||
}
|
||||
if (match.headword?.term !== preferredMatch.headword?.term) {
|
||||
return false;
|
||||
}
|
||||
const matchReading = typeof match.headword?.reading === 'string' ? match.headword.reading : '';
|
||||
const preferredReading =
|
||||
typeof preferredMatch.headword?.reading === 'string' ? preferredMatch.headword.reading : '';
|
||||
return matchReading === preferredReading;
|
||||
}
|
||||
function getBestFrequencyRankForMatches(matches, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
|
||||
let best = null;
|
||||
for (const match of matches) {
|
||||
const rank = getBestFrequencyRank(
|
||||
match.dictionaryEntry,
|
||||
match.headwordIndex,
|
||||
dictionaryPriorityByName,
|
||||
dictionaryFrequencyModeByName
|
||||
);
|
||||
if (rank === null) { continue; }
|
||||
if (best === null || rank < best) {
|
||||
best = rank;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
|
||||
function appendDictionaryNames(target, value) {
|
||||
if (!value || typeof value !== 'object') {
|
||||
return;
|
||||
@@ -813,36 +1008,33 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
|
||||
}
|
||||
return getDictionaryEntryNames(entry).some((name) => name.startsWith("SubMiner Character Dictionary"));
|
||||
}
|
||||
function hasExactPrimarySource(headword, token) {
|
||||
for (const src of headword.sources || []) {
|
||||
if (src.originalText !== token) { continue; }
|
||||
if (!src.isPrimary) { continue; }
|
||||
if (src.matchType !== 'exact') { continue; }
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
const exactPrimaryMatches = collectExactHeadwordMatches(dictionaryEntries, token, true);
|
||||
let matchedNameDictionary = false;
|
||||
if (includeNameMatchMetadata) {
|
||||
for (const dictionaryEntry of dictionaryEntries || []) {
|
||||
if (!isNameDictionaryEntry(dictionaryEntry)) { continue; }
|
||||
for (const headword of dictionaryEntry.headwords || []) {
|
||||
if (!hasExactPrimarySource(headword, token)) { continue; }
|
||||
for (const match of exactPrimaryMatches) {
|
||||
if (match.dictionaryEntry !== dictionaryEntry) { continue; }
|
||||
matchedNameDictionary = true;
|
||||
break;
|
||||
}
|
||||
if (matchedNameDictionary) { break; }
|
||||
}
|
||||
}
|
||||
for (const dictionaryEntry of dictionaryEntries || []) {
|
||||
for (const headword of dictionaryEntry.headwords || []) {
|
||||
if (!hasExactPrimarySource(headword, token)) { continue; }
|
||||
return {
|
||||
term: headword.term,
|
||||
reading: headword.reading,
|
||||
isNameMatch: matchedNameDictionary || isNameDictionaryEntry(dictionaryEntry)
|
||||
};
|
||||
}
|
||||
const preferredMatch = exactPrimaryMatches[0];
|
||||
if (preferredMatch) {
|
||||
const exactFrequencyMatches = collectExactHeadwordMatches(dictionaryEntries, token, false)
|
||||
.filter((match) => sameHeadword(match, preferredMatch));
|
||||
return {
|
||||
term: preferredMatch.headword.term,
|
||||
reading: preferredMatch.headword.reading,
|
||||
isNameMatch: matchedNameDictionary || isNameDictionaryEntry(preferredMatch.dictionaryEntry),
|
||||
frequencyRank: getBestFrequencyRankForMatches(
|
||||
exactFrequencyMatches.length > 0 ? exactFrequencyMatches : exactPrimaryMatches,
|
||||
dictionaryPriorityByName,
|
||||
dictionaryFrequencyModeByName
|
||||
)
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -853,6 +1045,8 @@ function buildYomitanScanningScript(
|
||||
profileIndex: number,
|
||||
scanLength: number,
|
||||
includeNameMatchMetadata: boolean,
|
||||
dictionaryPriorityByName: Record<string, number>,
|
||||
dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>,
|
||||
): string {
|
||||
return `
|
||||
(async () => {
|
||||
@@ -876,6 +1070,8 @@ function buildYomitanScanningScript(
|
||||
});
|
||||
${YOMITAN_SCANNING_HELPERS}
|
||||
const includeNameMatchMetadata = ${includeNameMatchMetadata ? 'true' : 'false'};
|
||||
const dictionaryPriorityByName = ${JSON.stringify(dictionaryPriorityByName)};
|
||||
const dictionaryFrequencyModeByName = ${JSON.stringify(dictionaryFrequencyModeByName)};
|
||||
const text = ${JSON.stringify(text)};
|
||||
const details = {matchType: "exact", deinflect: true};
|
||||
const tokens = [];
|
||||
@@ -889,7 +1085,12 @@ ${YOMITAN_SCANNING_HELPERS}
|
||||
const originalTextLength = typeof result?.originalTextLength === "number" ? result.originalTextLength : 0;
|
||||
if (dictionaryEntries.length > 0 && originalTextLength > 0 && (originalTextLength !== character.length || isCodePointJapanese(codePoint))) {
|
||||
const source = substring.substring(0, originalTextLength);
|
||||
const preferredHeadword = getPreferredHeadword(dictionaryEntries, source);
|
||||
const preferredHeadword = getPreferredHeadword(
|
||||
dictionaryEntries,
|
||||
source,
|
||||
dictionaryPriorityByName,
|
||||
dictionaryFrequencyModeByName
|
||||
);
|
||||
if (preferredHeadword && typeof preferredHeadword.term === "string") {
|
||||
const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
|
||||
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
|
||||
@@ -900,6 +1101,10 @@ ${YOMITAN_SCANNING_HELPERS}
|
||||
startPos: i,
|
||||
endPos: i + originalTextLength,
|
||||
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
|
||||
frequencyRank:
|
||||
typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
|
||||
? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
|
||||
: undefined,
|
||||
});
|
||||
i += originalTextLength;
|
||||
continue;
|
||||
@@ -1036,6 +1241,8 @@ export async function requestYomitanScanTokens(
|
||||
profileIndex,
|
||||
scanLength,
|
||||
options?.includeNameMatchMetadata === true,
|
||||
metadata?.dictionaryPriorityByName ?? {},
|
||||
metadata?.dictionaryFrequencyModeByName ?? {},
|
||||
),
|
||||
true,
|
||||
);
|
||||
@@ -1099,7 +1306,11 @@ async function fetchYomitanTermFrequencies(
|
||||
try {
|
||||
const rawResult = await parserWindow.webContents.executeJavaScript(script, true);
|
||||
return Array.isArray(rawResult)
|
||||
? normalizeFrequencyEntriesWithPriority(rawResult, metadata.dictionaryPriorityByName)
|
||||
? normalizeFrequencyEntriesWithPriority(
|
||||
rawResult,
|
||||
metadata.dictionaryPriorityByName,
|
||||
metadata.dictionaryFrequencyModeByName,
|
||||
)
|
||||
: [];
|
||||
} catch (err) {
|
||||
logger.error('Yomitan term frequency request failed:', (err as Error).message);
|
||||
@@ -1541,10 +1752,15 @@ export async function getYomitanDictionaryInfo(
|
||||
.map((entry) => {
|
||||
const title = typeof entry.title === 'string' ? entry.title.trim() : '';
|
||||
const revision = entry.revision;
|
||||
const frequencyMode: YomitanFrequencyMode | undefined =
|
||||
entry.frequencyMode === 'occurrence-based' || entry.frequencyMode === 'rank-based'
|
||||
? entry.frequencyMode
|
||||
: undefined;
|
||||
return {
|
||||
title,
|
||||
revision:
|
||||
typeof revision === 'string' || typeof revision === 'number' ? revision : undefined,
|
||||
frequencyMode,
|
||||
};
|
||||
})
|
||||
.filter((entry) => entry.title.length > 0);
|
||||
@@ -1763,3 +1979,34 @@ export async function removeYomitanDictionarySettings(
|
||||
|
||||
return await setYomitanSettingsFull(optionsFull, deps, logger);
|
||||
}
|
||||
|
||||
export async function addYomitanNoteViaSearch(
|
||||
word: string,
|
||||
deps: YomitanParserRuntimeDeps,
|
||||
logger: LoggerLike,
|
||||
): Promise<number | null> {
|
||||
const isReady = await ensureYomitanParserWindow(deps, logger);
|
||||
const parserWindow = deps.getYomitanParserWindow();
|
||||
if (!isReady || !parserWindow || parserWindow.isDestroyed()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const escapedWord = JSON.stringify(word);
|
||||
|
||||
const script = `
|
||||
(async () => {
|
||||
if (typeof window.__subminerAddNote !== 'function') {
|
||||
throw new Error('Yomitan search page bridge not initialized');
|
||||
}
|
||||
return await window.__subminerAddNote(${escapedWord});
|
||||
})();
|
||||
`;
|
||||
|
||||
try {
|
||||
const noteId = await parserWindow.webContents.executeJavaScript(script, true);
|
||||
return typeof noteId === 'number' ? noteId : null;
|
||||
} catch (err) {
|
||||
logger.error('Yomitan addNoteFromWord failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user