fix(overlay): resolve unspaced Japanese name splits and scan recovery (#146)

This commit is contained in:
2026-07-07 02:33:46 -07:00
committed by GitHub
parent e7739de51c
commit 61f39d1e09
19 changed files with 884 additions and 76 deletions
+2 -1
View File
@@ -150,7 +150,8 @@ function hasAnyAnnotationEnabled(options: TokenizerAnnotationOptions): boolean {
options.knownWordsEnabled ||
options.nPlusOneEnabled ||
options.jlptEnabled ||
options.frequencyEnabled
options.frequencyEnabled ||
options.nameMatchEnabled
);
}
@@ -33,6 +33,50 @@ function makeDeps(overrides: Partial<AnnotationStageDeps> = {}): AnnotationStage
};
}
test('annotateTokens keeps name matches on tokens the POS noise filter would strip', () => {
// MeCab tags 平 as 接頭詞 in contexts like あっ 平 これ…, which is in the
// POS1 exclusion list; a confirmed character-name match must survive it.
const tokens = [
makeToken({
surface: '平',
headword: '平',
reading: 'たいら',
pos1: '接頭詞',
isNameMatch: true,
}),
makeToken({
surface: '平',
headword: '平',
reading: 'ひら',
pos1: '接頭詞',
isNameMatch: false,
jlptLevel: 'N1',
}),
];
const result = annotateTokens(tokens, makeDeps(), { nameMatchEnabled: true });
assert.equal(result[0]?.isNameMatch, true);
assert.equal(result[1]?.isNameMatch, false);
assert.equal(result[1]?.jlptLevel, undefined);
});
test('annotateTokens strips name matches from POS-excluded tokens when name matching is disabled', () => {
const tokens = [
makeToken({
surface: '平',
headword: '平',
reading: 'たいら',
pos1: '接頭詞',
isNameMatch: true,
}),
];
const result = annotateTokens(tokens, makeDeps(), { nameMatchEnabled: false });
assert.equal(result[0]?.isNameMatch, false);
});
test('annotateTokens known-word match mode uses headword vs surface', () => {
const tokens = [makeToken({ surface: '食べた', headword: '食べる', reading: 'タベタ' })];
const isKnownWord = (text: string): boolean => text === '食べる';
@@ -1994,7 +2038,8 @@ test('annotateTokens keeps known status while clearing other annotations for aru
assert.equal(result[0]?.headword, '有る');
assert.equal(result[0]?.isKnown, true);
assert.equal(result[0]?.isNPlusOneTarget, false);
assert.equal(result[0]?.isNameMatch, false);
// Name matches take precedence over the annotation noise filter.
assert.equal(result[0]?.isNameMatch, true);
assert.equal(result[0]?.frequencyRank, undefined);
assert.equal(result[0]?.jlptLevel, undefined);
});
@@ -778,7 +778,13 @@ export function annotateTokens(
: false;
nPlusOneKnownStatuses[index] = isKnownForMatching;
const prioritizedNameMatch = nameMatchEnabled && token.isNameMatch === true;
// A confirmed character-name match must survive the POS noise filter:
// MeCab can tag a name like 平 as a prefix (接頭詞) depending on context,
// which would otherwise strip the name match and its portrait.
if (
!prioritizedNameMatch &&
sharedShouldExcludeTokenFromSubtitleAnnotations(token, {
pos1Exclusions,
pos2Exclusions,
@@ -794,8 +800,6 @@ export function annotateTokens(
};
}
const prioritizedNameMatch = nameMatchEnabled && token.isNameMatch === true;
const frequencyRank =
frequencyEnabled && !prioritizedNameMatch
? filterTokenFrequencyRank(token, pos1Exclusions, pos2Exclusions)
@@ -975,6 +975,105 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF
]);
});
test('requestYomitanScanTokens retries shorter windows when a greedy match has no exact-source headword', async () => {
let scannerScript = '';
const deps = createDeps(async (script) => {
if (script.includes('termsFind')) {
scannerScript = script;
return [];
}
if (script.includes('optionsGetFull')) {
return {
profileCurrent: 0,
profileIndex: 0,
scanLength: 40,
dictionaries: ['JMdict'],
dictionaryPriorityByName: { JMdict: 0 },
dictionaryFrequencyModeByName: {},
profiles: [
{
options: {
scanning: { length: 40 },
dictionaries: [{ name: 'JMdict', enabled: true, id: 0 }],
},
},
],
};
}
return null;
});
await requestYomitanScanTokens('平 (平)', deps, {
error: () => undefined,
});
const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
if (action !== 'termsFind') {
throw new Error(`unexpected action: ${action}`);
}
const text = (params as { text?: string } | undefined)?.text ?? '';
if (!text.startsWith('平')) {
return { originalTextLength: 0, dictionaryEntries: [] };
}
if (text.length >= 4) {
// Simulates Yomitan normalization consuming punctuation/whitespace:
// the greedy match spans 平 (平 but no headword source equals it.
return {
originalTextLength: 4,
dictionaryEntries: [
{
headwords: [
{
term: '平々',
reading: 'へいへい',
sources: [{ originalText: '平平', isPrimary: true, matchType: 'exact' }],
},
],
},
],
};
}
return {
originalTextLength: 1,
dictionaryEntries: [
{
headwords: [
{
term: '平',
reading: 'たいら',
sources: [{ originalText: '平', isPrimary: true, matchType: 'exact' }],
},
],
},
],
};
});
assert.deepEqual(result, [
{
surface: '平',
reading: 'たいら',
headword: '平',
headwordReading: 'たいら',
startPos: 0,
endPos: 1,
isNameMatch: false,
frequencyRank: undefined,
},
{
surface: '平',
reading: 'たいら',
headword: '平',
headwordReading: 'たいら',
startPos: 3,
endPos: 4,
isNameMatch: false,
frequencyRank: undefined,
},
]);
});
test('requestYomitanScanTokens emits complete readings for kanji-kana compounds', async () => {
let scannerScript = '';
const deps = createDeps(async (script) => {
@@ -1297,47 +1297,69 @@ ${YOMITAN_SCANNING_HELPERS}
const text = ${JSON.stringify(text)};
const details = {matchType: "exact", deinflect: true};
const tokens = [];
let i = 0;
while (i < text.length) {
const codePoint = text.codePointAt(i);
async function findTokenAt(position, windowLength) {
const codePoint = text.codePointAt(position);
const character = String.fromCodePoint(codePoint);
const substring = text.substring(i, i + ${scanLength});
const substring = text.substring(position, position + windowLength);
const result = await invoke("termsFind", { text: substring, details, optionsContext: { index: ${profileIndex} } });
const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
const originalTextLength = typeof result?.originalTextLength === "number" ? result.originalTextLength : 0;
if (dictionaryEntries.length > 0 && originalTextLength > 0 && (originalTextLength !== character.length || isCodePointJapanese(codePoint))) {
const source = substring.substring(0, originalTextLength);
const preferredHeadword = getPreferredHeadword(
dictionaryEntries,
source,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
);
if (preferredHeadword && typeof preferredHeadword.term === "string") {
const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
const tokenPayload = {
surface: segments.map((segment) => segment.text).join("") || source,
reading: segments.map(getSegmentReadingContribution).join(""),
headword: preferredHeadword.term,
headwordReading: reading || undefined,
startPos: i,
endPos: i + originalTextLength,
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
frequencyRank:
typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
: undefined,
};
if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
tokenPayload.wordClasses = preferredHeadword.wordClasses;
}
tokens.push(tokenPayload);
i += originalTextLength;
continue;
}
if (dictionaryEntries.length === 0 || originalTextLength <= 0 || (originalTextLength === character.length && !isCodePointJapanese(codePoint))) {
return { token: null, matchedLength: 0 };
}
i += character.length;
const source = substring.substring(0, originalTextLength);
const preferredHeadword = getPreferredHeadword(
dictionaryEntries,
source,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
);
if (!preferredHeadword || typeof preferredHeadword.term !== "string") {
return { token: null, matchedLength: originalTextLength };
}
const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
const tokenPayload = {
surface: segments.map((segment) => segment.text).join("") || source,
reading: segments.map(getSegmentReadingContribution).join(""),
headword: preferredHeadword.term,
headwordReading: reading || undefined,
startPos: position,
endPos: position + originalTextLength,
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
frequencyRank:
typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
: undefined,
};
if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
tokenPayload.wordClasses = preferredHeadword.wordClasses;
}
return { token: tokenPayload, matchedLength: originalTextLength };
}
let i = 0;
while (i < text.length) {
let attempt = await findTokenAt(i, ${scanLength});
// Yomitan text normalization can consume characters (whitespace,
// punctuation) beyond the matched term, leaving no headword whose
// source equals the consumed text. Retry with shorter windows so a
// valid prefix term (e.g. a character name before a paren) still
// tokenizes instead of the position being skipped.
let retryLength = Math.min(attempt.matchedLength, ${scanLength}) - 1;
while (!attempt.token && retryLength >= 1) {
const retry = await findTokenAt(i, retryLength);
if (retry.token) {
attempt = retry;
break;
}
retryLength = Math.min(retryLength - 1, retry.matchedLength - 1);
}
if (attempt.token) {
tokens.push(attempt.token);
i += attempt.matchedLength;
continue;
}
i += String.fromCodePoint(text.codePointAt(i)).length;
}
return tokens;
})();
+5
View File
@@ -2373,6 +2373,11 @@ const characterDictionaryRuntime = createCharacterDictionaryRuntimeService({
getNameMatchImagesEnabled: () => getResolvedConfig().subtitleStyle.nameMatchImagesEnabled,
getCollapsibleSectionOpenState: (section) =>
getResolvedConfig().anilist.characterDictionary.collapsibleSections[section],
tokenizeJapaneseName: async (text) => (await appState.mecabTokenizer?.tokenize(text)) ?? null,
getJapaneseNameTokenizerAvailable: () => {
const status = appState.mecabTokenizer?.getStatus();
return status?.available === true && status.enabled === true;
},
now: () => Date.now(),
logInfo: (message) => logger.info(message),
logWarn: (message) => logger.warn(message),
+25 -8
View File
@@ -38,6 +38,7 @@ import {
createCharacterDictionaryManualSelectionStore,
} from './character-dictionary-runtime/manual-selection';
import { snapshotHasCharacterNameImages } from './character-dictionary-runtime/image-lookup';
import { resolveJapaneseNameSplits } from './character-dictionary-runtime/name-split-resolver';
import type {
AniListMediaCandidate,
CharacterDictionaryBuildResult,
@@ -175,11 +176,19 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
userDataPath: deps.userDataPath,
});
const shouldRefreshCachedSnapshot = (snapshot: CharacterDictionarySnapshot): boolean => {
if (deps.getNameMatchImagesEnabled?.() !== true) {
return false;
const isNameSplitTokenizerAvailable = (): boolean =>
typeof deps.tokenizeJapaneseName === 'function' &&
deps.getJapaneseNameTokenizerAvailable?.() === true;
const getCachedSnapshotRefreshReason = (snapshot: CharacterDictionarySnapshot): string | null => {
if (deps.getNameMatchImagesEnabled?.() === true && !snapshotHasCharacterNameImages(snapshot)) {
return 'missing cached character images';
}
return !snapshotHasCharacterNameImages(snapshot);
// Heuristic name splits are upgraded once MeCab becomes available.
if (snapshot.nameSplitSource !== 'mecab' && isNameSplitTokenizerAvailable()) {
return 'name splits predate MeCab availability';
}
return null;
};
const createAniListRequestSlot = (): (() => Promise<void>) => {
@@ -323,7 +332,8 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
): Promise<CharacterDictionarySnapshotResult> => {
const snapshotPath = getSnapshotPath(outputDir, mediaId);
const cachedSnapshot = readSnapshot(snapshotPath);
if (cachedSnapshot && !shouldRefreshCachedSnapshot(cachedSnapshot)) {
const refreshReason = cachedSnapshot ? getCachedSnapshotRefreshReason(cachedSnapshot) : null;
if (cachedSnapshot && refreshReason === null) {
deps.logInfo?.(`[dictionary] snapshot hit for AniList ${mediaId}`);
return {
mediaId: cachedSnapshot.mediaId,
@@ -334,9 +344,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
};
}
if (cachedSnapshot) {
deps.logInfo?.(
`[dictionary] snapshot stale for AniList ${mediaId}: missing cached character images`,
);
deps.logInfo?.(`[dictionary] snapshot stale for AniList ${mediaId}: ${refreshReason}`);
}
progress?.onGenerating?.({
@@ -399,6 +407,13 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
}
}
const nameSplitTokenizerAvailable = isNameSplitTokenizerAvailable();
const resolvedNameSplits = nameSplitTokenizerAvailable
? await resolveJapaneseNameSplits(characters, deps.tokenizeJapaneseName!, deps.logWarn)
: undefined;
const nameSplitSource =
resolvedNameSplits && resolvedNameSplits.size > 0 ? 'mecab' : 'heuristic';
const snapshot = buildSnapshotFromCharacters(
mediaId,
fetchedMediaTitle || mediaTitleHint || `AniList ${mediaId}`,
@@ -407,6 +422,8 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
imagesByVaId,
deps.now(),
getCollapsibleSectionOpenState,
resolvedNameSplits,
nameSplitSource,
);
writeSnapshot(snapshotPath, snapshot);
deps.logInfo?.(
@@ -36,7 +36,20 @@ test('writeSnapshot persists and readSnapshot restores current-format snapshots'
writeSnapshot(snapshotPath, snapshot);
assert.deepEqual(readSnapshot(snapshotPath), snapshot);
assert.deepEqual(readSnapshot(snapshotPath), { ...snapshot, nameSplitSource: 'heuristic' });
});
test('readSnapshot preserves the mecab name-split source and defaults missing values to heuristic', () => {
const outputDir = makeTempDir();
const snapshotPath = getSnapshotPath(outputDir, 130298);
const snapshot: CharacterDictionarySnapshot = {
...createSnapshot(),
nameSplitSource: 'mecab',
};
writeSnapshot(snapshotPath, snapshot);
assert.equal(readSnapshot(snapshotPath)?.nameSplitSource, 'mecab');
});
test('readSnapshot ignores snapshots written with an older format version', () => {
@@ -141,6 +141,7 @@ export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot
mediaTitle: parsed.mediaTitle,
entryCount: parsed.entryCount,
updatedAt: parsed.updatedAt,
nameSplitSource: parsed.nameSplitSource === 'mecab' ? 'mecab' : 'heuristic',
termEntries: parsed.termEntries as CharacterDictionaryTermEntry[],
images: parsed.images as CharacterDictionarySnapshotImage[],
};
@@ -1,7 +1,7 @@
export const ANILIST_GRAPHQL_URL = 'https://graphql.anilist.co';
export const ANILIST_REQUEST_DELAY_MS = 2000;
export const CHARACTER_IMAGE_DOWNLOAD_DELAY_MS = 250;
export const CHARACTER_DICTIONARY_FORMAT_VERSION = 18;
export const CHARACTER_DICTIONARY_FORMAT_VERSION = 19;
export const CHARACTER_DICTIONARY_MERGED_TITLE = 'SubMiner Character Dictionary';
export const HONORIFIC_SUFFIXES = [
@@ -1,5 +1,5 @@
import { HONORIFIC_SUFFIXES } from './constants';
import type { JapaneseNameParts, NameReadings } from './types';
import type { JapaneseNameParts, NameReadings, ResolvedNameSplits } from './types';
export function hasKanaOnly(value: string): boolean {
return /^[\u3040-\u309f\u30a0-\u30ffー]+$/.test(value);
@@ -262,7 +262,7 @@ export function buildReadingFromRomanized(value: string): string {
return katakana ? katakanaToHiragana(katakana) : '';
}
function buildReadingFromHint(value: string): string {
export function buildReadingFromHint(value: string): string {
return buildReading(value) || buildReadingFromRomanized(value);
}
@@ -273,20 +273,22 @@ function scoreJapaneseNamePartLength(length: number): number {
return 0;
}
function inferJapaneseNameSplitIndex(
// Ranks every possible family/given boundary. Reading-length ratios cannot
// always identify the true boundary (あずま can be one kanji or two), so
// callers may take the top candidates rather than trusting only the best.
function inferJapaneseNameSplitIndices(
nameOriginal: string,
firstNameHint: string,
lastNameHint: string,
): number | null {
): number[] {
const chars = [...nameOriginal];
if (chars.length < 2) return null;
if (chars.length < 2) return [];
const familyHintLength = [...buildReadingFromHint(lastNameHint)].length;
const givenHintLength = [...buildReadingFromHint(firstNameHint)].length;
const totalHintLength = familyHintLength + givenHintLength;
const defaultBoundary = Math.round(chars.length / 2);
let bestIndex: number | null = null;
let bestScore = Number.NEGATIVE_INFINITY;
const scored: Array<{ index: number; score: number }> = [];
for (let index = 1; index < chars.length; index += 1) {
const familyLength = index;
@@ -309,13 +311,10 @@ function inferJapaneseNameSplitIndex(
score += 0.25;
}
if (score > bestScore) {
bestScore = score;
bestIndex = index;
}
scored.push({ index, score });
}
return bestIndex;
return scored.sort((left, right) => right.score - left.score).map((entry) => entry.index);
}
export function addRomanizedKanaAliases(values: Iterable<string>): string[] {
@@ -335,6 +334,7 @@ export function splitJapaneseName(
nameOriginal: string,
firstNameHint?: string,
lastNameHint?: string,
resolvedSplits?: ResolvedNameSplits,
): JapaneseNameParts {
const trimmed = nameOriginal.trim();
if (!trimmed) {
@@ -377,6 +377,22 @@ export function splitJapaneseName(
};
}
const resolvedSplit = resolvedSplits?.get(trimmed);
if (
resolvedSplit &&
resolvedSplit.family &&
resolvedSplit.given &&
`${resolvedSplit.family}${resolvedSplit.given}` === trimmed
) {
return {
hasSpace: true,
original: trimmed,
combined: trimmed,
family: resolvedSplit.family,
given: resolvedSplit.given,
};
}
const hintedFirst = firstNameHint?.trim() || '';
const hintedLast = lastNameHint?.trim() || '';
if (hintedFirst && hintedLast) {
@@ -404,7 +420,7 @@ export function splitJapaneseName(
}
if (hintedFirst && hintedLast && containsKanji(trimmed)) {
const splitIndex = inferJapaneseNameSplitIndex(trimmed, hintedFirst, hintedLast);
const splitIndex = inferJapaneseNameSplitIndices(trimmed, hintedFirst, hintedLast)[0] ?? null;
if (splitIndex != null) {
const chars = [...trimmed];
const family = chars.slice(0, splitIndex).join('');
@@ -430,11 +446,62 @@ export function splitJapaneseName(
};
}
const MAX_INFERRED_SPLIT_CANDIDATES = 2;
// Returns the possible family/given splits, best first. Only a boundary
// guessed by the length heuristic is ambiguous (あずま can be one kanji or
// two), so only that path yields a runner-up candidate; explicit separators,
// resolved (MeCab) splits, and exact hint matches are trusted as-is.
export function splitJapaneseNameCandidates(
nameOriginal: string,
firstNameHint?: string,
lastNameHint?: string,
resolvedSplits?: ResolvedNameSplits,
): JapaneseNameParts[] {
const primary = splitJapaneseName(nameOriginal, firstNameHint, lastNameHint, resolvedSplits);
if (!primary.family || !primary.given) {
return [primary];
}
const trimmed = nameOriginal.trim();
if (primary.combined !== trimmed) {
return [primary];
}
const resolvedSplit = resolvedSplits?.get(trimmed);
if (resolvedSplit && `${resolvedSplit.family}${resolvedSplit.given}` === trimmed) {
return [primary];
}
const hintedFirst = firstNameHint?.trim() || '';
const hintedLast = lastNameHint?.trim() || '';
if (`${hintedLast}${hintedFirst}` === trimmed || `${hintedFirst}${hintedLast}` === trimmed) {
return [primary];
}
const candidates = [primary];
const chars = [...trimmed];
const splitIndices = inferJapaneseNameSplitIndices(trimmed, hintedFirst, hintedLast);
for (const splitIndex of splitIndices.slice(1, MAX_INFERRED_SPLIT_CANDIDATES)) {
const family = chars.slice(0, splitIndex).join('');
const given = chars.slice(splitIndex).join('');
if (family && given) {
candidates.push({
hasSpace: true,
original: trimmed,
combined: trimmed,
family,
given,
});
}
}
return candidates;
}
export function generateNameReadings(
nameOriginal: string,
romanizedName: string,
firstNameHint?: string,
lastNameHint?: string,
resolvedSplits?: ResolvedNameSplits,
): NameReadings {
const trimmed = nameOriginal.trim();
if (!trimmed) {
@@ -447,7 +514,7 @@ export function generateNameReadings(
};
}
const nameParts = splitJapaneseName(trimmed, firstNameHint, lastNameHint);
const nameParts = splitJapaneseName(trimmed, firstNameHint, lastNameHint, resolvedSplits);
if (!nameParts.hasSpace || !nameParts.family || !nameParts.given) {
const full = containsKanji(trimmed)
? buildReadingFromRomanized(romanizedName)
@@ -0,0 +1,191 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { resolveJapaneseNameSplits } from './name-split-resolver';
import { splitJapaneseName, splitJapaneseNameCandidates } from './name-reading';
import { buildNameTerms } from './term-building';
import type { CharacterRecord, NameSplitToken } from './types';
function characterRecord(overrides: Partial<CharacterRecord>): CharacterRecord {
return {
id: 302626,
role: 'main',
firstNameHint: 'Shino',
fullName: 'Shino Azuma',
lastNameHint: 'Azuma',
nativeName: '東紫乃',
alternativeNames: [],
bloodType: '',
birthday: null,
description: '',
imageUrl: null,
age: '',
sex: '',
voiceActors: [],
...overrides,
};
}
function personNameToken(word: string, role: '姓' | '名', katakanaReading: string): NameSplitToken {
return { word, pos1: '名詞', pos2: '固有名詞', pos3: '人名', pos4: role, katakanaReading };
}
function tokenizerFor(
tokensByName: Record<string, NameSplitToken[]>,
): (text: string) => Promise<NameSplitToken[] | null> {
return async (text) => tokensByName[text] ?? null;
}
test('resolveJapaneseNameSplits splits a single-kanji surname via person-name POS tags', async () => {
const splits = await resolveJapaneseNameSplits(
[characterRecord({})],
tokenizerFor({
: [personNameToken('東', '姓', 'アズマ'), personNameToken('紫乃', '名', 'シノ')],
}),
);
assert.deepEqual(splits.get('東紫乃'), { family: '東', given: '紫乃' });
});
test('resolveJapaneseNameSplits corrects a hint-length-misleading surname boundary', async () => {
const splits = await resolveJapaneseNameSplits(
[
characterRecord({
nativeName: '渡辺真奈美',
fullName: 'Manami Watanabe',
firstNameHint: 'Manami',
lastNameHint: 'Watanabe',
}),
],
tokenizerFor({
: [
personNameToken('渡辺', '姓', 'ワタナベ'),
personNameToken('真奈美', '名', 'マナミ'),
],
}),
);
assert.deepEqual(splits.get('渡辺真奈美'), { family: '渡辺', given: '真奈美' });
});
test('resolveJapaneseNameSplits falls back to hint readings when POS tags are generic', async () => {
const splits = await resolveJapaneseNameSplits(
[
characterRecord({
nativeName: '鈴木みゆ',
fullName: 'Miyu Suzuki',
firstNameHint: 'Miyu',
lastNameHint: 'Suzuki',
}),
],
tokenizerFor({
: [
personNameToken('鈴木', '姓', 'スズキ'),
{ word: 'みゆ', pos1: '名詞', pos2: '一般', pos3: '*', pos4: '*', katakanaReading: 'ミユ' },
],
}),
);
assert.deepEqual(splits.get('鈴木みゆ'), { family: '鈴木', given: 'みゆ' });
});
test('resolveJapaneseNameSplits skips names whose tokens do not reconstruct the name', async () => {
const splits = await resolveJapaneseNameSplits(
[characterRecord({})],
tokenizerFor({
: [personNameToken('東', '姓', 'アズマ'), personNameToken('乃', '名', '')],
}),
);
assert.equal(splits.size, 0);
});
test('resolveJapaneseNameSplits skips ambiguous or untagged segmentations', async () => {
const splits = await resolveJapaneseNameSplits(
[
characterRecord({
nativeName: '担任',
fullName: 'Tannin',
firstNameHint: 'Tannin',
lastNameHint: '',
}),
],
tokenizerFor({
: [
{ word: '担', pos1: '名詞', pos2: '一般', pos3: '*', pos4: '*', katakanaReading: 'タン' },
{ word: '任', pos1: '名詞', pos2: '一般', pos3: '*', pos4: '*', katakanaReading: 'ニン' },
],
}),
);
assert.equal(splits.size, 0);
});
test('resolveJapaneseNameSplits survives tokenizer failures', async () => {
const warnings: string[] = [];
const splits = await resolveJapaneseNameSplits(
[characterRecord({})],
async () => {
throw new Error('mecab unavailable');
},
(message) => warnings.push(message),
);
assert.equal(splits.size, 0);
assert.equal(warnings.length, 1);
assert.match(warnings[0]!, /mecab unavailable/);
});
test('splitJapaneseName prefers a resolved split over hint-length inference', () => {
const resolved = new Map([['東紫乃', { family: '東', given: '紫乃' }]]);
const withResolved = splitJapaneseName('東紫乃', 'Shino', 'Azuma', resolved);
assert.equal(withResolved.family, '東');
assert.equal(withResolved.given, '紫乃');
const withoutResolved = splitJapaneseName('東紫乃', 'Shino', 'Azuma');
assert.notEqual(withoutResolved.family, '東');
});
test('splitJapaneseNameCandidates emits the runner-up boundary only for inferred splits', () => {
const inferred = splitJapaneseNameCandidates('東紫乃', 'Shino', 'Azuma');
assert.equal(inferred.length, 2);
assert.deepEqual(
inferred.map((parts) => `${parts.family}|${parts.given}`).sort(),
['東紫|乃', '東|紫乃'].sort(),
);
const resolved = new Map([['東紫乃', { family: '東', given: '紫乃' }]]);
const trusted = splitJapaneseNameCandidates('東紫乃', 'Shino', 'Azuma', resolved);
assert.equal(trusted.length, 1);
assert.equal(trusted[0]!.family, '東');
const spaced = splitJapaneseNameCandidates('須々木 心一', 'Shinichi', 'Susuki');
assert.equal(spaced.length, 1);
});
test('buildNameTerms without resolved splits still emits both candidate surnames', () => {
const terms = buildNameTerms(characterRecord({}));
assert.ok(terms.includes('東'));
assert.ok(terms.includes('紫乃'));
assert.ok(terms.includes('東紫'));
assert.ok(terms.includes('乃'));
});
test('buildNameTerms emits surname and given-name terms from resolved splits', () => {
const resolved = new Map([['渡辺真奈美', { family: '渡辺', given: '真奈美' }]]);
const terms = buildNameTerms(
characterRecord({
nativeName: '渡辺真奈美',
fullName: 'Manami Watanabe',
firstNameHint: 'Manami',
lastNameHint: 'Watanabe',
}),
resolved,
);
assert.ok(terms.includes('渡辺'));
assert.ok(terms.includes('真奈美'));
assert.ok(terms.includes('渡辺真奈美'));
assert.ok(!terms.includes('渡辺真'));
});
@@ -0,0 +1,118 @@
import { buildReading, buildReadingFromHint } from './name-reading';
import { expandRawNameVariants, isJapaneseNameSplitCandidate } from './term-building';
import type {
CharacterRecord,
NameSplitToken,
NameSplitTokenizer,
ResolvedNameSplit,
} from './types';
const NAME_SEPARATOR_PATTERN = /[\s ・・·•]/;
function joinSurfaces(tokens: NameSplitToken[]): string {
return tokens.map((token) => token.word).join('');
}
// MeCab tags dictionary-known person names as 名詞,固有名詞,人名,姓|名. A split is
// trusted only when every leading token is 姓 and every remaining token is 名.
function splitIndexFromPersonNamePos(tokens: NameSplitToken[]): number | null {
let familyEnd = 0;
while (
familyEnd < tokens.length &&
tokens[familyEnd]!.pos3 === '人名' &&
tokens[familyEnd]!.pos4 === '姓'
) {
familyEnd += 1;
}
if (familyEnd === 0 || familyEnd >= tokens.length) {
return null;
}
for (let index = familyEnd; index < tokens.length; index += 1) {
if (tokens[index]!.pos3 !== '人名' || tokens[index]!.pos4 !== '名') {
return null;
}
}
return familyEnd;
}
function splitIndexFromHintReadings(
tokens: NameSplitToken[],
familyHintReading: string,
givenHintReading: string,
): number | null {
if (!familyHintReading && !givenHintReading) {
return null;
}
const readings = tokens.map((token) => buildReading(token.katakanaReading || ''));
let matchedIndex: number | null = null;
for (let index = 1; index < tokens.length; index += 1) {
const familyReadings = readings.slice(0, index);
const givenReadings = readings.slice(index);
const familyMatches =
!!familyHintReading &&
familyReadings.every((reading) => reading.length > 0) &&
familyReadings.join('') === familyHintReading;
const givenMatches =
!!givenHintReading &&
givenReadings.every((reading) => reading.length > 0) &&
givenReadings.join('') === givenHintReading;
if (!familyMatches && !givenMatches) {
continue;
}
if (matchedIndex !== null && matchedIndex !== index) {
return null;
}
matchedIndex = index;
}
return matchedIndex;
}
function collectSplitCandidateNames(character: CharacterRecord): string[] {
const candidates = new Set<string>();
const rawNames = [character.nativeName, character.fullName, ...character.alternativeNames];
for (const rawName of rawNames) {
for (const name of expandRawNameVariants(rawName)) {
const trimmed = name.trim();
if (!trimmed || NAME_SEPARATOR_PATTERN.test(trimmed)) continue;
if (!isJapaneseNameSplitCandidate(trimmed)) continue;
if ([...trimmed].length < 2) continue;
candidates.add(trimmed);
}
}
return [...candidates];
}
export async function resolveJapaneseNameSplits(
characters: CharacterRecord[],
tokenize: NameSplitTokenizer,
logWarn?: (message: string) => void,
): Promise<Map<string, ResolvedNameSplit>> {
const splits = new Map<string, ResolvedNameSplit>();
for (const character of characters) {
const familyHintReading = buildReadingFromHint(character.lastNameHint?.trim() || '');
const givenHintReading = buildReadingFromHint(character.firstNameHint?.trim() || '');
for (const name of collectSplitCandidateNames(character)) {
if (splits.has(name)) continue;
let tokens: NameSplitToken[] | null = null;
try {
tokens = await tokenize(name);
} catch (err) {
logWarn?.(
`[dictionary] name split tokenization failed for "${name}": ${(err as Error).message}`,
);
continue;
}
if (!tokens || tokens.length < 2 || joinSurfaces(tokens) !== name) continue;
const splitIndex =
splitIndexFromPersonNamePos(tokens) ??
splitIndexFromHintReadings(tokens, familyHintReading, givenHintReading);
if (splitIndex === null) continue;
const family = joinSurfaces(tokens.slice(0, splitIndex));
const given = joinSurfaces(tokens.slice(splitIndex));
if (family && given) {
splits.set(name, { family, given });
}
}
}
return splits;
}
@@ -121,6 +121,175 @@ test('generateForCurrentMedia refreshes same-version snapshots missing images wh
}
});
test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable', async () => {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), {
...createSnapshotWithoutImages(),
nameSplitSource: 'heuristic',
});
const originalFetch = globalThis.fetch;
let characterPageRequests = 0;
globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
if (url === GRAPHQL_URL) {
const body = JSON.parse(String(init?.body ?? '{}')) as { query?: string };
if (body.query?.includes('characters(page: $page')) {
characterPageRequests += 1;
return new Response(
JSON.stringify({
data: {
Media: {
title: { english: 'The Eminence in Shadow' },
characters: {
pageInfo: { hasNextPage: false },
edges: [
{
role: 'SUPPORTING',
node: {
id: 123,
description: 'Alexia Midgar.',
image: { large: null, medium: null },
name: {
first: 'Taro',
last: 'Yamada',
full: 'Taro Yamada',
native: '山田太郎',
},
},
},
],
},
},
},
}),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
}
}
throw new Error(`Unexpected fetch URL: ${url}`);
}) as typeof globalThis.fetch;
try {
let tokenizerCalls = 0;
const runtime = createCharacterDictionaryRuntimeService({
userDataPath,
getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv',
getCurrentMediaTitle: () => 'The Eminence in Shadow - S01E05',
resolveMediaPathForJimaku: (mediaPath) => mediaPath,
guessAnilistMediaInfo: async () => ({
title: 'The Eminence in Shadow',
season: null,
episode: 5,
source: 'fallback',
}),
getNameMatchImagesEnabled: () => false,
tokenizeJapaneseName: async () => {
tokenizerCalls += 1;
return null;
},
getJapaneseNameTokenizerAvailable: () => true,
now: () => 1_700_000_000_500,
});
const result = await runtime.generateForCurrentMedia();
const refreshedSnapshot = JSON.parse(
fs.readFileSync(getSnapshotPath(outputDir, 130298), 'utf8'),
) as CharacterDictionarySnapshot;
assert.equal(result.fromCache, false);
assert.equal(refreshedSnapshot.nameSplitSource, 'heuristic');
const retriedResult = await runtime.generateForCurrentMedia();
assert.equal(retriedResult.fromCache, false);
assert.equal(characterPageRequests, 2);
assert.equal(tokenizerCalls, 2);
} finally {
globalThis.fetch = originalFetch;
}
});
test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is available', async () => {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), {
...createSnapshotWithoutImages(),
nameSplitSource: 'mecab',
});
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (input: string | URL | Request) => {
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
throw new Error(`Unexpected fetch URL: ${url}`);
}) as typeof globalThis.fetch;
try {
const runtime = createCharacterDictionaryRuntimeService({
userDataPath,
getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv',
getCurrentMediaTitle: () => 'The Eminence in Shadow - S01E05',
resolveMediaPathForJimaku: (mediaPath) => mediaPath,
guessAnilistMediaInfo: async () => ({
title: 'The Eminence in Shadow',
season: null,
episode: 5,
source: 'fallback',
}),
getNameMatchImagesEnabled: () => false,
tokenizeJapaneseName: async () => null,
getJapaneseNameTokenizerAvailable: () => true,
now: () => 1_700_000_000_500,
});
const result = await runtime.generateForCurrentMedia();
assert.equal(result.fromCache, true);
} finally {
globalThis.fetch = originalFetch;
}
});
test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is unavailable', async () => {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), {
...createSnapshotWithoutImages(),
nameSplitSource: 'heuristic',
});
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (input: string | URL | Request) => {
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
throw new Error(`Unexpected fetch URL: ${url}`);
}) as typeof globalThis.fetch;
try {
const runtime = createCharacterDictionaryRuntimeService({
userDataPath,
getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv',
getCurrentMediaTitle: () => 'The Eminence in Shadow - S01E05',
resolveMediaPathForJimaku: (mediaPath) => mediaPath,
guessAnilistMediaInfo: async () => ({
title: 'The Eminence in Shadow',
season: null,
episode: 5,
source: 'fallback',
}),
getNameMatchImagesEnabled: () => false,
tokenizeJapaneseName: async () => null,
getJapaneseNameTokenizerAvailable: () => false,
now: () => 1_700_000_000_500,
});
const result = await runtime.generateForCurrentMedia();
assert.equal(result.fromCache, true);
} finally {
globalThis.fetch = originalFetch;
}
});
test('generateForCurrentMedia keeps same-version snapshots without images when inline images are disabled', async () => {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
@@ -14,6 +14,8 @@ import type {
CharacterDictionarySnapshotImage,
CharacterDictionaryTermEntry,
CharacterRecord,
NameSplitSource,
ResolvedNameSplits,
} from './types';
export function buildSnapshotImagePath(mediaId: number, charId: number, ext: string): string {
@@ -34,6 +36,8 @@ export function buildSnapshotFromCharacters(
getCollapsibleSectionOpenState: (
section: AnilistCharacterDictionaryCollapsibleSectionKey,
) => boolean,
resolvedNameSplits?: ResolvedNameSplits,
nameSplitSource: NameSplitSource = 'heuristic',
): CharacterDictionarySnapshot {
const termEntries: CharacterDictionaryTermEntry[] = [];
@@ -45,7 +49,7 @@ export function buildSnapshotFromCharacters(
const vaImg = imagesByVaId.get(va.id);
if (vaImg) vaImagePaths.set(va.id, vaImg.path);
}
const candidateTerms = buildNameTerms(character);
const candidateTerms = buildNameTerms(character, resolvedNameSplits);
const glossary = createDefinitionGlossary(
character,
mediaId,
@@ -59,12 +63,14 @@ export function buildSnapshotFromCharacters(
character.nativeName,
character.firstNameHint,
character.lastNameHint,
resolvedNameSplits,
);
const readings = generateNameReadings(
character.nativeName,
character.fullName,
character.firstNameHint,
character.lastNameHint,
resolvedNameSplits,
);
for (const term of candidateTerms) {
if (seenTerms.has(term)) continue;
@@ -84,6 +90,7 @@ export function buildSnapshotFromCharacters(
mediaTitle,
entryCount: termEntries.length,
updatedAt,
nameSplitSource,
termEntries,
images: [...imagesByCharacterId.values(), ...imagesByVaId.values()],
};
@@ -7,6 +7,7 @@ import {
hasKanaOnly,
isRomanizedName,
splitJapaneseName,
splitJapaneseNameCandidates,
} from './name-reading';
import type {
CharacterDictionaryGlossaryEntry,
@@ -15,9 +16,10 @@ import type {
CharacterRecord,
JapaneseNameParts,
NameReadings,
ResolvedNameSplits,
} from './types';
function expandRawNameVariants(rawName: string): string[] {
export function expandRawNameVariants(rawName: string): string[] {
const trimmed = rawName.trim();
if (!trimmed) return [];
@@ -40,26 +42,41 @@ function expandRawNameVariants(rawName: string): string[] {
return [...variants];
}
function isJapaneseNameSplitCandidate(name: string): boolean {
export function isJapaneseNameSplitCandidate(name: string): boolean {
const compact = name.replace(/[\s\u3000・・·•]/g, '');
return (
containsKanji(compact) && /^[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff々〆ヵヶー]+$/.test(compact)
);
}
function addJapaneseNameParts(character: CharacterRecord, name: string, terms: Set<string>): void {
function addJapaneseNameParts(
character: CharacterRecord,
name: string,
terms: Set<string>,
resolvedSplits?: ResolvedNameSplits,
): void {
if (!isJapaneseNameSplitCandidate(name)) return;
const nameParts = splitJapaneseName(name, character.firstNameHint, character.lastNameHint);
if (nameParts.family) {
terms.add(nameParts.family);
}
if (nameParts.given) {
terms.add(nameParts.given);
const candidates = splitJapaneseNameCandidates(
name,
character.firstNameHint,
character.lastNameHint,
resolvedSplits,
);
for (const nameParts of candidates) {
if (nameParts.family) {
terms.add(nameParts.family);
}
if (nameParts.given) {
terms.add(nameParts.given);
}
}
}
export function buildNameTerms(character: CharacterRecord): string[] {
export function buildNameTerms(
character: CharacterRecord,
resolvedSplits?: ResolvedNameSplits,
): string[] {
const base = new Set<string>();
const romanizedBase = new Set<string>();
const rawNames = [character.nativeName, character.fullName, ...character.alternativeNames];
@@ -95,7 +112,7 @@ export function buildNameTerms(character: CharacterRecord): string[] {
}
if (target === base) {
addJapaneseNameParts(character, name, base);
addJapaneseNameParts(character, name, base, resolvedSplits);
}
}
}
@@ -108,6 +125,7 @@ export function buildNameTerms(character: CharacterRecord): string[] {
character.nativeName,
character.firstNameHint,
character.lastNameHint,
resolvedSplits,
);
if (nativeParts.family) {
base.add(nativeParts.family);
@@ -31,6 +31,26 @@ export type JapaneseNameParts = {
given: string | null;
};
export type ResolvedNameSplit = {
family: string;
given: string;
};
export type ResolvedNameSplits = ReadonlyMap<string, ResolvedNameSplit>;
export type NameSplitToken = {
word: string;
pos1?: string;
pos2?: string;
pos3?: string;
pos4?: string;
katakanaReading?: string;
};
export type NameSplitTokenizer = (text: string) => Promise<NameSplitToken[] | null>;
export type NameSplitSource = 'mecab' | 'heuristic';
export type NameReadings = {
hasSpace: boolean;
original: string;
@@ -45,6 +65,7 @@ export type CharacterDictionarySnapshot = {
mediaTitle: string;
entryCount: number;
updatedAt: number;
nameSplitSource?: NameSplitSource;
termEntries: CharacterDictionaryTermEntry[];
images: CharacterDictionarySnapshotImage[];
};
@@ -152,6 +173,8 @@ export interface CharacterDictionaryRuntimeDeps {
getCollapsibleSectionOpenState?: (
section: AnilistCharacterDictionaryCollapsibleSectionKey,
) => boolean;
tokenizeJapaneseName?: NameSplitTokenizer;
getJapaneseNameTokenizerAvailable?: () => boolean;
}
export type ResolvedAniListMedia = {