mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-27 04:49:49 -07:00
fix(overlay): resolve unspaced Japanese name splits and scan recovery (#146)
This commit is contained in:
@@ -150,7 +150,8 @@ function hasAnyAnnotationEnabled(options: TokenizerAnnotationOptions): boolean {
|
||||
options.knownWordsEnabled ||
|
||||
options.nPlusOneEnabled ||
|
||||
options.jlptEnabled ||
|
||||
options.frequencyEnabled
|
||||
options.frequencyEnabled ||
|
||||
options.nameMatchEnabled
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -33,6 +33,50 @@ function makeDeps(overrides: Partial<AnnotationStageDeps> = {}): AnnotationStage
|
||||
};
|
||||
}
|
||||
|
||||
test('annotateTokens keeps name matches on tokens the POS noise filter would strip', () => {
|
||||
// MeCab tags 平 as 接頭詞 in contexts like あっ 平 これ…, which is in the
|
||||
// POS1 exclusion list; a confirmed character-name match must survive it.
|
||||
const tokens = [
|
||||
makeToken({
|
||||
surface: '平',
|
||||
headword: '平',
|
||||
reading: 'たいら',
|
||||
pos1: '接頭詞',
|
||||
isNameMatch: true,
|
||||
}),
|
||||
makeToken({
|
||||
surface: '平',
|
||||
headword: '平',
|
||||
reading: 'ひら',
|
||||
pos1: '接頭詞',
|
||||
isNameMatch: false,
|
||||
jlptLevel: 'N1',
|
||||
}),
|
||||
];
|
||||
|
||||
const result = annotateTokens(tokens, makeDeps(), { nameMatchEnabled: true });
|
||||
|
||||
assert.equal(result[0]?.isNameMatch, true);
|
||||
assert.equal(result[1]?.isNameMatch, false);
|
||||
assert.equal(result[1]?.jlptLevel, undefined);
|
||||
});
|
||||
|
||||
test('annotateTokens strips name matches from POS-excluded tokens when name matching is disabled', () => {
|
||||
const tokens = [
|
||||
makeToken({
|
||||
surface: '平',
|
||||
headword: '平',
|
||||
reading: 'たいら',
|
||||
pos1: '接頭詞',
|
||||
isNameMatch: true,
|
||||
}),
|
||||
];
|
||||
|
||||
const result = annotateTokens(tokens, makeDeps(), { nameMatchEnabled: false });
|
||||
|
||||
assert.equal(result[0]?.isNameMatch, false);
|
||||
});
|
||||
|
||||
test('annotateTokens known-word match mode uses headword vs surface', () => {
|
||||
const tokens = [makeToken({ surface: '食べた', headword: '食べる', reading: 'タベタ' })];
|
||||
const isKnownWord = (text: string): boolean => text === '食べる';
|
||||
@@ -1994,7 +2038,8 @@ test('annotateTokens keeps known status while clearing other annotations for aru
|
||||
assert.equal(result[0]?.headword, '有る');
|
||||
assert.equal(result[0]?.isKnown, true);
|
||||
assert.equal(result[0]?.isNPlusOneTarget, false);
|
||||
assert.equal(result[0]?.isNameMatch, false);
|
||||
// Name matches take precedence over the annotation noise filter.
|
||||
assert.equal(result[0]?.isNameMatch, true);
|
||||
assert.equal(result[0]?.frequencyRank, undefined);
|
||||
assert.equal(result[0]?.jlptLevel, undefined);
|
||||
});
|
||||
|
||||
@@ -778,7 +778,13 @@ export function annotateTokens(
|
||||
: false;
|
||||
nPlusOneKnownStatuses[index] = isKnownForMatching;
|
||||
|
||||
const prioritizedNameMatch = nameMatchEnabled && token.isNameMatch === true;
|
||||
|
||||
// A confirmed character-name match must survive the POS noise filter:
|
||||
// MeCab can tag a name like 平 as a prefix (接頭詞) depending on context,
|
||||
// which would otherwise strip the name match and its portrait.
|
||||
if (
|
||||
!prioritizedNameMatch &&
|
||||
sharedShouldExcludeTokenFromSubtitleAnnotations(token, {
|
||||
pos1Exclusions,
|
||||
pos2Exclusions,
|
||||
@@ -794,8 +800,6 @@ export function annotateTokens(
|
||||
};
|
||||
}
|
||||
|
||||
const prioritizedNameMatch = nameMatchEnabled && token.isNameMatch === true;
|
||||
|
||||
const frequencyRank =
|
||||
frequencyEnabled && !prioritizedNameMatch
|
||||
? filterTokenFrequencyRank(token, pos1Exclusions, pos2Exclusions)
|
||||
|
||||
@@ -975,6 +975,105 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF
|
||||
]);
|
||||
});
|
||||
|
||||
test('requestYomitanScanTokens retries shorter windows when a greedy match has no exact-source headword', async () => {
|
||||
let scannerScript = '';
|
||||
const deps = createDeps(async (script) => {
|
||||
if (script.includes('termsFind')) {
|
||||
scannerScript = script;
|
||||
return [];
|
||||
}
|
||||
if (script.includes('optionsGetFull')) {
|
||||
return {
|
||||
profileCurrent: 0,
|
||||
profileIndex: 0,
|
||||
scanLength: 40,
|
||||
dictionaries: ['JMdict'],
|
||||
dictionaryPriorityByName: { JMdict: 0 },
|
||||
dictionaryFrequencyModeByName: {},
|
||||
profiles: [
|
||||
{
|
||||
options: {
|
||||
scanning: { length: 40 },
|
||||
dictionaries: [{ name: 'JMdict', enabled: true, id: 0 }],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return null;
|
||||
});
|
||||
|
||||
await requestYomitanScanTokens('平 (平)', deps, {
|
||||
error: () => undefined,
|
||||
});
|
||||
|
||||
const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
|
||||
if (action !== 'termsFind') {
|
||||
throw new Error(`unexpected action: ${action}`);
|
||||
}
|
||||
|
||||
const text = (params as { text?: string } | undefined)?.text ?? '';
|
||||
if (!text.startsWith('平')) {
|
||||
return { originalTextLength: 0, dictionaryEntries: [] };
|
||||
}
|
||||
if (text.length >= 4) {
|
||||
// Simulates Yomitan normalization consuming punctuation/whitespace:
|
||||
// the greedy match spans 平 (平 but no headword source equals it.
|
||||
return {
|
||||
originalTextLength: 4,
|
||||
dictionaryEntries: [
|
||||
{
|
||||
headwords: [
|
||||
{
|
||||
term: '平々',
|
||||
reading: 'へいへい',
|
||||
sources: [{ originalText: '平平', isPrimary: true, matchType: 'exact' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return {
|
||||
originalTextLength: 1,
|
||||
dictionaryEntries: [
|
||||
{
|
||||
headwords: [
|
||||
{
|
||||
term: '平',
|
||||
reading: 'たいら',
|
||||
sources: [{ originalText: '平', isPrimary: true, matchType: 'exact' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
|
||||
assert.deepEqual(result, [
|
||||
{
|
||||
surface: '平',
|
||||
reading: 'たいら',
|
||||
headword: '平',
|
||||
headwordReading: 'たいら',
|
||||
startPos: 0,
|
||||
endPos: 1,
|
||||
isNameMatch: false,
|
||||
frequencyRank: undefined,
|
||||
},
|
||||
{
|
||||
surface: '平',
|
||||
reading: 'たいら',
|
||||
headword: '平',
|
||||
headwordReading: 'たいら',
|
||||
startPos: 3,
|
||||
endPos: 4,
|
||||
isNameMatch: false,
|
||||
frequencyRank: undefined,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('requestYomitanScanTokens emits complete readings for kanji-kana compounds', async () => {
|
||||
let scannerScript = '';
|
||||
const deps = createDeps(async (script) => {
|
||||
|
||||
@@ -1297,47 +1297,69 @@ ${YOMITAN_SCANNING_HELPERS}
|
||||
const text = ${JSON.stringify(text)};
|
||||
const details = {matchType: "exact", deinflect: true};
|
||||
const tokens = [];
|
||||
let i = 0;
|
||||
while (i < text.length) {
|
||||
const codePoint = text.codePointAt(i);
|
||||
async function findTokenAt(position, windowLength) {
|
||||
const codePoint = text.codePointAt(position);
|
||||
const character = String.fromCodePoint(codePoint);
|
||||
const substring = text.substring(i, i + ${scanLength});
|
||||
const substring = text.substring(position, position + windowLength);
|
||||
const result = await invoke("termsFind", { text: substring, details, optionsContext: { index: ${profileIndex} } });
|
||||
const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
|
||||
const originalTextLength = typeof result?.originalTextLength === "number" ? result.originalTextLength : 0;
|
||||
if (dictionaryEntries.length > 0 && originalTextLength > 0 && (originalTextLength !== character.length || isCodePointJapanese(codePoint))) {
|
||||
const source = substring.substring(0, originalTextLength);
|
||||
const preferredHeadword = getPreferredHeadword(
|
||||
dictionaryEntries,
|
||||
source,
|
||||
dictionaryPriorityByName,
|
||||
dictionaryFrequencyModeByName
|
||||
);
|
||||
if (preferredHeadword && typeof preferredHeadword.term === "string") {
|
||||
const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
|
||||
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
|
||||
const tokenPayload = {
|
||||
surface: segments.map((segment) => segment.text).join("") || source,
|
||||
reading: segments.map(getSegmentReadingContribution).join(""),
|
||||
headword: preferredHeadword.term,
|
||||
headwordReading: reading || undefined,
|
||||
startPos: i,
|
||||
endPos: i + originalTextLength,
|
||||
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
|
||||
frequencyRank:
|
||||
typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
|
||||
? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
|
||||
: undefined,
|
||||
};
|
||||
if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
|
||||
tokenPayload.wordClasses = preferredHeadword.wordClasses;
|
||||
}
|
||||
tokens.push(tokenPayload);
|
||||
i += originalTextLength;
|
||||
continue;
|
||||
}
|
||||
if (dictionaryEntries.length === 0 || originalTextLength <= 0 || (originalTextLength === character.length && !isCodePointJapanese(codePoint))) {
|
||||
return { token: null, matchedLength: 0 };
|
||||
}
|
||||
i += character.length;
|
||||
const source = substring.substring(0, originalTextLength);
|
||||
const preferredHeadword = getPreferredHeadword(
|
||||
dictionaryEntries,
|
||||
source,
|
||||
dictionaryPriorityByName,
|
||||
dictionaryFrequencyModeByName
|
||||
);
|
||||
if (!preferredHeadword || typeof preferredHeadword.term !== "string") {
|
||||
return { token: null, matchedLength: originalTextLength };
|
||||
}
|
||||
const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
|
||||
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
|
||||
const tokenPayload = {
|
||||
surface: segments.map((segment) => segment.text).join("") || source,
|
||||
reading: segments.map(getSegmentReadingContribution).join(""),
|
||||
headword: preferredHeadword.term,
|
||||
headwordReading: reading || undefined,
|
||||
startPos: position,
|
||||
endPos: position + originalTextLength,
|
||||
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
|
||||
frequencyRank:
|
||||
typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
|
||||
? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
|
||||
: undefined,
|
||||
};
|
||||
if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
|
||||
tokenPayload.wordClasses = preferredHeadword.wordClasses;
|
||||
}
|
||||
return { token: tokenPayload, matchedLength: originalTextLength };
|
||||
}
|
||||
let i = 0;
|
||||
while (i < text.length) {
|
||||
let attempt = await findTokenAt(i, ${scanLength});
|
||||
// Yomitan text normalization can consume characters (whitespace,
|
||||
// punctuation) beyond the matched term, leaving no headword whose
|
||||
// source equals the consumed text. Retry with shorter windows so a
|
||||
// valid prefix term (e.g. a character name before a paren) still
|
||||
// tokenizes instead of the position being skipped.
|
||||
let retryLength = Math.min(attempt.matchedLength, ${scanLength}) - 1;
|
||||
while (!attempt.token && retryLength >= 1) {
|
||||
const retry = await findTokenAt(i, retryLength);
|
||||
if (retry.token) {
|
||||
attempt = retry;
|
||||
break;
|
||||
}
|
||||
retryLength = Math.min(retryLength - 1, retry.matchedLength - 1);
|
||||
}
|
||||
if (attempt.token) {
|
||||
tokens.push(attempt.token);
|
||||
i += attempt.matchedLength;
|
||||
continue;
|
||||
}
|
||||
i += String.fromCodePoint(text.codePointAt(i)).length;
|
||||
}
|
||||
return tokens;
|
||||
})();
|
||||
|
||||
@@ -2373,6 +2373,11 @@ const characterDictionaryRuntime = createCharacterDictionaryRuntimeService({
|
||||
getNameMatchImagesEnabled: () => getResolvedConfig().subtitleStyle.nameMatchImagesEnabled,
|
||||
getCollapsibleSectionOpenState: (section) =>
|
||||
getResolvedConfig().anilist.characterDictionary.collapsibleSections[section],
|
||||
tokenizeJapaneseName: async (text) => (await appState.mecabTokenizer?.tokenize(text)) ?? null,
|
||||
getJapaneseNameTokenizerAvailable: () => {
|
||||
const status = appState.mecabTokenizer?.getStatus();
|
||||
return status?.available === true && status.enabled === true;
|
||||
},
|
||||
now: () => Date.now(),
|
||||
logInfo: (message) => logger.info(message),
|
||||
logWarn: (message) => logger.warn(message),
|
||||
|
||||
@@ -38,6 +38,7 @@ import {
|
||||
createCharacterDictionaryManualSelectionStore,
|
||||
} from './character-dictionary-runtime/manual-selection';
|
||||
import { snapshotHasCharacterNameImages } from './character-dictionary-runtime/image-lookup';
|
||||
import { resolveJapaneseNameSplits } from './character-dictionary-runtime/name-split-resolver';
|
||||
import type {
|
||||
AniListMediaCandidate,
|
||||
CharacterDictionaryBuildResult,
|
||||
@@ -175,11 +176,19 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
userDataPath: deps.userDataPath,
|
||||
});
|
||||
|
||||
const shouldRefreshCachedSnapshot = (snapshot: CharacterDictionarySnapshot): boolean => {
|
||||
if (deps.getNameMatchImagesEnabled?.() !== true) {
|
||||
return false;
|
||||
const isNameSplitTokenizerAvailable = (): boolean =>
|
||||
typeof deps.tokenizeJapaneseName === 'function' &&
|
||||
deps.getJapaneseNameTokenizerAvailable?.() === true;
|
||||
|
||||
const getCachedSnapshotRefreshReason = (snapshot: CharacterDictionarySnapshot): string | null => {
|
||||
if (deps.getNameMatchImagesEnabled?.() === true && !snapshotHasCharacterNameImages(snapshot)) {
|
||||
return 'missing cached character images';
|
||||
}
|
||||
return !snapshotHasCharacterNameImages(snapshot);
|
||||
// Heuristic name splits are upgraded once MeCab becomes available.
|
||||
if (snapshot.nameSplitSource !== 'mecab' && isNameSplitTokenizerAvailable()) {
|
||||
return 'name splits predate MeCab availability';
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const createAniListRequestSlot = (): (() => Promise<void>) => {
|
||||
@@ -323,7 +332,8 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
): Promise<CharacterDictionarySnapshotResult> => {
|
||||
const snapshotPath = getSnapshotPath(outputDir, mediaId);
|
||||
const cachedSnapshot = readSnapshot(snapshotPath);
|
||||
if (cachedSnapshot && !shouldRefreshCachedSnapshot(cachedSnapshot)) {
|
||||
const refreshReason = cachedSnapshot ? getCachedSnapshotRefreshReason(cachedSnapshot) : null;
|
||||
if (cachedSnapshot && refreshReason === null) {
|
||||
deps.logInfo?.(`[dictionary] snapshot hit for AniList ${mediaId}`);
|
||||
return {
|
||||
mediaId: cachedSnapshot.mediaId,
|
||||
@@ -334,9 +344,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
};
|
||||
}
|
||||
if (cachedSnapshot) {
|
||||
deps.logInfo?.(
|
||||
`[dictionary] snapshot stale for AniList ${mediaId}: missing cached character images`,
|
||||
);
|
||||
deps.logInfo?.(`[dictionary] snapshot stale for AniList ${mediaId}: ${refreshReason}`);
|
||||
}
|
||||
|
||||
progress?.onGenerating?.({
|
||||
@@ -399,6 +407,13 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
}
|
||||
}
|
||||
|
||||
const nameSplitTokenizerAvailable = isNameSplitTokenizerAvailable();
|
||||
const resolvedNameSplits = nameSplitTokenizerAvailable
|
||||
? await resolveJapaneseNameSplits(characters, deps.tokenizeJapaneseName!, deps.logWarn)
|
||||
: undefined;
|
||||
const nameSplitSource =
|
||||
resolvedNameSplits && resolvedNameSplits.size > 0 ? 'mecab' : 'heuristic';
|
||||
|
||||
const snapshot = buildSnapshotFromCharacters(
|
||||
mediaId,
|
||||
fetchedMediaTitle || mediaTitleHint || `AniList ${mediaId}`,
|
||||
@@ -407,6 +422,8 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
imagesByVaId,
|
||||
deps.now(),
|
||||
getCollapsibleSectionOpenState,
|
||||
resolvedNameSplits,
|
||||
nameSplitSource,
|
||||
);
|
||||
writeSnapshot(snapshotPath, snapshot);
|
||||
deps.logInfo?.(
|
||||
|
||||
@@ -36,7 +36,20 @@ test('writeSnapshot persists and readSnapshot restores current-format snapshots'
|
||||
|
||||
writeSnapshot(snapshotPath, snapshot);
|
||||
|
||||
assert.deepEqual(readSnapshot(snapshotPath), snapshot);
|
||||
assert.deepEqual(readSnapshot(snapshotPath), { ...snapshot, nameSplitSource: 'heuristic' });
|
||||
});
|
||||
|
||||
test('readSnapshot preserves the mecab name-split source and defaults missing values to heuristic', () => {
|
||||
const outputDir = makeTempDir();
|
||||
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
||||
const snapshot: CharacterDictionarySnapshot = {
|
||||
...createSnapshot(),
|
||||
nameSplitSource: 'mecab',
|
||||
};
|
||||
|
||||
writeSnapshot(snapshotPath, snapshot);
|
||||
|
||||
assert.equal(readSnapshot(snapshotPath)?.nameSplitSource, 'mecab');
|
||||
});
|
||||
|
||||
test('readSnapshot ignores snapshots written with an older format version', () => {
|
||||
|
||||
@@ -141,6 +141,7 @@ export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot
|
||||
mediaTitle: parsed.mediaTitle,
|
||||
entryCount: parsed.entryCount,
|
||||
updatedAt: parsed.updatedAt,
|
||||
nameSplitSource: parsed.nameSplitSource === 'mecab' ? 'mecab' : 'heuristic',
|
||||
termEntries: parsed.termEntries as CharacterDictionaryTermEntry[],
|
||||
images: parsed.images as CharacterDictionarySnapshotImage[],
|
||||
};
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
export const ANILIST_GRAPHQL_URL = 'https://graphql.anilist.co';
|
||||
export const ANILIST_REQUEST_DELAY_MS = 2000;
|
||||
export const CHARACTER_IMAGE_DOWNLOAD_DELAY_MS = 250;
|
||||
export const CHARACTER_DICTIONARY_FORMAT_VERSION = 18;
|
||||
export const CHARACTER_DICTIONARY_FORMAT_VERSION = 19;
|
||||
export const CHARACTER_DICTIONARY_MERGED_TITLE = 'SubMiner Character Dictionary';
|
||||
|
||||
export const HONORIFIC_SUFFIXES = [
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { HONORIFIC_SUFFIXES } from './constants';
|
||||
import type { JapaneseNameParts, NameReadings } from './types';
|
||||
import type { JapaneseNameParts, NameReadings, ResolvedNameSplits } from './types';
|
||||
|
||||
export function hasKanaOnly(value: string): boolean {
|
||||
return /^[\u3040-\u309f\u30a0-\u30ffー]+$/.test(value);
|
||||
@@ -262,7 +262,7 @@ export function buildReadingFromRomanized(value: string): string {
|
||||
return katakana ? katakanaToHiragana(katakana) : '';
|
||||
}
|
||||
|
||||
function buildReadingFromHint(value: string): string {
|
||||
export function buildReadingFromHint(value: string): string {
|
||||
return buildReading(value) || buildReadingFromRomanized(value);
|
||||
}
|
||||
|
||||
@@ -273,20 +273,22 @@ function scoreJapaneseNamePartLength(length: number): number {
|
||||
return 0;
|
||||
}
|
||||
|
||||
function inferJapaneseNameSplitIndex(
|
||||
// Ranks every possible family/given boundary. Reading-length ratios cannot
|
||||
// always identify the true boundary (あずま can be one kanji or two), so
|
||||
// callers may take the top candidates rather than trusting only the best.
|
||||
function inferJapaneseNameSplitIndices(
|
||||
nameOriginal: string,
|
||||
firstNameHint: string,
|
||||
lastNameHint: string,
|
||||
): number | null {
|
||||
): number[] {
|
||||
const chars = [...nameOriginal];
|
||||
if (chars.length < 2) return null;
|
||||
if (chars.length < 2) return [];
|
||||
|
||||
const familyHintLength = [...buildReadingFromHint(lastNameHint)].length;
|
||||
const givenHintLength = [...buildReadingFromHint(firstNameHint)].length;
|
||||
const totalHintLength = familyHintLength + givenHintLength;
|
||||
const defaultBoundary = Math.round(chars.length / 2);
|
||||
let bestIndex: number | null = null;
|
||||
let bestScore = Number.NEGATIVE_INFINITY;
|
||||
const scored: Array<{ index: number; score: number }> = [];
|
||||
|
||||
for (let index = 1; index < chars.length; index += 1) {
|
||||
const familyLength = index;
|
||||
@@ -309,13 +311,10 @@ function inferJapaneseNameSplitIndex(
|
||||
score += 0.25;
|
||||
}
|
||||
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
bestIndex = index;
|
||||
}
|
||||
scored.push({ index, score });
|
||||
}
|
||||
|
||||
return bestIndex;
|
||||
return scored.sort((left, right) => right.score - left.score).map((entry) => entry.index);
|
||||
}
|
||||
|
||||
export function addRomanizedKanaAliases(values: Iterable<string>): string[] {
|
||||
@@ -335,6 +334,7 @@ export function splitJapaneseName(
|
||||
nameOriginal: string,
|
||||
firstNameHint?: string,
|
||||
lastNameHint?: string,
|
||||
resolvedSplits?: ResolvedNameSplits,
|
||||
): JapaneseNameParts {
|
||||
const trimmed = nameOriginal.trim();
|
||||
if (!trimmed) {
|
||||
@@ -377,6 +377,22 @@ export function splitJapaneseName(
|
||||
};
|
||||
}
|
||||
|
||||
const resolvedSplit = resolvedSplits?.get(trimmed);
|
||||
if (
|
||||
resolvedSplit &&
|
||||
resolvedSplit.family &&
|
||||
resolvedSplit.given &&
|
||||
`${resolvedSplit.family}${resolvedSplit.given}` === trimmed
|
||||
) {
|
||||
return {
|
||||
hasSpace: true,
|
||||
original: trimmed,
|
||||
combined: trimmed,
|
||||
family: resolvedSplit.family,
|
||||
given: resolvedSplit.given,
|
||||
};
|
||||
}
|
||||
|
||||
const hintedFirst = firstNameHint?.trim() || '';
|
||||
const hintedLast = lastNameHint?.trim() || '';
|
||||
if (hintedFirst && hintedLast) {
|
||||
@@ -404,7 +420,7 @@ export function splitJapaneseName(
|
||||
}
|
||||
|
||||
if (hintedFirst && hintedLast && containsKanji(trimmed)) {
|
||||
const splitIndex = inferJapaneseNameSplitIndex(trimmed, hintedFirst, hintedLast);
|
||||
const splitIndex = inferJapaneseNameSplitIndices(trimmed, hintedFirst, hintedLast)[0] ?? null;
|
||||
if (splitIndex != null) {
|
||||
const chars = [...trimmed];
|
||||
const family = chars.slice(0, splitIndex).join('');
|
||||
@@ -430,11 +446,62 @@ export function splitJapaneseName(
|
||||
};
|
||||
}
|
||||
|
||||
const MAX_INFERRED_SPLIT_CANDIDATES = 2;
|
||||
|
||||
// Returns the possible family/given splits, best first. Only a boundary
|
||||
// guessed by the length heuristic is ambiguous (あずま can be one kanji or
|
||||
// two), so only that path yields a runner-up candidate; explicit separators,
|
||||
// resolved (MeCab) splits, and exact hint matches are trusted as-is.
|
||||
export function splitJapaneseNameCandidates(
|
||||
nameOriginal: string,
|
||||
firstNameHint?: string,
|
||||
lastNameHint?: string,
|
||||
resolvedSplits?: ResolvedNameSplits,
|
||||
): JapaneseNameParts[] {
|
||||
const primary = splitJapaneseName(nameOriginal, firstNameHint, lastNameHint, resolvedSplits);
|
||||
if (!primary.family || !primary.given) {
|
||||
return [primary];
|
||||
}
|
||||
|
||||
const trimmed = nameOriginal.trim();
|
||||
if (primary.combined !== trimmed) {
|
||||
return [primary];
|
||||
}
|
||||
const resolvedSplit = resolvedSplits?.get(trimmed);
|
||||
if (resolvedSplit && `${resolvedSplit.family}${resolvedSplit.given}` === trimmed) {
|
||||
return [primary];
|
||||
}
|
||||
const hintedFirst = firstNameHint?.trim() || '';
|
||||
const hintedLast = lastNameHint?.trim() || '';
|
||||
if (`${hintedLast}${hintedFirst}` === trimmed || `${hintedFirst}${hintedLast}` === trimmed) {
|
||||
return [primary];
|
||||
}
|
||||
|
||||
const candidates = [primary];
|
||||
const chars = [...trimmed];
|
||||
const splitIndices = inferJapaneseNameSplitIndices(trimmed, hintedFirst, hintedLast);
|
||||
for (const splitIndex of splitIndices.slice(1, MAX_INFERRED_SPLIT_CANDIDATES)) {
|
||||
const family = chars.slice(0, splitIndex).join('');
|
||||
const given = chars.slice(splitIndex).join('');
|
||||
if (family && given) {
|
||||
candidates.push({
|
||||
hasSpace: true,
|
||||
original: trimmed,
|
||||
combined: trimmed,
|
||||
family,
|
||||
given,
|
||||
});
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
export function generateNameReadings(
|
||||
nameOriginal: string,
|
||||
romanizedName: string,
|
||||
firstNameHint?: string,
|
||||
lastNameHint?: string,
|
||||
resolvedSplits?: ResolvedNameSplits,
|
||||
): NameReadings {
|
||||
const trimmed = nameOriginal.trim();
|
||||
if (!trimmed) {
|
||||
@@ -447,7 +514,7 @@ export function generateNameReadings(
|
||||
};
|
||||
}
|
||||
|
||||
const nameParts = splitJapaneseName(trimmed, firstNameHint, lastNameHint);
|
||||
const nameParts = splitJapaneseName(trimmed, firstNameHint, lastNameHint, resolvedSplits);
|
||||
if (!nameParts.hasSpace || !nameParts.family || !nameParts.given) {
|
||||
const full = containsKanji(trimmed)
|
||||
? buildReadingFromRomanized(romanizedName)
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { resolveJapaneseNameSplits } from './name-split-resolver';
|
||||
import { splitJapaneseName, splitJapaneseNameCandidates } from './name-reading';
|
||||
import { buildNameTerms } from './term-building';
|
||||
import type { CharacterRecord, NameSplitToken } from './types';
|
||||
|
||||
function characterRecord(overrides: Partial<CharacterRecord>): CharacterRecord {
|
||||
return {
|
||||
id: 302626,
|
||||
role: 'main',
|
||||
firstNameHint: 'Shino',
|
||||
fullName: 'Shino Azuma',
|
||||
lastNameHint: 'Azuma',
|
||||
nativeName: '東紫乃',
|
||||
alternativeNames: [],
|
||||
bloodType: '',
|
||||
birthday: null,
|
||||
description: '',
|
||||
imageUrl: null,
|
||||
age: '',
|
||||
sex: '',
|
||||
voiceActors: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function personNameToken(word: string, role: '姓' | '名', katakanaReading: string): NameSplitToken {
|
||||
return { word, pos1: '名詞', pos2: '固有名詞', pos3: '人名', pos4: role, katakanaReading };
|
||||
}
|
||||
|
||||
function tokenizerFor(
|
||||
tokensByName: Record<string, NameSplitToken[]>,
|
||||
): (text: string) => Promise<NameSplitToken[] | null> {
|
||||
return async (text) => tokensByName[text] ?? null;
|
||||
}
|
||||
|
||||
test('resolveJapaneseNameSplits splits a single-kanji surname via person-name POS tags', async () => {
|
||||
const splits = await resolveJapaneseNameSplits(
|
||||
[characterRecord({})],
|
||||
tokenizerFor({
|
||||
東紫乃: [personNameToken('東', '姓', 'アズマ'), personNameToken('紫乃', '名', 'シノ')],
|
||||
}),
|
||||
);
|
||||
|
||||
assert.deepEqual(splits.get('東紫乃'), { family: '東', given: '紫乃' });
|
||||
});
|
||||
|
||||
test('resolveJapaneseNameSplits corrects a hint-length-misleading surname boundary', async () => {
|
||||
const splits = await resolveJapaneseNameSplits(
|
||||
[
|
||||
characterRecord({
|
||||
nativeName: '渡辺真奈美',
|
||||
fullName: 'Manami Watanabe',
|
||||
firstNameHint: 'Manami',
|
||||
lastNameHint: 'Watanabe',
|
||||
}),
|
||||
],
|
||||
tokenizerFor({
|
||||
渡辺真奈美: [
|
||||
personNameToken('渡辺', '姓', 'ワタナベ'),
|
||||
personNameToken('真奈美', '名', 'マナミ'),
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
assert.deepEqual(splits.get('渡辺真奈美'), { family: '渡辺', given: '真奈美' });
|
||||
});
|
||||
|
||||
test('resolveJapaneseNameSplits falls back to hint readings when POS tags are generic', async () => {
|
||||
const splits = await resolveJapaneseNameSplits(
|
||||
[
|
||||
characterRecord({
|
||||
nativeName: '鈴木みゆ',
|
||||
fullName: 'Miyu Suzuki',
|
||||
firstNameHint: 'Miyu',
|
||||
lastNameHint: 'Suzuki',
|
||||
}),
|
||||
],
|
||||
tokenizerFor({
|
||||
鈴木みゆ: [
|
||||
personNameToken('鈴木', '姓', 'スズキ'),
|
||||
{ word: 'みゆ', pos1: '名詞', pos2: '一般', pos3: '*', pos4: '*', katakanaReading: 'ミユ' },
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
assert.deepEqual(splits.get('鈴木みゆ'), { family: '鈴木', given: 'みゆ' });
|
||||
});
|
||||
|
||||
test('resolveJapaneseNameSplits skips names whose tokens do not reconstruct the name', async () => {
|
||||
const splits = await resolveJapaneseNameSplits(
|
||||
[characterRecord({})],
|
||||
tokenizerFor({
|
||||
東紫乃: [personNameToken('東', '姓', 'アズマ'), personNameToken('乃', '名', 'ノ')],
|
||||
}),
|
||||
);
|
||||
|
||||
assert.equal(splits.size, 0);
|
||||
});
|
||||
|
||||
test('resolveJapaneseNameSplits skips ambiguous or untagged segmentations', async () => {
|
||||
const splits = await resolveJapaneseNameSplits(
|
||||
[
|
||||
characterRecord({
|
||||
nativeName: '担任',
|
||||
fullName: 'Tannin',
|
||||
firstNameHint: 'Tannin',
|
||||
lastNameHint: '',
|
||||
}),
|
||||
],
|
||||
tokenizerFor({
|
||||
担任: [
|
||||
{ word: '担', pos1: '名詞', pos2: '一般', pos3: '*', pos4: '*', katakanaReading: 'タン' },
|
||||
{ word: '任', pos1: '名詞', pos2: '一般', pos3: '*', pos4: '*', katakanaReading: 'ニン' },
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
assert.equal(splits.size, 0);
|
||||
});
|
||||
|
||||
test('resolveJapaneseNameSplits survives tokenizer failures', async () => {
|
||||
const warnings: string[] = [];
|
||||
const splits = await resolveJapaneseNameSplits(
|
||||
[characterRecord({})],
|
||||
async () => {
|
||||
throw new Error('mecab unavailable');
|
||||
},
|
||||
(message) => warnings.push(message),
|
||||
);
|
||||
|
||||
assert.equal(splits.size, 0);
|
||||
assert.equal(warnings.length, 1);
|
||||
assert.match(warnings[0]!, /mecab unavailable/);
|
||||
});
|
||||
|
||||
test('splitJapaneseName prefers a resolved split over hint-length inference', () => {
|
||||
const resolved = new Map([['東紫乃', { family: '東', given: '紫乃' }]]);
|
||||
|
||||
const withResolved = splitJapaneseName('東紫乃', 'Shino', 'Azuma', resolved);
|
||||
assert.equal(withResolved.family, '東');
|
||||
assert.equal(withResolved.given, '紫乃');
|
||||
|
||||
const withoutResolved = splitJapaneseName('東紫乃', 'Shino', 'Azuma');
|
||||
assert.notEqual(withoutResolved.family, '東');
|
||||
});
|
||||
|
||||
test('splitJapaneseNameCandidates emits the runner-up boundary only for inferred splits', () => {
|
||||
const inferred = splitJapaneseNameCandidates('東紫乃', 'Shino', 'Azuma');
|
||||
assert.equal(inferred.length, 2);
|
||||
assert.deepEqual(
|
||||
inferred.map((parts) => `${parts.family}|${parts.given}`).sort(),
|
||||
['東紫|乃', '東|紫乃'].sort(),
|
||||
);
|
||||
|
||||
const resolved = new Map([['東紫乃', { family: '東', given: '紫乃' }]]);
|
||||
const trusted = splitJapaneseNameCandidates('東紫乃', 'Shino', 'Azuma', resolved);
|
||||
assert.equal(trusted.length, 1);
|
||||
assert.equal(trusted[0]!.family, '東');
|
||||
|
||||
const spaced = splitJapaneseNameCandidates('須々木 心一', 'Shinichi', 'Susuki');
|
||||
assert.equal(spaced.length, 1);
|
||||
});
|
||||
|
||||
test('buildNameTerms without resolved splits still emits both candidate surnames', () => {
|
||||
const terms = buildNameTerms(characterRecord({}));
|
||||
|
||||
assert.ok(terms.includes('東'));
|
||||
assert.ok(terms.includes('紫乃'));
|
||||
assert.ok(terms.includes('東紫'));
|
||||
assert.ok(terms.includes('乃'));
|
||||
});
|
||||
|
||||
test('buildNameTerms emits surname and given-name terms from resolved splits', () => {
|
||||
const resolved = new Map([['渡辺真奈美', { family: '渡辺', given: '真奈美' }]]);
|
||||
const terms = buildNameTerms(
|
||||
characterRecord({
|
||||
nativeName: '渡辺真奈美',
|
||||
fullName: 'Manami Watanabe',
|
||||
firstNameHint: 'Manami',
|
||||
lastNameHint: 'Watanabe',
|
||||
}),
|
||||
resolved,
|
||||
);
|
||||
|
||||
assert.ok(terms.includes('渡辺'));
|
||||
assert.ok(terms.includes('真奈美'));
|
||||
assert.ok(terms.includes('渡辺真奈美'));
|
||||
assert.ok(!terms.includes('渡辺真'));
|
||||
});
|
||||
@@ -0,0 +1,118 @@
|
||||
import { buildReading, buildReadingFromHint } from './name-reading';
|
||||
import { expandRawNameVariants, isJapaneseNameSplitCandidate } from './term-building';
|
||||
import type {
|
||||
CharacterRecord,
|
||||
NameSplitToken,
|
||||
NameSplitTokenizer,
|
||||
ResolvedNameSplit,
|
||||
} from './types';
|
||||
|
||||
const NAME_SEPARATOR_PATTERN = /[\s ・・·•]/;
|
||||
|
||||
function joinSurfaces(tokens: NameSplitToken[]): string {
|
||||
return tokens.map((token) => token.word).join('');
|
||||
}
|
||||
|
||||
// MeCab tags dictionary-known person names as 名詞,固有名詞,人名,姓|名. A split is
|
||||
// trusted only when every leading token is 姓 and every remaining token is 名.
|
||||
function splitIndexFromPersonNamePos(tokens: NameSplitToken[]): number | null {
|
||||
let familyEnd = 0;
|
||||
while (
|
||||
familyEnd < tokens.length &&
|
||||
tokens[familyEnd]!.pos3 === '人名' &&
|
||||
tokens[familyEnd]!.pos4 === '姓'
|
||||
) {
|
||||
familyEnd += 1;
|
||||
}
|
||||
if (familyEnd === 0 || familyEnd >= tokens.length) {
|
||||
return null;
|
||||
}
|
||||
for (let index = familyEnd; index < tokens.length; index += 1) {
|
||||
if (tokens[index]!.pos3 !== '人名' || tokens[index]!.pos4 !== '名') {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return familyEnd;
|
||||
}
|
||||
|
||||
function splitIndexFromHintReadings(
|
||||
tokens: NameSplitToken[],
|
||||
familyHintReading: string,
|
||||
givenHintReading: string,
|
||||
): number | null {
|
||||
if (!familyHintReading && !givenHintReading) {
|
||||
return null;
|
||||
}
|
||||
const readings = tokens.map((token) => buildReading(token.katakanaReading || ''));
|
||||
let matchedIndex: number | null = null;
|
||||
for (let index = 1; index < tokens.length; index += 1) {
|
||||
const familyReadings = readings.slice(0, index);
|
||||
const givenReadings = readings.slice(index);
|
||||
const familyMatches =
|
||||
!!familyHintReading &&
|
||||
familyReadings.every((reading) => reading.length > 0) &&
|
||||
familyReadings.join('') === familyHintReading;
|
||||
const givenMatches =
|
||||
!!givenHintReading &&
|
||||
givenReadings.every((reading) => reading.length > 0) &&
|
||||
givenReadings.join('') === givenHintReading;
|
||||
if (!familyMatches && !givenMatches) {
|
||||
continue;
|
||||
}
|
||||
if (matchedIndex !== null && matchedIndex !== index) {
|
||||
return null;
|
||||
}
|
||||
matchedIndex = index;
|
||||
}
|
||||
return matchedIndex;
|
||||
}
|
||||
|
||||
function collectSplitCandidateNames(character: CharacterRecord): string[] {
|
||||
const candidates = new Set<string>();
|
||||
const rawNames = [character.nativeName, character.fullName, ...character.alternativeNames];
|
||||
for (const rawName of rawNames) {
|
||||
for (const name of expandRawNameVariants(rawName)) {
|
||||
const trimmed = name.trim();
|
||||
if (!trimmed || NAME_SEPARATOR_PATTERN.test(trimmed)) continue;
|
||||
if (!isJapaneseNameSplitCandidate(trimmed)) continue;
|
||||
if ([...trimmed].length < 2) continue;
|
||||
candidates.add(trimmed);
|
||||
}
|
||||
}
|
||||
return [...candidates];
|
||||
}
|
||||
|
||||
export async function resolveJapaneseNameSplits(
|
||||
characters: CharacterRecord[],
|
||||
tokenize: NameSplitTokenizer,
|
||||
logWarn?: (message: string) => void,
|
||||
): Promise<Map<string, ResolvedNameSplit>> {
|
||||
const splits = new Map<string, ResolvedNameSplit>();
|
||||
for (const character of characters) {
|
||||
const familyHintReading = buildReadingFromHint(character.lastNameHint?.trim() || '');
|
||||
const givenHintReading = buildReadingFromHint(character.firstNameHint?.trim() || '');
|
||||
for (const name of collectSplitCandidateNames(character)) {
|
||||
if (splits.has(name)) continue;
|
||||
let tokens: NameSplitToken[] | null = null;
|
||||
try {
|
||||
tokens = await tokenize(name);
|
||||
} catch (err) {
|
||||
logWarn?.(
|
||||
`[dictionary] name split tokenization failed for "${name}": ${(err as Error).message}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!tokens || tokens.length < 2 || joinSurfaces(tokens) !== name) continue;
|
||||
const splitIndex =
|
||||
splitIndexFromPersonNamePos(tokens) ??
|
||||
splitIndexFromHintReadings(tokens, familyHintReading, givenHintReading);
|
||||
if (splitIndex === null) continue;
|
||||
const family = joinSurfaces(tokens.slice(0, splitIndex));
|
||||
const given = joinSurfaces(tokens.slice(splitIndex));
|
||||
if (family && given) {
|
||||
splits.set(name, { family, given });
|
||||
}
|
||||
}
|
||||
}
|
||||
return splits;
|
||||
}
|
||||
@@ -121,6 +121,175 @@ test('generateForCurrentMedia refreshes same-version snapshots missing images wh
|
||||
}
|
||||
});
|
||||
|
||||
test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||
writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
||||
...createSnapshotWithoutImages(),
|
||||
nameSplitSource: 'heuristic',
|
||||
});
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
let characterPageRequests = 0;
|
||||
globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {
|
||||
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
|
||||
if (url === GRAPHQL_URL) {
|
||||
const body = JSON.parse(String(init?.body ?? '{}')) as { query?: string };
|
||||
if (body.query?.includes('characters(page: $page')) {
|
||||
characterPageRequests += 1;
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
data: {
|
||||
Media: {
|
||||
title: { english: 'The Eminence in Shadow' },
|
||||
characters: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
edges: [
|
||||
{
|
||||
role: 'SUPPORTING',
|
||||
node: {
|
||||
id: 123,
|
||||
description: 'Alexia Midgar.',
|
||||
image: { large: null, medium: null },
|
||||
name: {
|
||||
first: 'Taro',
|
||||
last: 'Yamada',
|
||||
full: 'Taro Yamada',
|
||||
native: '山田太郎',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { 'content-type': 'application/json' } },
|
||||
);
|
||||
}
|
||||
}
|
||||
throw new Error(`Unexpected fetch URL: ${url}`);
|
||||
}) as typeof globalThis.fetch;
|
||||
|
||||
try {
|
||||
let tokenizerCalls = 0;
|
||||
const runtime = createCharacterDictionaryRuntimeService({
|
||||
userDataPath,
|
||||
getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv',
|
||||
getCurrentMediaTitle: () => 'The Eminence in Shadow - S01E05',
|
||||
resolveMediaPathForJimaku: (mediaPath) => mediaPath,
|
||||
guessAnilistMediaInfo: async () => ({
|
||||
title: 'The Eminence in Shadow',
|
||||
season: null,
|
||||
episode: 5,
|
||||
source: 'fallback',
|
||||
}),
|
||||
getNameMatchImagesEnabled: () => false,
|
||||
tokenizeJapaneseName: async () => {
|
||||
tokenizerCalls += 1;
|
||||
return null;
|
||||
},
|
||||
getJapaneseNameTokenizerAvailable: () => true,
|
||||
now: () => 1_700_000_000_500,
|
||||
});
|
||||
|
||||
const result = await runtime.generateForCurrentMedia();
|
||||
const refreshedSnapshot = JSON.parse(
|
||||
fs.readFileSync(getSnapshotPath(outputDir, 130298), 'utf8'),
|
||||
) as CharacterDictionarySnapshot;
|
||||
|
||||
assert.equal(result.fromCache, false);
|
||||
assert.equal(refreshedSnapshot.nameSplitSource, 'heuristic');
|
||||
|
||||
const retriedResult = await runtime.generateForCurrentMedia();
|
||||
assert.equal(retriedResult.fromCache, false);
|
||||
assert.equal(characterPageRequests, 2);
|
||||
assert.equal(tokenizerCalls, 2);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is available', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||
writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
||||
...createSnapshotWithoutImages(),
|
||||
nameSplitSource: 'mecab',
|
||||
});
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = (async (input: string | URL | Request) => {
|
||||
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
|
||||
throw new Error(`Unexpected fetch URL: ${url}`);
|
||||
}) as typeof globalThis.fetch;
|
||||
|
||||
try {
|
||||
const runtime = createCharacterDictionaryRuntimeService({
|
||||
userDataPath,
|
||||
getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv',
|
||||
getCurrentMediaTitle: () => 'The Eminence in Shadow - S01E05',
|
||||
resolveMediaPathForJimaku: (mediaPath) => mediaPath,
|
||||
guessAnilistMediaInfo: async () => ({
|
||||
title: 'The Eminence in Shadow',
|
||||
season: null,
|
||||
episode: 5,
|
||||
source: 'fallback',
|
||||
}),
|
||||
getNameMatchImagesEnabled: () => false,
|
||||
tokenizeJapaneseName: async () => null,
|
||||
getJapaneseNameTokenizerAvailable: () => true,
|
||||
now: () => 1_700_000_000_500,
|
||||
});
|
||||
|
||||
const result = await runtime.generateForCurrentMedia();
|
||||
|
||||
assert.equal(result.fromCache, true);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is unavailable', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||
writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
||||
...createSnapshotWithoutImages(),
|
||||
nameSplitSource: 'heuristic',
|
||||
});
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = (async (input: string | URL | Request) => {
|
||||
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
|
||||
throw new Error(`Unexpected fetch URL: ${url}`);
|
||||
}) as typeof globalThis.fetch;
|
||||
|
||||
try {
|
||||
const runtime = createCharacterDictionaryRuntimeService({
|
||||
userDataPath,
|
||||
getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv',
|
||||
getCurrentMediaTitle: () => 'The Eminence in Shadow - S01E05',
|
||||
resolveMediaPathForJimaku: (mediaPath) => mediaPath,
|
||||
guessAnilistMediaInfo: async () => ({
|
||||
title: 'The Eminence in Shadow',
|
||||
season: null,
|
||||
episode: 5,
|
||||
source: 'fallback',
|
||||
}),
|
||||
getNameMatchImagesEnabled: () => false,
|
||||
tokenizeJapaneseName: async () => null,
|
||||
getJapaneseNameTokenizerAvailable: () => false,
|
||||
now: () => 1_700_000_000_500,
|
||||
});
|
||||
|
||||
const result = await runtime.generateForCurrentMedia();
|
||||
|
||||
assert.equal(result.fromCache, true);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test('generateForCurrentMedia keeps same-version snapshots without images when inline images are disabled', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||
|
||||
@@ -14,6 +14,8 @@ import type {
|
||||
CharacterDictionarySnapshotImage,
|
||||
CharacterDictionaryTermEntry,
|
||||
CharacterRecord,
|
||||
NameSplitSource,
|
||||
ResolvedNameSplits,
|
||||
} from './types';
|
||||
|
||||
export function buildSnapshotImagePath(mediaId: number, charId: number, ext: string): string {
|
||||
@@ -34,6 +36,8 @@ export function buildSnapshotFromCharacters(
|
||||
getCollapsibleSectionOpenState: (
|
||||
section: AnilistCharacterDictionaryCollapsibleSectionKey,
|
||||
) => boolean,
|
||||
resolvedNameSplits?: ResolvedNameSplits,
|
||||
nameSplitSource: NameSplitSource = 'heuristic',
|
||||
): CharacterDictionarySnapshot {
|
||||
const termEntries: CharacterDictionaryTermEntry[] = [];
|
||||
|
||||
@@ -45,7 +49,7 @@ export function buildSnapshotFromCharacters(
|
||||
const vaImg = imagesByVaId.get(va.id);
|
||||
if (vaImg) vaImagePaths.set(va.id, vaImg.path);
|
||||
}
|
||||
const candidateTerms = buildNameTerms(character);
|
||||
const candidateTerms = buildNameTerms(character, resolvedNameSplits);
|
||||
const glossary = createDefinitionGlossary(
|
||||
character,
|
||||
mediaId,
|
||||
@@ -59,12 +63,14 @@ export function buildSnapshotFromCharacters(
|
||||
character.nativeName,
|
||||
character.firstNameHint,
|
||||
character.lastNameHint,
|
||||
resolvedNameSplits,
|
||||
);
|
||||
const readings = generateNameReadings(
|
||||
character.nativeName,
|
||||
character.fullName,
|
||||
character.firstNameHint,
|
||||
character.lastNameHint,
|
||||
resolvedNameSplits,
|
||||
);
|
||||
for (const term of candidateTerms) {
|
||||
if (seenTerms.has(term)) continue;
|
||||
@@ -84,6 +90,7 @@ export function buildSnapshotFromCharacters(
|
||||
mediaTitle,
|
||||
entryCount: termEntries.length,
|
||||
updatedAt,
|
||||
nameSplitSource,
|
||||
termEntries,
|
||||
images: [...imagesByCharacterId.values(), ...imagesByVaId.values()],
|
||||
};
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
hasKanaOnly,
|
||||
isRomanizedName,
|
||||
splitJapaneseName,
|
||||
splitJapaneseNameCandidates,
|
||||
} from './name-reading';
|
||||
import type {
|
||||
CharacterDictionaryGlossaryEntry,
|
||||
@@ -15,9 +16,10 @@ import type {
|
||||
CharacterRecord,
|
||||
JapaneseNameParts,
|
||||
NameReadings,
|
||||
ResolvedNameSplits,
|
||||
} from './types';
|
||||
|
||||
function expandRawNameVariants(rawName: string): string[] {
|
||||
export function expandRawNameVariants(rawName: string): string[] {
|
||||
const trimmed = rawName.trim();
|
||||
if (!trimmed) return [];
|
||||
|
||||
@@ -40,26 +42,41 @@ function expandRawNameVariants(rawName: string): string[] {
|
||||
return [...variants];
|
||||
}
|
||||
|
||||
function isJapaneseNameSplitCandidate(name: string): boolean {
|
||||
export function isJapaneseNameSplitCandidate(name: string): boolean {
|
||||
const compact = name.replace(/[\s\u3000・・·•]/g, '');
|
||||
return (
|
||||
containsKanji(compact) && /^[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff々〆ヵヶー]+$/.test(compact)
|
||||
);
|
||||
}
|
||||
|
||||
function addJapaneseNameParts(character: CharacterRecord, name: string, terms: Set<string>): void {
|
||||
function addJapaneseNameParts(
|
||||
character: CharacterRecord,
|
||||
name: string,
|
||||
terms: Set<string>,
|
||||
resolvedSplits?: ResolvedNameSplits,
|
||||
): void {
|
||||
if (!isJapaneseNameSplitCandidate(name)) return;
|
||||
|
||||
const nameParts = splitJapaneseName(name, character.firstNameHint, character.lastNameHint);
|
||||
if (nameParts.family) {
|
||||
terms.add(nameParts.family);
|
||||
}
|
||||
if (nameParts.given) {
|
||||
terms.add(nameParts.given);
|
||||
const candidates = splitJapaneseNameCandidates(
|
||||
name,
|
||||
character.firstNameHint,
|
||||
character.lastNameHint,
|
||||
resolvedSplits,
|
||||
);
|
||||
for (const nameParts of candidates) {
|
||||
if (nameParts.family) {
|
||||
terms.add(nameParts.family);
|
||||
}
|
||||
if (nameParts.given) {
|
||||
terms.add(nameParts.given);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildNameTerms(character: CharacterRecord): string[] {
|
||||
export function buildNameTerms(
|
||||
character: CharacterRecord,
|
||||
resolvedSplits?: ResolvedNameSplits,
|
||||
): string[] {
|
||||
const base = new Set<string>();
|
||||
const romanizedBase = new Set<string>();
|
||||
const rawNames = [character.nativeName, character.fullName, ...character.alternativeNames];
|
||||
@@ -95,7 +112,7 @@ export function buildNameTerms(character: CharacterRecord): string[] {
|
||||
}
|
||||
|
||||
if (target === base) {
|
||||
addJapaneseNameParts(character, name, base);
|
||||
addJapaneseNameParts(character, name, base, resolvedSplits);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -108,6 +125,7 @@ export function buildNameTerms(character: CharacterRecord): string[] {
|
||||
character.nativeName,
|
||||
character.firstNameHint,
|
||||
character.lastNameHint,
|
||||
resolvedSplits,
|
||||
);
|
||||
if (nativeParts.family) {
|
||||
base.add(nativeParts.family);
|
||||
|
||||
@@ -31,6 +31,26 @@ export type JapaneseNameParts = {
|
||||
given: string | null;
|
||||
};
|
||||
|
||||
export type ResolvedNameSplit = {
|
||||
family: string;
|
||||
given: string;
|
||||
};
|
||||
|
||||
export type ResolvedNameSplits = ReadonlyMap<string, ResolvedNameSplit>;
|
||||
|
||||
export type NameSplitToken = {
|
||||
word: string;
|
||||
pos1?: string;
|
||||
pos2?: string;
|
||||
pos3?: string;
|
||||
pos4?: string;
|
||||
katakanaReading?: string;
|
||||
};
|
||||
|
||||
export type NameSplitTokenizer = (text: string) => Promise<NameSplitToken[] | null>;
|
||||
|
||||
export type NameSplitSource = 'mecab' | 'heuristic';
|
||||
|
||||
export type NameReadings = {
|
||||
hasSpace: boolean;
|
||||
original: string;
|
||||
@@ -45,6 +65,7 @@ export type CharacterDictionarySnapshot = {
|
||||
mediaTitle: string;
|
||||
entryCount: number;
|
||||
updatedAt: number;
|
||||
nameSplitSource?: NameSplitSource;
|
||||
termEntries: CharacterDictionaryTermEntry[];
|
||||
images: CharacterDictionarySnapshotImage[];
|
||||
};
|
||||
@@ -152,6 +173,8 @@ export interface CharacterDictionaryRuntimeDeps {
|
||||
getCollapsibleSectionOpenState?: (
|
||||
section: AnilistCharacterDictionaryCollapsibleSectionKey,
|
||||
) => boolean;
|
||||
tokenizeJapaneseName?: NameSplitTokenizer;
|
||||
getJapaneseNameTokenizerAvailable?: () => boolean;
|
||||
}
|
||||
|
||||
export type ResolvedAniListMedia = {
|
||||
|
||||
Reference in New Issue
Block a user