fix: pace AniList character dictionary requests

This commit is contained in:
2026-03-05 23:57:38 -08:00
parent 72b18110b5
commit ac4fd60098
7 changed files with 269 additions and 32 deletions

View File

@@ -344,3 +344,143 @@ test('generateForCurrentMedia regenerates dictionary when cached format version
globalThis.fetch = originalFetch;
}
});
test('generateForCurrentMedia paces AniList requests and character image downloads', async () => {
const userDataPath = makeTempDir();
const originalFetch = globalThis.fetch;
const sleepCalls: number[] = [];
const imageRequests: string[] = [];
globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
if (url === GRAPHQL_URL) {
const body = JSON.parse(String(init?.body ?? '{}')) as {
query?: string;
};
if (body.query?.includes('Page(perPage: 10)')) {
return new Response(
JSON.stringify({
data: {
Page: {
media: [
{
id: 130298,
episodes: 20,
title: {
romaji: 'Kage no Jitsuryokusha ni Naritakute!',
english: 'The Eminence in Shadow',
native: '陰の実力者になりたくて!',
},
},
],
},
},
}),
{
status: 200,
headers: { 'content-type': 'application/json' },
},
);
}
if (body.query?.includes('characters(page: $page')) {
return new Response(
JSON.stringify({
data: {
Media: {
title: {
romaji: 'Kage no Jitsuryokusha ni Naritakute!',
english: 'The Eminence in Shadow',
native: '陰の実力者になりたくて!',
},
characters: {
pageInfo: { hasNextPage: false },
edges: [
{
role: 'MAIN',
node: {
id: 111,
description: 'First character.',
image: {
large: 'https://example.com/alpha.png',
medium: null,
},
name: {
full: 'Alpha',
native: 'アルファ',
},
},
},
{
role: 'SUPPORTING',
node: {
id: 222,
description: 'Second character.',
image: {
large: 'https://example.com/beta.png',
medium: null,
},
name: {
full: 'Beta',
native: 'ベータ',
},
},
},
],
},
},
},
}),
{
status: 200,
headers: { 'content-type': 'application/json' },
},
);
}
}
if (url === 'https://example.com/alpha.png') {
imageRequests.push(url);
return new Response('missing', {
status: 404,
headers: { 'content-type': 'text/plain' },
});
}
if (url === 'https://example.com/beta.png') {
imageRequests.push(url);
return new Response(PNG_1X1, {
status: 200,
headers: { 'content-type': 'image/png' },
});
}
throw new Error(`Unexpected fetch URL: ${url}`);
}) as typeof globalThis.fetch;
try {
const runtime = createCharacterDictionaryRuntimeService({
userDataPath,
getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv',
getCurrentMediaTitle: () => 'The Eminence in Shadow - S01E05',
resolveMediaPathForJimaku: (mediaPath) => mediaPath,
guessAnilistMediaInfo: async () => ({
title: 'The Eminence in Shadow',
episode: 5,
source: 'fallback',
}),
now: () => 1_700_000_000_000,
sleep: async (ms) => {
sleepCalls.push(ms);
},
});
await runtime.generateForCurrentMedia();
assert.deepEqual(sleepCalls, [2000, 250]);
assert.deepEqual(imageRequests, ['https://example.com/alpha.png', 'https://example.com/beta.png']);
} finally {
globalThis.fetch = originalFetch;
}
});

View File

@@ -2,8 +2,11 @@ import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import type { AnilistMediaGuess } from '../core/services/anilist/anilist-updater';
import { hasVideoExtension } from '../shared/video-extensions';
const ANILIST_GRAPHQL_URL = 'https://graphql.anilist.co';
const ANILIST_REQUEST_DELAY_MS = 2000;
const CHARACTER_IMAGE_DOWNLOAD_DELAY_MS = 250;
const HONORIFIC_SUFFIXES = [
'さん',
'様',
@@ -21,19 +24,6 @@ const HONORIFIC_SUFFIXES = [
'社長',
'部長',
] as const;
const VIDEO_EXTENSIONS = new Set([
'.mkv',
'.mp4',
'.avi',
'.webm',
'.mov',
'.flv',
'.wmv',
'.m4v',
'.ts',
'.m2ts',
]);
type CharacterDictionaryRole = 'main' | 'primary' | 'side' | 'appears';
type CharacterDictionaryCacheEntry = {
@@ -137,6 +127,7 @@ export interface CharacterDictionaryRuntimeDeps {
mediaTitle: string | null,
) => Promise<AnilistMediaGuess | null>;
now: () => number;
sleep?: (ms: number) => Promise<void>;
logInfo?: (message: string) => void;
logWarn?: (message: string) => void;
}
@@ -325,8 +316,7 @@ function expandUserPath(input: string): string {
}
function isVideoFile(filePath: string): boolean {
const ext = path.extname(filePath).toLowerCase();
return VIDEO_EXTENSIONS.has(ext);
return hasVideoExtension(path.extname(filePath));
}
function findFirstVideoFileInDirectory(directoryPath: string): string | null {
@@ -604,7 +594,11 @@ function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
async function fetchAniList<T>(
query: string,
variables: Record<string, unknown>,
beforeRequest?: () => Promise<void>,
): Promise<T> {
if (beforeRequest) {
await beforeRequest();
}
const response = await fetch(ANILIST_GRAPHQL_URL, {
method: 'POST',
headers: {
@@ -634,6 +628,7 @@ async function fetchAniList<T>(
async function resolveAniListMediaIdFromGuess(
guess: AnilistMediaGuess,
beforeRequest?: () => Promise<void>,
): Promise<ResolvedAniListMedia> {
const data = await fetchAniList<AniListSearchResponse>(
`
@@ -654,6 +649,7 @@ async function resolveAniListMediaIdFromGuess(
{
search: guess.title,
},
beforeRequest,
);
const media = data.Page?.media ?? [];
@@ -664,7 +660,10 @@ async function resolveAniListMediaIdFromGuess(
return resolved;
}
async function fetchCharactersForMedia(mediaId: number): Promise<{
async function fetchCharactersForMedia(
mediaId: number,
beforeRequest?: () => Promise<void>,
): Promise<{
mediaTitle: string;
characters: CharacterRecord[];
}> {
@@ -708,6 +707,7 @@ async function fetchCharactersForMedia(mediaId: number): Promise<{
id: mediaId,
page,
},
beforeRequest,
);
const media = data.Media;
@@ -744,7 +744,6 @@ async function fetchCharactersForMedia(mediaId: number): Promise<{
break;
}
page += 1;
await sleep(300);
}
return {
@@ -805,12 +804,22 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
} {
const outputDir = path.join(deps.userDataPath, 'character-dictionaries');
const cachePath = path.join(outputDir, 'cache.json');
const sleepMs = deps.sleep ?? sleep;
return {
generateForCurrentMedia: async (
targetPath?: string,
options?: CharacterDictionaryGenerateOptions,
) => {
let hasAniListRequest = false;
const waitForAniListRequestSlot = async (): Promise<void> => {
if (!hasAniListRequest) {
hasAniListRequest = true;
return;
}
await sleepMs(ANILIST_REQUEST_DELAY_MS);
};
const dictionaryTarget = targetPath?.trim() || '';
const guessInput =
dictionaryTarget.length > 0
@@ -826,7 +835,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
throw new Error('Unable to resolve current anime from media path/title.');
}
const resolvedMedia = await resolveAniListMediaIdFromGuess(guessed);
const resolvedMedia = await resolveAniListMediaIdFromGuess(guessed, waitForAniListRequestSlot);
const cache = readCache(cachePath);
const cached = cache.anilistById[String(resolvedMedia.id)];
const refreshTtlMsRaw = options?.refreshTtlMs;
@@ -859,6 +868,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
const { mediaTitle: fetchedMediaTitle, characters } = await fetchCharactersForMedia(
resolvedMedia.id,
waitForAniListRequestSlot,
);
if (characters.length === 0) {
throw new Error(`No characters returned for AniList media ${resolvedMedia.id}.`);
@@ -870,9 +880,14 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
[];
const seen = new Set<string>();
let hasAttemptedCharacterImageDownload = false;
for (const character of characters) {
let imagePath: string | null = null;
if (character.imageUrl) {
if (hasAttemptedCharacterImageDownload) {
await sleepMs(CHARACTER_IMAGE_DOWNLOAD_DELAY_MS);
}
hasAttemptedCharacterImageDownload = true;
const image = await downloadCharacterImage(character.imageUrl, character.id);
if (image) {
imagePath = `img/${image.filename}`;

View File

@@ -0,0 +1,17 @@
export const VIDEO_EXTENSIONS = new Set([
'mkv',
'mp4',
'avi',
'webm',
'mov',
'flv',
'wmv',
'm4v',
'ts',
'm2ts',
]);
export function hasVideoExtension(value: string): boolean {
const normalized = value.trim().toLowerCase().replace(/^\./, '');
return normalized.length > 0 && VIDEO_EXTENSIONS.has(normalized);
}