mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-03-07 03:22:17 -08:00
fix: pace AniList character dictionary requests
This commit is contained in:
@@ -344,3 +344,143 @@ test('generateForCurrentMedia regenerates dictionary when cached format version
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test('generateForCurrentMedia paces AniList requests and character image downloads', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const originalFetch = globalThis.fetch;
|
||||
const sleepCalls: number[] = [];
|
||||
const imageRequests: string[] = [];
|
||||
|
||||
globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {
|
||||
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
|
||||
if (url === GRAPHQL_URL) {
|
||||
const body = JSON.parse(String(init?.body ?? '{}')) as {
|
||||
query?: string;
|
||||
};
|
||||
|
||||
if (body.query?.includes('Page(perPage: 10)')) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
data: {
|
||||
Page: {
|
||||
media: [
|
||||
{
|
||||
id: 130298,
|
||||
episodes: 20,
|
||||
title: {
|
||||
romaji: 'Kage no Jitsuryokusha ni Naritakute!',
|
||||
english: 'The Eminence in Shadow',
|
||||
native: '陰の実力者になりたくて!',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (body.query?.includes('characters(page: $page')) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
data: {
|
||||
Media: {
|
||||
title: {
|
||||
romaji: 'Kage no Jitsuryokusha ni Naritakute!',
|
||||
english: 'The Eminence in Shadow',
|
||||
native: '陰の実力者になりたくて!',
|
||||
},
|
||||
characters: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
edges: [
|
||||
{
|
||||
role: 'MAIN',
|
||||
node: {
|
||||
id: 111,
|
||||
description: 'First character.',
|
||||
image: {
|
||||
large: 'https://example.com/alpha.png',
|
||||
medium: null,
|
||||
},
|
||||
name: {
|
||||
full: 'Alpha',
|
||||
native: 'アルファ',
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
role: 'SUPPORTING',
|
||||
node: {
|
||||
id: 222,
|
||||
description: 'Second character.',
|
||||
image: {
|
||||
large: 'https://example.com/beta.png',
|
||||
medium: null,
|
||||
},
|
||||
name: {
|
||||
full: 'Beta',
|
||||
native: 'ベータ',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (url === 'https://example.com/alpha.png') {
|
||||
imageRequests.push(url);
|
||||
return new Response('missing', {
|
||||
status: 404,
|
||||
headers: { 'content-type': 'text/plain' },
|
||||
});
|
||||
}
|
||||
|
||||
if (url === 'https://example.com/beta.png') {
|
||||
imageRequests.push(url);
|
||||
return new Response(PNG_1X1, {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'image/png' },
|
||||
});
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected fetch URL: ${url}`);
|
||||
}) as typeof globalThis.fetch;
|
||||
|
||||
try {
|
||||
const runtime = createCharacterDictionaryRuntimeService({
|
||||
userDataPath,
|
||||
getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv',
|
||||
getCurrentMediaTitle: () => 'The Eminence in Shadow - S01E05',
|
||||
resolveMediaPathForJimaku: (mediaPath) => mediaPath,
|
||||
guessAnilistMediaInfo: async () => ({
|
||||
title: 'The Eminence in Shadow',
|
||||
episode: 5,
|
||||
source: 'fallback',
|
||||
}),
|
||||
now: () => 1_700_000_000_000,
|
||||
sleep: async (ms) => {
|
||||
sleepCalls.push(ms);
|
||||
},
|
||||
});
|
||||
|
||||
await runtime.generateForCurrentMedia();
|
||||
|
||||
assert.deepEqual(sleepCalls, [2000, 250]);
|
||||
assert.deepEqual(imageRequests, ['https://example.com/alpha.png', 'https://example.com/beta.png']);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -2,8 +2,11 @@ import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import type { AnilistMediaGuess } from '../core/services/anilist/anilist-updater';
|
||||
import { hasVideoExtension } from '../shared/video-extensions';
|
||||
|
||||
const ANILIST_GRAPHQL_URL = 'https://graphql.anilist.co';
|
||||
const ANILIST_REQUEST_DELAY_MS = 2000;
|
||||
const CHARACTER_IMAGE_DOWNLOAD_DELAY_MS = 250;
|
||||
const HONORIFIC_SUFFIXES = [
|
||||
'さん',
|
||||
'様',
|
||||
@@ -21,19 +24,6 @@ const HONORIFIC_SUFFIXES = [
|
||||
'社長',
|
||||
'部長',
|
||||
] as const;
|
||||
const VIDEO_EXTENSIONS = new Set([
|
||||
'.mkv',
|
||||
'.mp4',
|
||||
'.avi',
|
||||
'.webm',
|
||||
'.mov',
|
||||
'.flv',
|
||||
'.wmv',
|
||||
'.m4v',
|
||||
'.ts',
|
||||
'.m2ts',
|
||||
]);
|
||||
|
||||
type CharacterDictionaryRole = 'main' | 'primary' | 'side' | 'appears';
|
||||
|
||||
type CharacterDictionaryCacheEntry = {
|
||||
@@ -137,6 +127,7 @@ export interface CharacterDictionaryRuntimeDeps {
|
||||
mediaTitle: string | null,
|
||||
) => Promise<AnilistMediaGuess | null>;
|
||||
now: () => number;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
logInfo?: (message: string) => void;
|
||||
logWarn?: (message: string) => void;
|
||||
}
|
||||
@@ -325,8 +316,7 @@ function expandUserPath(input: string): string {
|
||||
}
|
||||
|
||||
function isVideoFile(filePath: string): boolean {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
return VIDEO_EXTENSIONS.has(ext);
|
||||
return hasVideoExtension(path.extname(filePath));
|
||||
}
|
||||
|
||||
function findFirstVideoFileInDirectory(directoryPath: string): string | null {
|
||||
@@ -604,7 +594,11 @@ function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
|
||||
async function fetchAniList<T>(
|
||||
query: string,
|
||||
variables: Record<string, unknown>,
|
||||
beforeRequest?: () => Promise<void>,
|
||||
): Promise<T> {
|
||||
if (beforeRequest) {
|
||||
await beforeRequest();
|
||||
}
|
||||
const response = await fetch(ANILIST_GRAPHQL_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
@@ -634,6 +628,7 @@ async function fetchAniList<T>(
|
||||
|
||||
async function resolveAniListMediaIdFromGuess(
|
||||
guess: AnilistMediaGuess,
|
||||
beforeRequest?: () => Promise<void>,
|
||||
): Promise<ResolvedAniListMedia> {
|
||||
const data = await fetchAniList<AniListSearchResponse>(
|
||||
`
|
||||
@@ -654,6 +649,7 @@ async function resolveAniListMediaIdFromGuess(
|
||||
{
|
||||
search: guess.title,
|
||||
},
|
||||
beforeRequest,
|
||||
);
|
||||
|
||||
const media = data.Page?.media ?? [];
|
||||
@@ -664,7 +660,10 @@ async function resolveAniListMediaIdFromGuess(
|
||||
return resolved;
|
||||
}
|
||||
|
||||
async function fetchCharactersForMedia(mediaId: number): Promise<{
|
||||
async function fetchCharactersForMedia(
|
||||
mediaId: number,
|
||||
beforeRequest?: () => Promise<void>,
|
||||
): Promise<{
|
||||
mediaTitle: string;
|
||||
characters: CharacterRecord[];
|
||||
}> {
|
||||
@@ -708,6 +707,7 @@ async function fetchCharactersForMedia(mediaId: number): Promise<{
|
||||
id: mediaId,
|
||||
page,
|
||||
},
|
||||
beforeRequest,
|
||||
);
|
||||
|
||||
const media = data.Media;
|
||||
@@ -744,7 +744,6 @@ async function fetchCharactersForMedia(mediaId: number): Promise<{
|
||||
break;
|
||||
}
|
||||
page += 1;
|
||||
await sleep(300);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -805,12 +804,22 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
} {
|
||||
const outputDir = path.join(deps.userDataPath, 'character-dictionaries');
|
||||
const cachePath = path.join(outputDir, 'cache.json');
|
||||
const sleepMs = deps.sleep ?? sleep;
|
||||
|
||||
return {
|
||||
generateForCurrentMedia: async (
|
||||
targetPath?: string,
|
||||
options?: CharacterDictionaryGenerateOptions,
|
||||
) => {
|
||||
let hasAniListRequest = false;
|
||||
const waitForAniListRequestSlot = async (): Promise<void> => {
|
||||
if (!hasAniListRequest) {
|
||||
hasAniListRequest = true;
|
||||
return;
|
||||
}
|
||||
await sleepMs(ANILIST_REQUEST_DELAY_MS);
|
||||
};
|
||||
|
||||
const dictionaryTarget = targetPath?.trim() || '';
|
||||
const guessInput =
|
||||
dictionaryTarget.length > 0
|
||||
@@ -826,7 +835,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
throw new Error('Unable to resolve current anime from media path/title.');
|
||||
}
|
||||
|
||||
const resolvedMedia = await resolveAniListMediaIdFromGuess(guessed);
|
||||
const resolvedMedia = await resolveAniListMediaIdFromGuess(guessed, waitForAniListRequestSlot);
|
||||
const cache = readCache(cachePath);
|
||||
const cached = cache.anilistById[String(resolvedMedia.id)];
|
||||
const refreshTtlMsRaw = options?.refreshTtlMs;
|
||||
@@ -859,6 +868,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
|
||||
const { mediaTitle: fetchedMediaTitle, characters } = await fetchCharactersForMedia(
|
||||
resolvedMedia.id,
|
||||
waitForAniListRequestSlot,
|
||||
);
|
||||
if (characters.length === 0) {
|
||||
throw new Error(`No characters returned for AniList media ${resolvedMedia.id}.`);
|
||||
@@ -870,9 +880,14 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
[];
|
||||
const seen = new Set<string>();
|
||||
|
||||
let hasAttemptedCharacterImageDownload = false;
|
||||
for (const character of characters) {
|
||||
let imagePath: string | null = null;
|
||||
if (character.imageUrl) {
|
||||
if (hasAttemptedCharacterImageDownload) {
|
||||
await sleepMs(CHARACTER_IMAGE_DOWNLOAD_DELAY_MS);
|
||||
}
|
||||
hasAttemptedCharacterImageDownload = true;
|
||||
const image = await downloadCharacterImage(character.imageUrl, character.id);
|
||||
if (image) {
|
||||
imagePath = `img/${image.filename}`;
|
||||
|
||||
Reference in New Issue
Block a user