mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-29 19:21:33 -07:00
feat(youtube): harden media cache and drain queued cards on failure
- Add --force-ipv4 and retry flags to yt-dlp background downloads - Clean stale cache dirs on startup and before new downloads - Notify and drain pending card updates when cache download fails - Fix generateAudio/generateImage config checks (missing = enabled)
This commit is contained in:
@@ -82,6 +82,10 @@ test('YouTube media cache exposes the downloaded file after the background job c
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
assert.equal(spawnCalls[0]?.command, 'yt-dlp');
|
||||
assert.ok(spawnCalls[0]?.args.includes('--no-playlist'));
|
||||
assert.ok(spawnCalls[0]?.args.includes('--force-ipv4'));
|
||||
assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--retries') + 1], '5');
|
||||
assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--fragment-retries') + 1], '5');
|
||||
assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--extractor-retries') + 1], '5');
|
||||
assert.ok(spawnCalls[0]?.args.includes('--merge-output-format'));
|
||||
assert.equal(
|
||||
spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-f') + 1],
|
||||
@@ -107,6 +111,68 @@ test('YouTube media cache exposes the downloaded file after the background job c
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache reports failed background downloads', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const failedEvents: Array<{ url: string }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
onFailed: (event) => {
|
||||
failedEvents.push(event);
|
||||
},
|
||||
spawn: () => {
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
spawnedProcesses[0]?.emit('close', 1);
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
assert.deepEqual(failedEvents, [{ url: 'https://youtu.be/demo' }]);
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), null);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache only reports a failed download once when error is followed by close', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const failedEvents: Array<{ url: string }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
onFailed: (event) => {
|
||||
failedEvents.push(event);
|
||||
},
|
||||
spawn: () => {
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
spawnedProcesses[0]?.emit('error', new Error('spawn failed'));
|
||||
spawnedProcesses[0]?.emit('close', 1);
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
assert.deepEqual(failedEvents, [{ url: 'https://youtu.be/demo' }]);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache can disable the download height cap', () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
@@ -159,6 +225,31 @@ test('YouTube media cache applies the configured download height cap', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache removes stale files from previous runs on startup', () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const staleDir = path.join(cacheRoot, 'stale-session');
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
|
||||
try {
|
||||
fs.mkdirSync(staleDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(staleDir, 'media.mkv'), 'stale cached media');
|
||||
|
||||
createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args) => {
|
||||
spawnCalls.push({ command, args });
|
||||
return new FakeYtDlpProcess();
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(fs.existsSync(staleDir), false);
|
||||
assert.deepEqual(spawnCalls, []);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache restarts when a ready cached file was deleted externally', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
@@ -198,6 +289,32 @@ test('YouTube media cache restarts when a ready cached file was deleted external
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache removes stale disk siblings before starting a new cache', () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const staleDir = path.join(cacheRoot, 'stale-sibling');
|
||||
const spawnCalls: Array<{ command: string; args: string[] }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args) => {
|
||||
spawnCalls.push({ command, args });
|
||||
return new FakeYtDlpProcess();
|
||||
},
|
||||
});
|
||||
fs.mkdirSync(staleDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(staleDir, 'media.mkv'), 'stale cached media');
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
|
||||
assert.equal(fs.existsSync(staleDir), false);
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache drops old sessions when a new background cache starts', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
|
||||
@@ -40,6 +40,7 @@ export interface YoutubeMediaCacheServiceDeps {
|
||||
spawn?: SpawnProcess;
|
||||
onDownloadStarted?: (event: { url: string }) => void;
|
||||
onReady?: (event: { url: string; path: string }) => void;
|
||||
onFailed?: (event: { url: string }) => void;
|
||||
logInfo?: (message: string) => void;
|
||||
logWarn?: (message: string) => void;
|
||||
}
|
||||
@@ -88,6 +89,13 @@ function createYtDlpArgs(url: string, outputTemplate: string, maxHeight?: number
|
||||
return [
|
||||
'--no-playlist',
|
||||
'--no-warnings',
|
||||
'--force-ipv4',
|
||||
'--retries',
|
||||
'5',
|
||||
'--fragment-retries',
|
||||
'5',
|
||||
'--extractor-retries',
|
||||
'5',
|
||||
'-f',
|
||||
getFormatSelector(normalizeMaxHeight(maxHeight)),
|
||||
'--merge-output-format',
|
||||
@@ -109,6 +117,29 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
let activeKey: string | null = null;
|
||||
|
||||
const getSessionDir = (url: string): string => path.join(cacheRoot, cacheKeyForUrl(url));
|
||||
const removeCacheDir = (dir: string): void => {
|
||||
try {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Temp cache cleanup should not block shutdown or playback startup.
|
||||
}
|
||||
};
|
||||
const removeCacheRootEntriesExcept = (dirsToKeep: string[]): void => {
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = fs.readdirSync(cacheRoot, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const keepDirs = new Set(dirsToKeep.map((dir) => path.resolve(dir)));
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(cacheRoot, entry.name);
|
||||
if (keepDirs.has(path.resolve(entryPath))) {
|
||||
continue;
|
||||
}
|
||||
removeCacheDir(entryPath);
|
||||
}
|
||||
};
|
||||
const removeSession = (key: string): void => {
|
||||
const session = sessions.get(key);
|
||||
if (!session) {
|
||||
@@ -121,11 +152,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
if (activeKey === key) {
|
||||
activeKey = null;
|
||||
}
|
||||
try {
|
||||
fs.rmSync(session.dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Temp cache cleanup should not block shutdown or playback startup.
|
||||
}
|
||||
removeCacheDir(session.dir);
|
||||
};
|
||||
const removeInactiveSessions = (keyToKeep: string): void => {
|
||||
for (const key of [...sessions.keys()]) {
|
||||
@@ -134,6 +161,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
}
|
||||
}
|
||||
};
|
||||
removeCacheRootEntriesExcept([]);
|
||||
|
||||
const getCachedMediaPath = async (url: string): Promise<string | null> => {
|
||||
const key = cacheKeyForUrl(url);
|
||||
@@ -172,9 +200,11 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
|
||||
const key = cacheKeyForUrl(url);
|
||||
activeKey = key;
|
||||
const dir = getSessionDir(url);
|
||||
const existingSession = sessions.get(key);
|
||||
if (existingSession?.state === 'running') {
|
||||
removeInactiveSessions(key);
|
||||
removeCacheRootEntriesExcept([existingSession.dir]);
|
||||
return;
|
||||
}
|
||||
if (existingSession) {
|
||||
@@ -184,14 +214,15 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
findReadyMediaPath(existingSession.dir))
|
||||
) {
|
||||
removeInactiveSessions(key);
|
||||
removeCacheRootEntriesExcept([existingSession.dir]);
|
||||
return;
|
||||
}
|
||||
removeSession(key);
|
||||
activeKey = key;
|
||||
}
|
||||
removeInactiveSessions(key);
|
||||
removeCacheRootEntriesExcept([dir]);
|
||||
|
||||
const dir = getSessionDir(url);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const outputTemplate = path.join(dir, 'media.%(ext)s');
|
||||
const args = createYtDlpArgs(url, outputTemplate, options.maxHeight);
|
||||
@@ -219,6 +250,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
deps.onFailed?.({ url });
|
||||
});
|
||||
|
||||
child.once('close', (code) => {
|
||||
@@ -226,6 +258,9 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
if (currentSession !== session) {
|
||||
return;
|
||||
}
|
||||
if (session.state === 'failed') {
|
||||
return;
|
||||
}
|
||||
session.process = null;
|
||||
if (code === 0) {
|
||||
const readyPath = findReadyMediaPath(dir);
|
||||
@@ -239,6 +274,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
}
|
||||
session.state = 'failed';
|
||||
deps.logWarn?.(`YouTube media cache download exited without a usable media file.`);
|
||||
deps.onFailed?.({ url });
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user