mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-07 13:08:54 -07:00
feat(youtube): add mediaCache mode and safer stream media extraction (#130)
This commit is contained in:
@@ -40,6 +40,11 @@ export interface AnkiJimakuIpcRuntimeOptions {
|
||||
getAnkiIntegration: () => AnkiIntegration | null;
|
||||
setAnkiIntegration: (integration: AnkiIntegration | null) => void;
|
||||
getKnownWordCacheStatePath: () => string;
|
||||
getCachedMediaPath?: (
|
||||
currentVideoPath: string,
|
||||
kind: 'audio' | 'video',
|
||||
) => Promise<string | null>;
|
||||
shouldRequireRemoteMediaCache?: () => boolean;
|
||||
showDesktopNotification: (title: string, options: { body?: string; icon?: string }) => void;
|
||||
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
|
||||
createFieldGroupingCallback: () => (
|
||||
@@ -107,6 +112,8 @@ export function registerAnkiJimakuIpcRuntime(
|
||||
mergeAiConfig(config.ai, config.ankiConnect?.ai) as AiConfig,
|
||||
undefined,
|
||||
options.showOverlayNotification,
|
||||
options.getCachedMediaPath,
|
||||
options.shouldRequireRemoteMediaCache,
|
||||
);
|
||||
integration.start();
|
||||
options.setAnkiIntegration(integration);
|
||||
|
||||
@@ -25,6 +25,11 @@ type CreateAnkiIntegrationArgs = {
|
||||
data: KikuFieldGroupingRequestData,
|
||||
) => Promise<KikuFieldGroupingChoice>;
|
||||
knownWordCacheStatePath: string;
|
||||
getCachedMediaPath?: (
|
||||
currentVideoPath: string,
|
||||
kind: 'audio' | 'video',
|
||||
) => Promise<string | null>;
|
||||
shouldRequireRemoteMediaCache?: () => boolean;
|
||||
};
|
||||
|
||||
export type OverlayWindowTrackerOptions = {
|
||||
@@ -65,6 +70,8 @@ function createDefaultAnkiIntegration(args: CreateAnkiIntegrationArgs): AnkiInte
|
||||
args.aiConfig,
|
||||
undefined,
|
||||
args.showOverlayNotification,
|
||||
args.getCachedMediaPath,
|
||||
args.shouldRequireRemoteMediaCache,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -132,6 +139,11 @@ export function initializeOverlayRuntime(
|
||||
data: KikuFieldGroupingRequestData,
|
||||
) => Promise<KikuFieldGroupingChoice>;
|
||||
getKnownWordCacheStatePath: () => string;
|
||||
getCachedMediaPath?: (
|
||||
currentVideoPath: string,
|
||||
kind: 'audio' | 'video',
|
||||
) => Promise<string | null>;
|
||||
shouldRequireRemoteMediaCache?: () => boolean;
|
||||
shouldStartAnkiIntegration?: () => boolean;
|
||||
createAnkiIntegration?: (args: CreateAnkiIntegrationArgs) => AnkiIntegrationLike;
|
||||
backendOverride: string | null;
|
||||
@@ -166,6 +178,11 @@ export function initializeOverlayAnkiIntegration(options: {
|
||||
data: KikuFieldGroupingRequestData,
|
||||
) => Promise<KikuFieldGroupingChoice>;
|
||||
getKnownWordCacheStatePath: () => string;
|
||||
getCachedMediaPath?: (
|
||||
currentVideoPath: string,
|
||||
kind: 'audio' | 'video',
|
||||
) => Promise<string | null>;
|
||||
shouldRequireRemoteMediaCache?: () => boolean;
|
||||
shouldStartAnkiIntegration?: () => boolean;
|
||||
createAnkiIntegration?: (args: CreateAnkiIntegrationArgs) => AnkiIntegrationLike;
|
||||
}): boolean {
|
||||
@@ -200,6 +217,10 @@ export function initializeOverlayAnkiIntegration(options: {
|
||||
showOverlayNotification: options.showOverlayNotification,
|
||||
createFieldGroupingCallback: options.createFieldGroupingCallback,
|
||||
knownWordCacheStatePath: options.getKnownWordCacheStatePath(),
|
||||
...(options.getCachedMediaPath ? { getCachedMediaPath: options.getCachedMediaPath } : {}),
|
||||
...(options.shouldRequireRemoteMediaCache
|
||||
? { shouldRequireRemoteMediaCache: options.shouldRequireRemoteMediaCache }
|
||||
: {}),
|
||||
});
|
||||
if (options.shouldStartAnkiIntegration?.() !== false) {
|
||||
integration.start();
|
||||
|
||||
@@ -0,0 +1,540 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import test from 'node:test';
|
||||
|
||||
import { createYoutubeMediaCacheService } from './media-cache';
|
||||
|
||||
class FakeYtDlpProcess extends EventEmitter {
|
||||
killed = false;
|
||||
stdout = new EventEmitter();
|
||||
stderr = new EventEmitter();
|
||||
|
||||
kill(): boolean {
|
||||
this.killed = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
type SpawnCall = {
|
||||
command: string;
|
||||
args: string[];
|
||||
options?: { stdio?: Array<'ignore' | 'pipe'> };
|
||||
};
|
||||
|
||||
function makeTempCacheRoot(): string {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-youtube-media-cache-test-'));
|
||||
}
|
||||
|
||||
test('YouTube media cache does nothing in direct mode', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args) => {
|
||||
spawnCalls.push({ command, args });
|
||||
return new FakeYtDlpProcess();
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'direct' });
|
||||
|
||||
assert.deepEqual(spawnCalls, []);
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), null);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache exposes the downloaded file after the background job completes', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
const readyEvents: Array<{ url: string; path: string }> = [];
|
||||
const startedEvents: Array<{ url: string }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
onDownloadStarted: (event) => {
|
||||
startedEvents.push(event);
|
||||
},
|
||||
onReady: (event) => {
|
||||
readyEvents.push(event);
|
||||
},
|
||||
spawn: (command, args, options) => {
|
||||
spawnCalls.push({ command, args, options });
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
|
||||
assert.deepEqual(startedEvents, [{ url: 'https://youtu.be/demo' }]);
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
assert.equal(spawnCalls[0]?.command, 'yt-dlp');
|
||||
assert.ok(spawnCalls[0]?.args.includes('--no-playlist'));
|
||||
assert.ok(spawnCalls[0]?.args.includes('--force-ipv4'));
|
||||
assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--retries') + 1], '5');
|
||||
assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--fragment-retries') + 1], '5');
|
||||
assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--extractor-retries') + 1], '5');
|
||||
assert.ok(spawnCalls[0]?.args.includes('--merge-output-format'));
|
||||
assert.equal(
|
||||
spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-f') + 1],
|
||||
'bestvideo*[height<=720]+bestaudio/best[height<=720]',
|
||||
);
|
||||
assert.deepEqual(spawnCalls[0]?.options?.stdio, ['ignore', 'ignore', 'ignore']);
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), null);
|
||||
|
||||
const outputTemplate = spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-o') + 1];
|
||||
assert.equal(typeof outputTemplate, 'string');
|
||||
const outputDir = path.dirname(outputTemplate!);
|
||||
const outputPath = path.join(outputDir, 'media.mkv');
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
fs.writeFileSync(outputPath, 'cached media');
|
||||
spawnedProcesses[0]?.emit('close', 0);
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), outputPath);
|
||||
assert.deepEqual(readyEvents, [{ url: 'https://youtu.be/demo', path: outputPath }]);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache clears the active cache when direct mode starts', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args, options) => {
|
||||
spawnCalls.push({ command, args, options });
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/background', { mode: 'background' });
|
||||
const outputTemplate = spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-o') + 1];
|
||||
assert.equal(typeof outputTemplate, 'string');
|
||||
const outputDir = path.dirname(outputTemplate!);
|
||||
const outputPath = path.join(outputDir, 'media.mkv');
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
fs.writeFileSync(outputPath, 'cached media');
|
||||
spawnedProcesses[0]?.emit('close', 0);
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(await cache.getActiveCachedMediaPath(), outputPath);
|
||||
|
||||
cache.start('https://youtu.be/direct', { mode: 'direct' });
|
||||
|
||||
assert.equal(await cache.getActiveCachedMediaPath(), null);
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/background'), outputPath);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache cancels the active download when direct mode starts', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
const readyEvents: Array<{ url: string; path: string }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
onReady: (event) => {
|
||||
readyEvents.push(event);
|
||||
},
|
||||
spawn: (command, args, options) => {
|
||||
spawnCalls.push({ command, args, options });
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/background', { mode: 'background' });
|
||||
const outputTemplate = spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-o') + 1];
|
||||
assert.equal(typeof outputTemplate, 'string');
|
||||
const outputDir = path.dirname(outputTemplate!);
|
||||
const outputPath = path.join(outputDir, 'media.mkv');
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
fs.writeFileSync(outputPath, 'cached media');
|
||||
|
||||
cache.start('https://youtu.be/direct', { mode: 'direct' });
|
||||
spawnedProcesses[0]?.emit('close', 0);
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
assert.equal(spawnedProcesses[0]?.killed, true);
|
||||
assert.deepEqual(readyEvents, []);
|
||||
assert.equal(await cache.getActiveCachedMediaPath(), null);
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/background'), null);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache reports failed background downloads', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const failedEvents: Array<{ url: string }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
onFailed: (event) => {
|
||||
failedEvents.push(event);
|
||||
},
|
||||
spawn: () => {
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
spawnedProcesses[0]?.emit('close', 1);
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
assert.deepEqual(failedEvents, [{ url: 'https://youtu.be/demo' }]);
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), null);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache only reports a failed download once when error is followed by close', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const failedEvents: Array<{ url: string }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
onFailed: (event) => {
|
||||
failedEvents.push(event);
|
||||
},
|
||||
spawn: () => {
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
spawnedProcesses[0]?.emit('error', new Error('spawn failed'));
|
||||
spawnedProcesses[0]?.emit('close', 1);
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
assert.deepEqual(failedEvents, [{ url: 'https://youtu.be/demo' }]);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache can disable the download height cap', () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args, options) => {
|
||||
spawnCalls.push({ command, args, options });
|
||||
return new FakeYtDlpProcess();
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background', maxHeight: 0 });
|
||||
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
assert.equal(
|
||||
spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-f') + 1],
|
||||
'bestvideo*+bestaudio/best',
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache applies the configured download height cap', () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args, options) => {
|
||||
spawnCalls.push({ command, args, options });
|
||||
return new FakeYtDlpProcess();
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background', maxHeight: 480 });
|
||||
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
assert.equal(
|
||||
spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-f') + 1],
|
||||
'bestvideo*[height<=480]+bestaudio/best[height<=480]',
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache restarts ready sessions when the height cap changes', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args, options) => {
|
||||
spawnCalls.push({ command, args, options });
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background', maxHeight: 720 });
|
||||
const firstOutputTemplate = spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-o') + 1];
|
||||
assert.equal(typeof firstOutputTemplate, 'string');
|
||||
const firstOutputDir = path.dirname(firstOutputTemplate!);
|
||||
const firstOutputPath = path.join(firstOutputDir, 'media.mkv');
|
||||
fs.mkdirSync(firstOutputDir, { recursive: true });
|
||||
fs.writeFileSync(firstOutputPath, 'cached media');
|
||||
spawnedProcesses[0]?.emit('close', 0);
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), firstOutputPath);
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background', maxHeight: 480 });
|
||||
|
||||
assert.equal(spawnCalls.length, 2);
|
||||
assert.equal(fs.existsSync(firstOutputPath), false);
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), null);
|
||||
assert.equal(
|
||||
spawnCalls[1]?.args[spawnCalls[1].args.indexOf('-f') + 1],
|
||||
'bestvideo*[height<=480]+bestaudio/best[height<=480]',
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache restarts running sessions when the height cap changes', () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args, options) => {
|
||||
spawnCalls.push({ command, args, options });
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background', maxHeight: 720 });
|
||||
cache.start('https://youtu.be/demo', { mode: 'background', maxHeight: 480 });
|
||||
|
||||
assert.equal(spawnedProcesses[0]?.killed, true);
|
||||
assert.equal(spawnCalls.length, 2);
|
||||
assert.equal(
|
||||
spawnCalls[1]?.args[spawnCalls[1].args.indexOf('-f') + 1],
|
||||
'bestvideo*[height<=480]+bestaudio/best[height<=480]',
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache removes stale files from previous runs on startup', () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const staleDir = path.join(cacheRoot, 'stale-session');
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
|
||||
try {
|
||||
fs.mkdirSync(staleDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(staleDir, 'media.mkv'), 'stale cached media');
|
||||
|
||||
createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args) => {
|
||||
spawnCalls.push({ command, args });
|
||||
return new FakeYtDlpProcess();
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(fs.existsSync(staleDir), false);
|
||||
assert.deepEqual(spawnCalls, []);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache restarts when a ready cached file was deleted externally', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const spawnCalls: Array<{ command: string; args: string[] }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args) => {
|
||||
spawnCalls.push({ command, args });
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
const outputTemplate = spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-o') + 1];
|
||||
assert.equal(typeof outputTemplate, 'string');
|
||||
const outputDir = path.dirname(outputTemplate!);
|
||||
const outputPath = path.join(outputDir, 'media.mkv');
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
fs.writeFileSync(outputPath, 'cached media');
|
||||
spawnedProcesses[0]?.emit('close', 0);
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), outputPath);
|
||||
|
||||
fs.rmSync(outputPath);
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
|
||||
assert.equal(spawnCalls.length, 2);
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), null);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache removes stale disk siblings before starting a new cache', () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const staleDir = path.join(cacheRoot, 'stale-sibling');
|
||||
const spawnCalls: Array<{ command: string; args: string[] }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args) => {
|
||||
spawnCalls.push({ command, args });
|
||||
return new FakeYtDlpProcess();
|
||||
},
|
||||
});
|
||||
fs.mkdirSync(staleDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(staleDir, 'media.mkv'), 'stale cached media');
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
|
||||
assert.equal(fs.existsSync(staleDir), false);
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache drops old sessions when a new background cache starts', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const spawnCalls: Array<{ command: string; args: string[] }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args) => {
|
||||
spawnCalls.push({ command, args });
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/first', { mode: 'background' });
|
||||
const firstOutputTemplate = spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-o') + 1];
|
||||
assert.equal(typeof firstOutputTemplate, 'string');
|
||||
const firstOutputDir = path.dirname(firstOutputTemplate!);
|
||||
fs.mkdirSync(firstOutputDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(firstOutputDir, 'media.mkv'), 'cached media');
|
||||
|
||||
cache.start('https://youtu.be/second', { mode: 'background' });
|
||||
|
||||
assert.equal(spawnedProcesses[0]?.killed, true);
|
||||
assert.equal(fs.existsSync(firstOutputDir), false);
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/first'), null);
|
||||
assert.equal(spawnCalls.length, 2);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache cleanup kills downloads and removes temp files', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const spawnCalls: Array<{ command: string; args: string[] }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args) => {
|
||||
spawnCalls.push({ command, args });
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
const outputTemplate = spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-o') + 1];
|
||||
assert.equal(typeof outputTemplate, 'string');
|
||||
const outputDir = path.dirname(outputTemplate!);
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(outputDir, 'media.mkv'), 'cached media');
|
||||
|
||||
cache.cleanup();
|
||||
|
||||
assert.equal(spawnedProcesses[0]?.killed, true);
|
||||
assert.equal(fs.existsSync(outputDir), false);
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), null);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,307 @@
|
||||
import { spawn as spawnProcess } from 'node:child_process';
|
||||
import * as crypto from 'node:crypto';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
|
||||
import type { YoutubeMediaCacheMode } from '../../../types/integrations';
|
||||
import { getYoutubeYtDlpCommand } from './ytdlp-command';
|
||||
|
||||
type MediaCacheSessionState = 'running' | 'ready' | 'failed';
|
||||
|
||||
type SpawnedProcess = EventEmitter & {
|
||||
killed?: boolean;
|
||||
kill?: (signal?: NodeJS.Signals | number) => boolean;
|
||||
};
|
||||
|
||||
type SpawnProcess = (
|
||||
command: string,
|
||||
args: string[],
|
||||
options?: { stdio?: Array<'ignore' | 'pipe'> },
|
||||
) => SpawnedProcess;
|
||||
|
||||
interface MediaCacheSession {
|
||||
url: string;
|
||||
dir: string;
|
||||
maxHeight: number;
|
||||
process: SpawnedProcess | null;
|
||||
readyPath: string | null;
|
||||
state: MediaCacheSessionState;
|
||||
}
|
||||
|
||||
export interface YoutubeMediaCacheStartOptions {
|
||||
mode: YoutubeMediaCacheMode;
|
||||
maxHeight?: number;
|
||||
}
|
||||
|
||||
export interface YoutubeMediaCacheServiceDeps {
|
||||
cacheRoot?: string;
|
||||
getYtDlpCommand?: () => string;
|
||||
spawn?: SpawnProcess;
|
||||
onDownloadStarted?: (event: { url: string }) => void;
|
||||
onReady?: (event: { url: string; path: string }) => void;
|
||||
onFailed?: (event: { url: string }) => void;
|
||||
logInfo?: (message: string) => void;
|
||||
logWarn?: (message: string) => void;
|
||||
}
|
||||
|
||||
const MEDIA_FILE_EXTENSIONS = new Set(['.mkv', '.mp4', '.webm', '.m4a', '.mp3', '.opus']);
|
||||
const DEFAULT_MAX_HEIGHT = 720;
|
||||
|
||||
function cacheKeyForUrl(url: string): string {
|
||||
return crypto.createHash('sha256').update(url).digest('hex').slice(0, 24);
|
||||
}
|
||||
|
||||
function isFinalMediaFile(fileName: string): boolean {
|
||||
if (!fileName.startsWith('media.')) {
|
||||
return false;
|
||||
}
|
||||
if (fileName.endsWith('.part') || fileName.endsWith('.ytdl') || fileName.endsWith('.tmp')) {
|
||||
return false;
|
||||
}
|
||||
return MEDIA_FILE_EXTENSIONS.has(path.extname(fileName).toLowerCase());
|
||||
}
|
||||
|
||||
function findReadyMediaPath(dir: string): string | null {
|
||||
try {
|
||||
const files = fs.readdirSync(dir);
|
||||
const mediaFile = files.find(isFinalMediaFile);
|
||||
return mediaFile ? path.join(dir, mediaFile) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function getFormatSelector(maxHeight: number): string {
|
||||
return maxHeight > 0
|
||||
? `bestvideo*[height<=${maxHeight}]+bestaudio/best[height<=${maxHeight}]`
|
||||
: 'bestvideo*+bestaudio/best';
|
||||
}
|
||||
|
||||
function normalizeMaxHeight(maxHeight: number | undefined): number {
|
||||
if (maxHeight === undefined) {
|
||||
return DEFAULT_MAX_HEIGHT;
|
||||
}
|
||||
return Number.isInteger(maxHeight) && maxHeight >= 0 ? maxHeight : DEFAULT_MAX_HEIGHT;
|
||||
}
|
||||
|
||||
function createYtDlpArgs(url: string, outputTemplate: string, maxHeight?: number): string[] {
|
||||
return [
|
||||
'--no-playlist',
|
||||
'--no-warnings',
|
||||
'--force-ipv4',
|
||||
'--retries',
|
||||
'5',
|
||||
'--fragment-retries',
|
||||
'5',
|
||||
'--extractor-retries',
|
||||
'5',
|
||||
'-f',
|
||||
getFormatSelector(normalizeMaxHeight(maxHeight)),
|
||||
'--merge-output-format',
|
||||
'mkv',
|
||||
'-o',
|
||||
outputTemplate,
|
||||
url,
|
||||
];
|
||||
}
|
||||
|
||||
export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDeps = {}) {
|
||||
const cacheRoot = deps.cacheRoot ?? path.join(os.tmpdir(), 'subminer-youtube-media-cache');
|
||||
const getYtDlpCommand = deps.getYtDlpCommand ?? getYoutubeYtDlpCommand;
|
||||
const spawn: SpawnProcess =
|
||||
deps.spawn ??
|
||||
((command, args, options) =>
|
||||
spawnProcess(command, args, options ?? {}) as unknown as SpawnedProcess);
|
||||
const sessions = new Map<string, MediaCacheSession>();
|
||||
let activeKey: string | null = null;
|
||||
|
||||
const getSessionDir = (url: string): string => path.join(cacheRoot, cacheKeyForUrl(url));
|
||||
const removeCacheDir = (dir: string): void => {
|
||||
try {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Temp cache cleanup should not block shutdown or playback startup.
|
||||
}
|
||||
};
|
||||
const removeCacheRootEntriesExcept = (dirsToKeep: string[]): void => {
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = fs.readdirSync(cacheRoot, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const keepDirs = new Set(dirsToKeep.map((dir) => path.resolve(dir)));
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(cacheRoot, entry.name);
|
||||
if (keepDirs.has(path.resolve(entryPath))) {
|
||||
continue;
|
||||
}
|
||||
removeCacheDir(entryPath);
|
||||
}
|
||||
};
|
||||
const removeSession = (key: string): void => {
|
||||
const session = sessions.get(key);
|
||||
if (!session) {
|
||||
return;
|
||||
}
|
||||
if (session.state === 'running' && session.process?.kill && !session.process.killed) {
|
||||
session.process.kill();
|
||||
}
|
||||
sessions.delete(key);
|
||||
if (activeKey === key) {
|
||||
activeKey = null;
|
||||
}
|
||||
removeCacheDir(session.dir);
|
||||
};
|
||||
const removeInactiveSessions = (keyToKeep: string): void => {
|
||||
for (const key of [...sessions.keys()]) {
|
||||
if (key !== keyToKeep) {
|
||||
removeSession(key);
|
||||
}
|
||||
}
|
||||
};
|
||||
removeCacheRootEntriesExcept([]);
|
||||
|
||||
const getCachedMediaPath = async (url: string): Promise<string | null> => {
|
||||
const key = cacheKeyForUrl(url);
|
||||
const session = sessions.get(key);
|
||||
if (session?.readyPath && fs.existsSync(session.readyPath)) {
|
||||
return session.readyPath;
|
||||
}
|
||||
|
||||
const readyPath = findReadyMediaPath(session?.dir ?? getSessionDir(url));
|
||||
if (readyPath) {
|
||||
sessions.set(key, {
|
||||
url,
|
||||
dir: path.dirname(readyPath),
|
||||
maxHeight: session?.maxHeight ?? DEFAULT_MAX_HEIGHT,
|
||||
process: null,
|
||||
readyPath,
|
||||
state: 'ready',
|
||||
});
|
||||
return readyPath;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
const getActiveCachedMediaPath = async (): Promise<string | null> => {
|
||||
if (!activeKey) {
|
||||
return null;
|
||||
}
|
||||
const session = sessions.get(activeKey);
|
||||
return session ? getCachedMediaPath(session.url) : null;
|
||||
};
|
||||
|
||||
const start = (url: string, options: YoutubeMediaCacheStartOptions): void => {
|
||||
if (options.mode !== 'background') {
|
||||
if (activeKey) {
|
||||
const activeSession = sessions.get(activeKey);
|
||||
if (activeSession?.state === 'running') {
|
||||
removeSession(activeKey);
|
||||
}
|
||||
}
|
||||
activeKey = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const key = cacheKeyForUrl(url);
|
||||
const maxHeight = normalizeMaxHeight(options.maxHeight);
|
||||
activeKey = key;
|
||||
const dir = getSessionDir(url);
|
||||
const existingSession = sessions.get(key);
|
||||
const canReuseExistingSession = existingSession?.maxHeight === maxHeight;
|
||||
if (existingSession?.state === 'running' && canReuseExistingSession) {
|
||||
removeInactiveSessions(key);
|
||||
removeCacheRootEntriesExcept([existingSession.dir]);
|
||||
return;
|
||||
}
|
||||
if (existingSession) {
|
||||
if (
|
||||
canReuseExistingSession &&
|
||||
existingSession.state === 'ready' &&
|
||||
((existingSession.readyPath && fs.existsSync(existingSession.readyPath)) ||
|
||||
findReadyMediaPath(existingSession.dir))
|
||||
) {
|
||||
removeInactiveSessions(key);
|
||||
removeCacheRootEntriesExcept([existingSession.dir]);
|
||||
return;
|
||||
}
|
||||
removeSession(key);
|
||||
activeKey = key;
|
||||
}
|
||||
removeInactiveSessions(key);
|
||||
removeCacheRootEntriesExcept([dir]);
|
||||
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const outputTemplate = path.join(dir, 'media.%(ext)s');
|
||||
const args = createYtDlpArgs(url, outputTemplate, maxHeight);
|
||||
const child = spawn(getYtDlpCommand(), args, { stdio: ['ignore', 'ignore', 'ignore'] });
|
||||
const session: MediaCacheSession = {
|
||||
url,
|
||||
dir,
|
||||
maxHeight,
|
||||
process: child,
|
||||
readyPath: null,
|
||||
state: 'running',
|
||||
};
|
||||
sessions.set(key, session);
|
||||
deps.logInfo?.(`Started YouTube media cache download for ${url}`);
|
||||
deps.onDownloadStarted?.({ url });
|
||||
|
||||
child.once('error', (error) => {
|
||||
const currentSession = sessions.get(key);
|
||||
if (currentSession !== session) {
|
||||
return;
|
||||
}
|
||||
session.state = 'failed';
|
||||
session.process = null;
|
||||
deps.logWarn?.(
|
||||
`YouTube media cache download failed: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
deps.onFailed?.({ url });
|
||||
});
|
||||
|
||||
child.once('close', (code) => {
|
||||
const currentSession = sessions.get(key);
|
||||
if (currentSession !== session) {
|
||||
return;
|
||||
}
|
||||
if (session.state === 'failed') {
|
||||
return;
|
||||
}
|
||||
session.process = null;
|
||||
if (code === 0) {
|
||||
const readyPath = findReadyMediaPath(dir);
|
||||
if (readyPath) {
|
||||
session.state = 'ready';
|
||||
session.readyPath = readyPath;
|
||||
deps.logInfo?.(`YouTube media cache ready at ${readyPath}`);
|
||||
deps.onReady?.({ url, path: readyPath });
|
||||
return;
|
||||
}
|
||||
}
|
||||
session.state = 'failed';
|
||||
deps.logWarn?.(`YouTube media cache download exited without a usable media file.`);
|
||||
deps.onFailed?.({ url });
|
||||
});
|
||||
};
|
||||
|
||||
const cleanup = (): void => {
|
||||
for (const key of [...sessions.keys()]) {
|
||||
removeSession(key);
|
||||
}
|
||||
activeKey = null;
|
||||
};
|
||||
|
||||
return {
|
||||
cleanup,
|
||||
getActiveCachedMediaPath,
|
||||
getCachedMediaPath,
|
||||
start,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user