feat(mining): cache remote media windows during card creation

- Reuse one temporary download for timing review, audio, and screenshots
- Expire cached windows after inactivity and clean them up on exit
This commit is contained in:
2026-09-02 01:21:26 -07:00
parent 760b3e1d3d
commit f90ee78204
19 changed files with 1424 additions and 79 deletions
@@ -23,6 +23,21 @@ describe('buildMediaTimingPreviewArgs', () => {
assert.equal(args.at(-1), '/video/show.mkv');
});
test('keeps source timestamps for cached remote windows', () => {
const args = buildMediaTimingPreviewArgs('/tmp/review.sock', {
mediaPath: '/tmp/window.mkv',
absoluteTimestamps: true,
});
assert.ok(args.includes('--rebase-start-time=no'));
assert.equal(
buildMediaTimingPreviewArgs('/tmp/review.sock', { mediaPath: '/video/show.mkv' }).includes(
'--rebase-start-time=no',
),
false,
);
});
test('separates an option-like media path without adding optional audio arguments', () => {
const args = buildMediaTimingPreviewArgs('/tmp/review.sock', {
mediaPath: '--fullscreen',
@@ -14,6 +14,8 @@ export interface MediaTimingPreviewStartOptions {
executablePath?: string;
audioTrackId?: number;
volume?: number;
/** The file keeps source timestamps (a cached remote window); seek with the original times. */
absoluteTimestamps?: boolean;
}
type PreviewProcess = Pick<ChildProcess, 'kill' | 'once'>;
@@ -51,6 +53,9 @@ export function buildMediaTimingPreviewArgs(
if (typeof options.volume === 'number' && Number.isFinite(options.volume)) {
args.push(`--volume=${Math.max(0, options.volume)}`);
}
if (options.absoluteTimestamps) {
args.push('--rebase-start-time=no');
}
args.push('--', options.mediaPath);
return args;
}
@@ -35,6 +35,29 @@ test('speech waveform maps the selected FFmpeg stream and visible range', () =>
assert.match(args[args.indexOf('-af') + 1] ?? '', /c0=FC/);
});
test('speech waveform seeks cached windows by source timestamps', () => {
const args = buildSpeechWaveformArgs(
{
mediaPath: { path: '/tmp/window.mkv', absoluteTimestamps: true, singleResolvedStream: true },
startTime: 8,
endTime: 15,
},
'downmix',
);
assert.deepEqual(args.slice(args.indexOf('-ss'), args.indexOf('-t') + 2), [
'-ss',
'8',
'-seek_timestamp',
'1',
'-i',
'/tmp/window.mkv',
'-t',
'7',
]);
assert.equal(args.includes('-map'), false);
});
test('waveform peaks are normalized without flattening quieter sections', () => {
const peaks = computeWaveformPeaks(pcm([0, 1_000, -2_000, 4_000, -8_000, 16_000]), 3);
+5 -2
View File
@@ -1,4 +1,5 @@
import { spawn } from 'node:child_process';
import { normalizeMediaInput, type MediaInput } from '../../media-input';
const WAVEFORM_SAMPLE_RATE = 8_000;
const WAVEFORM_POINT_COUNT = 480;
@@ -9,7 +10,7 @@ const CENTER_CHANNEL_FILTER = `pan=mono|c0=FC,${SPEECH_FILTER}`;
const DOWNMIX_FILTER = `aformat=channel_layouts=mono,${SPEECH_FILTER}`;
export interface SpeechWaveformOptions {
mediaPath: string;
mediaPath: MediaInput;
startTime: number;
endTime: number;
audioStreamIndex?: number;
@@ -22,6 +23,7 @@ export function buildSpeechWaveformArgs(
mode: 'center' | 'downmix',
): string[] {
const duration = options.endTime - options.startTime;
const input = normalizeMediaInput(options.mediaPath);
const args = [
'-hide_banner',
'-nostdin',
@@ -29,8 +31,9 @@ export function buildSpeechWaveformArgs(
'error',
'-ss',
String(options.startTime),
...input.inputArgs,
'-i',
options.mediaPath,
input.path,
'-t',
String(duration),
];
@@ -0,0 +1,253 @@
import assert from 'node:assert/strict';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import test from 'node:test';
import {
buildRemoteMediaWindowArgs,
RemoteMediaWindowCache,
REMOTE_MEDIA_WINDOW_MAX_SECONDS,
type RemoteMediaWindowCacheOptions,
} from './remote-media-window-cache';
const SOURCE = {
path: 'https://jellyfin.example/Videos/abc/stream?static=true',
audioStreamIndex: 2,
};
type ExecFileStub = NonNullable<RemoteMediaWindowCacheOptions['execFile']>;
function createStub(options: { fail?: boolean; empty?: boolean; defer?: boolean } = {}) {
const calls: string[][] = [];
const pendingCallbacks: Array<() => void> = [];
const execFile: ExecFileStub = (_file, args, _options, callback) => {
calls.push([...args]);
const finish = (): void => {
const outputPath = args.at(-1);
assert.ok(outputPath);
if (options.fail) {
callback(Object.assign(new Error('boom'), { code: 1 }));
return;
}
if (!options.empty) {
fs.writeFileSync(outputPath, 'mkv', 'utf8');
}
callback(null);
};
if (options.defer) {
pendingCallbacks.push(finish);
} else {
queueMicrotask(finish);
}
};
return {
calls,
execFile,
flush: () => {
for (const finish of pendingCallbacks.splice(0)) finish();
},
};
}
async function withCache(
stubOptions: Parameters<typeof createStub>[0],
cacheOptions: Omit<RemoteMediaWindowCacheOptions, 'execFile' | 'tempDir'>,
run: (cache: RemoteMediaWindowCache, stub: ReturnType<typeof createStub>) => Promise<void>,
): Promise<void> {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-media-window-test-'));
const stub = createStub(stubOptions);
const cache = new RemoteMediaWindowCache({
tempDir,
execFile: stub.execFile,
idleTtlMs: 0,
logDebug: () => undefined,
...cacheOptions,
});
try {
await run(cache, stub);
} finally {
cache.cleanup();
fs.rmSync(tempDir, { recursive: true, force: true });
}
}
function argValue(args: string[], flag: string): string | undefined {
const index = args.indexOf(flag);
return index === -1 ? undefined : args[index + 1];
}
test('buildRemoteMediaWindowArgs stream-copies the window with source timestamps intact', () => {
const args = buildRemoteMediaWindowArgs(
{ ...SOURCE, inputOptions: { reconnect: true, headers: { Referer: 'https://a.example/' } } },
{ startTime: 22.75, endTime: 33 },
'/tmp/window.mkv',
);
const inputIndex = args.indexOf('-i');
assert.equal(args[inputIndex + 1], SOURCE.path);
assert.ok(args.indexOf('-reconnect') < inputIndex);
assert.ok(args.indexOf('-headers') < inputIndex);
assert.equal(argValue(args, '-ss'), '22.75');
assert.equal(argValue(args, '-t'), '10.25');
assert.ok(args.indexOf('-t') < inputIndex);
assert.deepEqual(args.slice(args.indexOf('-map'), args.indexOf('-map') + 4), [
'-map',
'0:v:0?',
'-map',
'0:2',
]);
assert.equal(argValue(args, '-c'), 'copy');
assert.ok(args.includes('-copyts'));
assert.ok(args.includes('-start_at_zero'));
assert.equal(argValue(args, '-f'), 'matroska');
assert.equal(args.at(-1), '/tmp/window.mkv');
});
test('buildRemoteMediaWindowArgs keeps every audio stream when none is selected', () => {
const args = buildRemoteMediaWindowArgs(
{ path: SOURCE.path, audioStreamIndex: null },
{ startTime: 0, endTime: 5 },
'/tmp/window.mkv',
);
assert.equal(args[args.lastIndexOf('-map') + 1], '0:a');
});
test('acquire downloads once and reuses the window for covered ranges', async () => {
await withCache({}, {}, async (cache, stub) => {
const window = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
assert.equal(stub.calls.length, 1);
assert.equal(argValue(stub.calls[0]!, '-ss'), '9.75');
assert.equal(argValue(stub.calls[0]!, '-t'), '5.25');
assert.equal(window.startTime, 9.75);
assert.equal(window.endTime, 15);
assert.equal(window.audioStreamIndex, 2);
assert.ok(fs.existsSync(window.path));
assert.deepEqual(window.media, {
path: window.path,
source: 'remote-window',
singleResolvedStream: true,
absoluteTimestamps: true,
});
assert.equal(await cache.acquire(SOURCE, { startTime: 11, endTime: 15 }), window);
assert.equal(await cache.lookup(SOURCE, { startTime: 12, endTime: 12 }), window);
assert.equal(
await cache.lookup(
{ path: SOURCE.path, audioStreamIndex: null },
{ startTime: 12, endTime: 13 },
),
window,
);
assert.equal(stub.calls.length, 1);
});
});
test('lookup never downloads and misses on other ranges, sources, or audio streams', async () => {
await withCache({}, {}, async (cache, stub) => {
assert.equal(await cache.lookup(SOURCE, { startTime: 10, endTime: 14 }), null);
assert.equal(stub.calls.length, 0);
await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
assert.equal(await cache.lookup(SOURCE, { startTime: 14, endTime: 16 }), null);
assert.equal(
await cache.lookup(
{ path: 'https://other.example/stream', audioStreamIndex: 2 },
{
startTime: 11,
endTime: 12,
},
),
null,
);
assert.equal(
await cache.lookup(
{ path: SOURCE.path, audioStreamIndex: 3 },
{ startTime: 11, endTime: 12 },
),
null,
);
assert.equal(stub.calls.length, 1);
});
});
test('acquire widens to the union of the old window and replaces the old file', async () => {
await withCache({}, {}, async (cache, stub) => {
const first = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
const second = await cache.acquire(SOURCE, { startTime: 8, endTime: 12 });
assert.equal(stub.calls.length, 2);
assert.equal(argValue(stub.calls[1]!, '-ss'), '7.75');
assert.equal(second.startTime, 7.75);
assert.equal(second.endTime, 15);
assert.notEqual(second.path, first.path);
assert.equal(fs.existsSync(first.path), false);
assert.ok(fs.existsSync(second.path));
assert.equal(cache.currentWindow, second);
});
});
test('acquire shares an in-flight download between concurrent callers', async () => {
await withCache({ defer: true }, {}, async (cache, stub) => {
const first = cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
await Promise.resolve();
const second = cache.acquire(SOURCE, { startTime: 11, endTime: 13 });
const lookup = cache.lookup(SOURCE, { startTime: 12, endTime: 12 });
await Promise.resolve();
assert.equal(stub.calls.length, 1);
stub.flush();
const [a, b, c] = await Promise.all([first, second, lookup]);
assert.equal(a, b);
assert.equal(a, c);
assert.equal(stub.calls.length, 1);
});
});
test('acquire rejects on ffmpeg failure, leaves no file, and can retry', async () => {
await withCache({ fail: true }, {}, async (cache, stub) => {
await assert.rejects(
cache.acquire(SOURCE, { startTime: 10, endTime: 14 }),
/FFmpeg media window failed: boom/,
);
assert.equal(cache.currentWindow, null);
assert.equal(await cache.lookup(SOURCE, { startTime: 10, endTime: 14 }), null);
await assert.rejects(cache.acquire(SOURCE, { startTime: 10, endTime: 14 }));
assert.equal(stub.calls.length, 2);
});
await withCache({ empty: true }, {}, async (cache) => {
await assert.rejects(
cache.acquire(SOURCE, { startTime: 10, endTime: 14 }),
/exited without creating a media window/,
);
});
});
test('acquire refuses invalid and oversized ranges without spawning ffmpeg', async () => {
await withCache({}, {}, async (cache, stub) => {
await assert.rejects(cache.acquire(SOURCE, { startTime: 10, endTime: 10 }), /invalid/);
await assert.rejects(cache.acquire(SOURCE, { startTime: -1, endTime: 10 }), /invalid/);
await assert.rejects(
cache.acquire(SOURCE, { startTime: 0, endTime: REMOTE_MEDIA_WINDOW_MAX_SECONDS + 1 }),
/too long/,
);
assert.equal(stub.calls.length, 0);
});
});
test('the window is deleted after the idle timeout and on cleanup', async () => {
await withCache({}, { idleTtlMs: 20 }, async (cache) => {
const window = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
await new Promise((resolve) => setTimeout(resolve, 60));
assert.equal(cache.currentWindow, null);
assert.equal(fs.existsSync(window.path), false);
const again = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
cache.cleanup();
assert.equal(fs.existsSync(again.path), false);
assert.equal(fs.existsSync(path.dirname(again.path)), false);
});
});
@@ -0,0 +1,377 @@
import { execFile as nodeExecFile, type ExecFileException } from 'child_process';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { createLogger } from '../../logger';
import { normalizeMediaInput, type MediaInput, type MediaInputOptions } from '../../media-input';
const log = createLogger('media-window');
export const REMOTE_MEDIA_WINDOW_TIMEOUT_MS = 120_000;
export const REMOTE_MEDIA_WINDOW_MAX_SECONDS = 180;
const HEAD_SLACK_SECONDS = 0.25;
const TAIL_SLACK_SECONDS = 1;
const DEFAULT_IDLE_TTL_MS = 10 * 60_000;
const COVERAGE_EPSILON_SECONDS = 0.01;
export interface RemoteMediaWindowSource {
path: string;
inputOptions?: MediaInputOptions;
/** FFmpeg stream index to keep; `null`/undefined keeps every audio stream. */
audioStreamIndex?: number | null;
}
export interface RemoteMediaWindowRange {
startTime: number;
endTime: number;
}
export interface RemoteMediaWindow {
path: string;
startTime: number;
endTime: number;
sourcePath: string;
audioStreamIndex: number | null;
/** Input descriptor for FFmpeg reads; timestamps stay absolute so callers keep source times. */
media: MediaInput;
}
type WindowExecFile = (
file: string,
args: readonly string[],
options: { timeout: number },
callback: (error: ExecFileException | null) => void,
) => void;
export interface RemoteMediaWindowCacheOptions {
tempDir?: string;
execFile?: WindowExecFile;
idleTtlMs?: number;
logDebug?: (message: string) => void;
}
interface PendingFetch extends RemoteMediaWindowRange {
sourcePath: string;
audioStreamIndex: number | null;
promise: Promise<RemoteMediaWindow>;
}
export function isRemoteMediaWindowSourcePath(value: string): boolean {
return /^https?:\/\//i.test(value.trim());
}
function describeSourceForDebugLog(sourcePath: string): string {
try {
return `remote:${new URL(sourcePath).hostname.toLowerCase() || 'unknown'}`;
} catch {
return 'remote:unknown';
}
}
function isUsableRange(range: RemoteMediaWindowRange, allowEmpty: boolean): boolean {
return (
Number.isFinite(range.startTime) &&
Number.isFinite(range.endTime) &&
range.startTime >= 0 &&
(allowEmpty ? range.endTime >= range.startTime : range.endTime > range.startTime)
);
}
function audioStreamMatches(
windowIndex: number | null,
requested: number | null | undefined,
): boolean {
return requested == null || windowIndex === requested;
}
function covers(
candidate: RemoteMediaWindowRange & { sourcePath: string; audioStreamIndex: number | null },
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): boolean {
return (
candidate.sourcePath === source.path &&
audioStreamMatches(candidate.audioStreamIndex, source.audioStreamIndex) &&
candidate.startTime <= range.startTime + COVERAGE_EPSILON_SECONDS &&
candidate.endTime >= range.endTime - COVERAGE_EPSILON_SECONDS
);
}
/**
* Stream-copies `[startTime, endTime]` of a remote source into a local Matroska file.
* `-copyts -start_at_zero` keeps the source timestamps, so later reads seek with the
* original times via `-seek_timestamp 1` (see `MediaInput.absoluteTimestamps`).
*/
export function buildRemoteMediaWindowArgs(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
outputPath: string,
): string[] {
const input = normalizeMediaInput({ path: source.path, inputOptions: source.inputOptions });
const audioMap =
typeof source.audioStreamIndex === 'number' && Number.isInteger(source.audioStreamIndex)
? `0:${source.audioStreamIndex}`
: '0:a';
return [
'-hide_banner',
'-nostdin',
'-loglevel',
'error',
'-ss',
String(range.startTime),
'-t',
String(range.endTime - range.startTime),
...input.inputArgs,
'-i',
input.path,
'-map',
'0:v:0?',
'-map',
audioMap,
'-c',
'copy',
'-sn',
'-dn',
'-copyts',
'-start_at_zero',
'-f',
'matroska',
'-y',
outputPath,
];
}
/**
* Holds one downloaded window of the current remote stream so the timing review,
* audio extraction, and screenshot all read the same local bytes instead of each
* re-fetching the clip over HTTP. A new window replaces the old one; the file is
* deleted after `idleTtlMs` without use, on `clear()`, or on `cleanup()`.
*/
export class RemoteMediaWindowCache {
private readonly tempDir: string;
private readonly execFile: WindowExecFile;
private readonly idleTtlMs: number;
private readonly logDebug: (message: string) => void;
private current: RemoteMediaWindow | null = null;
private pending: PendingFetch | null = null;
private idleTimer: ReturnType<typeof setTimeout> | null = null;
private sequence = 0;
constructor(options: RemoteMediaWindowCacheOptions = {}) {
this.tempDir = options.tempDir ?? path.join(os.tmpdir(), 'subminer-media-windows');
this.execFile = options.execFile ?? nodeExecFile;
this.idleTtlMs = options.idleTtlMs ?? DEFAULT_IDLE_TTL_MS;
this.logDebug = options.logDebug ?? ((message) => log.debug(message));
}
get currentWindow(): RemoteMediaWindow | null {
return this.current;
}
/** Returns a ready or in-flight window covering the range; never starts a download. */
async lookup(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow | null> {
if (!isUsableRange(range, true)) return null;
if (this.current && covers(this.current, source, range)) {
this.touch();
return this.current;
}
const pending = this.pending;
if (pending && covers(pending, source, range)) {
try {
const window = await pending.promise;
this.touch();
return window;
} catch {
return null;
}
}
return null;
}
/** Returns a window covering the range, downloading (and widening) one when needed. */
async acquire(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow> {
if (!isUsableRange(range, false)) {
throw new Error('Media window range is invalid.');
}
if (range.endTime - range.startTime > REMOTE_MEDIA_WINDOW_MAX_SECONDS) {
throw new Error('Media window range is too long to download.');
}
for (;;) {
const hit = await this.lookup(source, range);
if (hit) return hit;
const pending = this.pending;
if (!pending) break;
// Another caller is already downloading; wait for it, then re-check coverage.
await pending.promise.catch(() => null);
}
return this.fetch(source, this.planFetchRange(source, range));
}
clear(): void {
this.cancelIdleTimer();
const current = this.current;
this.current = null;
if (current) this.removeFile(current.path);
}
cleanup(): void {
this.clear();
try {
fs.rmSync(this.tempDir, { recursive: true, force: true });
} catch (error) {
log.error('Failed to cleanup media window directory:', error);
}
}
private planFetchRange(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): RemoteMediaWindowRange {
let startTime = Math.max(0, range.startTime - HEAD_SLACK_SECONDS);
let endTime = range.endTime + TAIL_SLACK_SECONDS;
const current = this.current;
if (
current &&
current.sourcePath === source.path &&
audioStreamMatches(current.audioStreamIndex, source.audioStreamIndex)
) {
// Keep what was already downloaded when the review timeline grows in one direction.
const unionStart = Math.min(startTime, current.startTime);
const unionEnd = Math.max(endTime, current.endTime);
if (unionEnd - unionStart <= REMOTE_MEDIA_WINDOW_MAX_SECONDS) {
startTime = unionStart;
endTime = unionEnd;
}
}
return { startTime, endTime };
}
private fetch(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow> {
fs.mkdirSync(this.tempDir, { recursive: true });
this.sequence += 1;
const outputPath = path.join(this.tempDir, `window_${Date.now()}_${this.sequence}.mkv`);
const audioStreamIndex =
typeof source.audioStreamIndex === 'number' ? source.audioStreamIndex : null;
const description = describeSourceForDebugLog(source.path);
const startedAt = Date.now();
this.logDebug(
`[media-window] fetch start ${description} start=${range.startTime} end=${range.endTime} audioStream=${audioStreamIndex ?? 'all'}`,
);
const promise = new Promise<RemoteMediaWindow>((resolve, reject) => {
this.execFile(
'ffmpeg',
buildRemoteMediaWindowArgs(source, range, outputPath),
{ timeout: REMOTE_MEDIA_WINDOW_TIMEOUT_MS },
(error) => {
const elapsedMs = Math.max(0, Date.now() - startedAt);
const size = error ? 0 : this.fileSize(outputPath);
if (error || size === 0) {
this.removeFile(outputPath);
const reason = error
? error.code === 'ENOENT'
? 'FFmpeg not found. Install FFmpeg to enable media generation.'
: `FFmpeg media window failed: ${error.message}`
: 'FFmpeg exited without creating a media window.';
this.logDebug(`[media-window] fetch failed ${description} elapsedMs=${elapsedMs}`);
reject(new Error(reason));
return;
}
const window: RemoteMediaWindow = {
path: outputPath,
startTime: range.startTime,
endTime: range.endTime,
sourcePath: source.path,
audioStreamIndex,
media: {
path: outputPath,
source: 'remote-window',
singleResolvedStream: true,
absoluteTimestamps: true,
},
};
this.logDebug(
`[media-window] fetch complete ${description} elapsedMs=${elapsedMs} bytes=${size}`,
);
this.replaceCurrent(window);
resolve(window);
},
);
});
const pending: PendingFetch = {
sourcePath: source.path,
audioStreamIndex,
startTime: range.startTime,
endTime: range.endTime,
promise,
};
this.pending = pending;
promise
.catch(() => undefined)
.then(() => {
if (this.pending === pending) this.pending = null;
});
return promise;
}
private replaceCurrent(window: RemoteMediaWindow): void {
const previous = this.current;
this.current = window;
if (previous && previous.path !== window.path) this.removeFile(previous.path);
this.touch();
}
private touch(): void {
this.cancelIdleTimer();
if (this.idleTtlMs <= 0 || !this.current) return;
const timer = setTimeout(() => {
if (this.idleTimer === timer) this.idleTimer = null;
this.clear();
}, this.idleTtlMs);
timer.unref?.();
this.idleTimer = timer;
}
private cancelIdleTimer(): void {
if (this.idleTimer) clearTimeout(this.idleTimer);
this.idleTimer = null;
}
private fileSize(filePath: string): number {
try {
return fs.statSync(filePath).size;
} catch {
return 0;
}
}
private removeFile(filePath: string): void {
try {
fs.unlinkSync(filePath);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
log.debug(`Failed to remove media window ${filePath}:`, (error as Error).message);
}
}
}
}
let sharedCache: RemoteMediaWindowCache | null = null;
/** Process-wide cache so the review modal and card media generation share one download. */
export function getSharedRemoteMediaWindowCache(): RemoteMediaWindowCache {
sharedCache ??= new RemoteMediaWindowCache();
return sharedCache;
}