feat(mining): cache remote media windows during card creation

- Reuse one temporary download for timing review, audio, and screenshots
- Expire cached windows after inactivity and clean them up on exit
This commit is contained in:
2026-09-02 01:21:26 -07:00
parent 760b3e1d3d
commit f90ee78204
19 changed files with 1424 additions and 79 deletions
@@ -23,6 +23,21 @@ describe('buildMediaTimingPreviewArgs', () => {
assert.equal(args.at(-1), '/video/show.mkv');
});
test('keeps source timestamps for cached remote windows', () => {
const args = buildMediaTimingPreviewArgs('/tmp/review.sock', {
mediaPath: '/tmp/window.mkv',
absoluteTimestamps: true,
});
assert.ok(args.includes('--rebase-start-time=no'));
assert.equal(
buildMediaTimingPreviewArgs('/tmp/review.sock', { mediaPath: '/video/show.mkv' }).includes(
'--rebase-start-time=no',
),
false,
);
});
test('separates an option-like media path without adding optional audio arguments', () => {
const args = buildMediaTimingPreviewArgs('/tmp/review.sock', {
mediaPath: '--fullscreen',
@@ -14,6 +14,8 @@ export interface MediaTimingPreviewStartOptions {
executablePath?: string;
audioTrackId?: number;
volume?: number;
/** The file keeps source timestamps (a cached remote window); seek with the original times. */
absoluteTimestamps?: boolean;
}
type PreviewProcess = Pick<ChildProcess, 'kill' | 'once'>;
@@ -51,6 +53,9 @@ export function buildMediaTimingPreviewArgs(
if (typeof options.volume === 'number' && Number.isFinite(options.volume)) {
args.push(`--volume=${Math.max(0, options.volume)}`);
}
if (options.absoluteTimestamps) {
args.push('--rebase-start-time=no');
}
args.push('--', options.mediaPath);
return args;
}
@@ -35,6 +35,29 @@ test('speech waveform maps the selected FFmpeg stream and visible range', () =>
assert.match(args[args.indexOf('-af') + 1] ?? '', /c0=FC/);
});
test('speech waveform seeks cached windows by source timestamps', () => {
const args = buildSpeechWaveformArgs(
{
mediaPath: { path: '/tmp/window.mkv', absoluteTimestamps: true, singleResolvedStream: true },
startTime: 8,
endTime: 15,
},
'downmix',
);
assert.deepEqual(args.slice(args.indexOf('-ss'), args.indexOf('-t') + 2), [
'-ss',
'8',
'-seek_timestamp',
'1',
'-i',
'/tmp/window.mkv',
'-t',
'7',
]);
assert.equal(args.includes('-map'), false);
});
test('waveform peaks are normalized without flattening quieter sections', () => {
const peaks = computeWaveformPeaks(pcm([0, 1_000, -2_000, 4_000, -8_000, 16_000]), 3);
+5 -2
View File
@@ -1,4 +1,5 @@
import { spawn } from 'node:child_process';
import { normalizeMediaInput, type MediaInput } from '../../media-input';
const WAVEFORM_SAMPLE_RATE = 8_000;
const WAVEFORM_POINT_COUNT = 480;
@@ -9,7 +10,7 @@ const CENTER_CHANNEL_FILTER = `pan=mono|c0=FC,${SPEECH_FILTER}`;
const DOWNMIX_FILTER = `aformat=channel_layouts=mono,${SPEECH_FILTER}`;
export interface SpeechWaveformOptions {
mediaPath: string;
mediaPath: MediaInput;
startTime: number;
endTime: number;
audioStreamIndex?: number;
@@ -22,6 +23,7 @@ export function buildSpeechWaveformArgs(
mode: 'center' | 'downmix',
): string[] {
const duration = options.endTime - options.startTime;
const input = normalizeMediaInput(options.mediaPath);
const args = [
'-hide_banner',
'-nostdin',
@@ -29,8 +31,9 @@ export function buildSpeechWaveformArgs(
'error',
'-ss',
String(options.startTime),
...input.inputArgs,
'-i',
options.mediaPath,
input.path,
'-t',
String(duration),
];
@@ -0,0 +1,253 @@
import assert from 'node:assert/strict';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import test from 'node:test';
import {
buildRemoteMediaWindowArgs,
RemoteMediaWindowCache,
REMOTE_MEDIA_WINDOW_MAX_SECONDS,
type RemoteMediaWindowCacheOptions,
} from './remote-media-window-cache';
const SOURCE = {
path: 'https://jellyfin.example/Videos/abc/stream?static=true',
audioStreamIndex: 2,
};
type ExecFileStub = NonNullable<RemoteMediaWindowCacheOptions['execFile']>;
function createStub(options: { fail?: boolean; empty?: boolean; defer?: boolean } = {}) {
const calls: string[][] = [];
const pendingCallbacks: Array<() => void> = [];
const execFile: ExecFileStub = (_file, args, _options, callback) => {
calls.push([...args]);
const finish = (): void => {
const outputPath = args.at(-1);
assert.ok(outputPath);
if (options.fail) {
callback(Object.assign(new Error('boom'), { code: 1 }));
return;
}
if (!options.empty) {
fs.writeFileSync(outputPath, 'mkv', 'utf8');
}
callback(null);
};
if (options.defer) {
pendingCallbacks.push(finish);
} else {
queueMicrotask(finish);
}
};
return {
calls,
execFile,
flush: () => {
for (const finish of pendingCallbacks.splice(0)) finish();
},
};
}
async function withCache(
stubOptions: Parameters<typeof createStub>[0],
cacheOptions: Omit<RemoteMediaWindowCacheOptions, 'execFile' | 'tempDir'>,
run: (cache: RemoteMediaWindowCache, stub: ReturnType<typeof createStub>) => Promise<void>,
): Promise<void> {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-media-window-test-'));
const stub = createStub(stubOptions);
const cache = new RemoteMediaWindowCache({
tempDir,
execFile: stub.execFile,
idleTtlMs: 0,
logDebug: () => undefined,
...cacheOptions,
});
try {
await run(cache, stub);
} finally {
cache.cleanup();
fs.rmSync(tempDir, { recursive: true, force: true });
}
}
function argValue(args: string[], flag: string): string | undefined {
const index = args.indexOf(flag);
return index === -1 ? undefined : args[index + 1];
}
test('buildRemoteMediaWindowArgs stream-copies the window with source timestamps intact', () => {
const args = buildRemoteMediaWindowArgs(
{ ...SOURCE, inputOptions: { reconnect: true, headers: { Referer: 'https://a.example/' } } },
{ startTime: 22.75, endTime: 33 },
'/tmp/window.mkv',
);
const inputIndex = args.indexOf('-i');
assert.equal(args[inputIndex + 1], SOURCE.path);
assert.ok(args.indexOf('-reconnect') < inputIndex);
assert.ok(args.indexOf('-headers') < inputIndex);
assert.equal(argValue(args, '-ss'), '22.75');
assert.equal(argValue(args, '-t'), '10.25');
assert.ok(args.indexOf('-t') < inputIndex);
assert.deepEqual(args.slice(args.indexOf('-map'), args.indexOf('-map') + 4), [
'-map',
'0:v:0?',
'-map',
'0:2',
]);
assert.equal(argValue(args, '-c'), 'copy');
assert.ok(args.includes('-copyts'));
assert.ok(args.includes('-start_at_zero'));
assert.equal(argValue(args, '-f'), 'matroska');
assert.equal(args.at(-1), '/tmp/window.mkv');
});
test('buildRemoteMediaWindowArgs keeps every audio stream when none is selected', () => {
const args = buildRemoteMediaWindowArgs(
{ path: SOURCE.path, audioStreamIndex: null },
{ startTime: 0, endTime: 5 },
'/tmp/window.mkv',
);
assert.equal(args[args.lastIndexOf('-map') + 1], '0:a');
});
test('acquire downloads once and reuses the window for covered ranges', async () => {
await withCache({}, {}, async (cache, stub) => {
const window = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
assert.equal(stub.calls.length, 1);
assert.equal(argValue(stub.calls[0]!, '-ss'), '9.75');
assert.equal(argValue(stub.calls[0]!, '-t'), '5.25');
assert.equal(window.startTime, 9.75);
assert.equal(window.endTime, 15);
assert.equal(window.audioStreamIndex, 2);
assert.ok(fs.existsSync(window.path));
assert.deepEqual(window.media, {
path: window.path,
source: 'remote-window',
singleResolvedStream: true,
absoluteTimestamps: true,
});
assert.equal(await cache.acquire(SOURCE, { startTime: 11, endTime: 15 }), window);
assert.equal(await cache.lookup(SOURCE, { startTime: 12, endTime: 12 }), window);
assert.equal(
await cache.lookup(
{ path: SOURCE.path, audioStreamIndex: null },
{ startTime: 12, endTime: 13 },
),
window,
);
assert.equal(stub.calls.length, 1);
});
});
test('lookup never downloads and misses on other ranges, sources, or audio streams', async () => {
await withCache({}, {}, async (cache, stub) => {
assert.equal(await cache.lookup(SOURCE, { startTime: 10, endTime: 14 }), null);
assert.equal(stub.calls.length, 0);
await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
assert.equal(await cache.lookup(SOURCE, { startTime: 14, endTime: 16 }), null);
assert.equal(
await cache.lookup(
{ path: 'https://other.example/stream', audioStreamIndex: 2 },
{
startTime: 11,
endTime: 12,
},
),
null,
);
assert.equal(
await cache.lookup(
{ path: SOURCE.path, audioStreamIndex: 3 },
{ startTime: 11, endTime: 12 },
),
null,
);
assert.equal(stub.calls.length, 1);
});
});
test('acquire widens to the union of the old window and replaces the old file', async () => {
await withCache({}, {}, async (cache, stub) => {
const first = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
const second = await cache.acquire(SOURCE, { startTime: 8, endTime: 12 });
assert.equal(stub.calls.length, 2);
assert.equal(argValue(stub.calls[1]!, '-ss'), '7.75');
assert.equal(second.startTime, 7.75);
assert.equal(second.endTime, 15);
assert.notEqual(second.path, first.path);
assert.equal(fs.existsSync(first.path), false);
assert.ok(fs.existsSync(second.path));
assert.equal(cache.currentWindow, second);
});
});
test('acquire shares an in-flight download between concurrent callers', async () => {
await withCache({ defer: true }, {}, async (cache, stub) => {
const first = cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
await Promise.resolve();
const second = cache.acquire(SOURCE, { startTime: 11, endTime: 13 });
const lookup = cache.lookup(SOURCE, { startTime: 12, endTime: 12 });
await Promise.resolve();
assert.equal(stub.calls.length, 1);
stub.flush();
const [a, b, c] = await Promise.all([first, second, lookup]);
assert.equal(a, b);
assert.equal(a, c);
assert.equal(stub.calls.length, 1);
});
});
test('acquire rejects on ffmpeg failure, leaves no file, and can retry', async () => {
await withCache({ fail: true }, {}, async (cache, stub) => {
await assert.rejects(
cache.acquire(SOURCE, { startTime: 10, endTime: 14 }),
/FFmpeg media window failed: boom/,
);
assert.equal(cache.currentWindow, null);
assert.equal(await cache.lookup(SOURCE, { startTime: 10, endTime: 14 }), null);
await assert.rejects(cache.acquire(SOURCE, { startTime: 10, endTime: 14 }));
assert.equal(stub.calls.length, 2);
});
await withCache({ empty: true }, {}, async (cache) => {
await assert.rejects(
cache.acquire(SOURCE, { startTime: 10, endTime: 14 }),
/exited without creating a media window/,
);
});
});
test('acquire refuses invalid and oversized ranges without spawning ffmpeg', async () => {
await withCache({}, {}, async (cache, stub) => {
await assert.rejects(cache.acquire(SOURCE, { startTime: 10, endTime: 10 }), /invalid/);
await assert.rejects(cache.acquire(SOURCE, { startTime: -1, endTime: 10 }), /invalid/);
await assert.rejects(
cache.acquire(SOURCE, { startTime: 0, endTime: REMOTE_MEDIA_WINDOW_MAX_SECONDS + 1 }),
/too long/,
);
assert.equal(stub.calls.length, 0);
});
});
test('the window is deleted after the idle timeout and on cleanup', async () => {
await withCache({}, { idleTtlMs: 20 }, async (cache) => {
const window = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
await new Promise((resolve) => setTimeout(resolve, 60));
assert.equal(cache.currentWindow, null);
assert.equal(fs.existsSync(window.path), false);
const again = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
cache.cleanup();
assert.equal(fs.existsSync(again.path), false);
assert.equal(fs.existsSync(path.dirname(again.path)), false);
});
});
@@ -0,0 +1,377 @@
import { execFile as nodeExecFile, type ExecFileException } from 'child_process';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { createLogger } from '../../logger';
import { normalizeMediaInput, type MediaInput, type MediaInputOptions } from '../../media-input';
const log = createLogger('media-window');
export const REMOTE_MEDIA_WINDOW_TIMEOUT_MS = 120_000;
export const REMOTE_MEDIA_WINDOW_MAX_SECONDS = 180;
const HEAD_SLACK_SECONDS = 0.25;
const TAIL_SLACK_SECONDS = 1;
const DEFAULT_IDLE_TTL_MS = 10 * 60_000;
const COVERAGE_EPSILON_SECONDS = 0.01;
export interface RemoteMediaWindowSource {
path: string;
inputOptions?: MediaInputOptions;
/** FFmpeg stream index to keep; `null`/undefined keeps every audio stream. */
audioStreamIndex?: number | null;
}
export interface RemoteMediaWindowRange {
startTime: number;
endTime: number;
}
export interface RemoteMediaWindow {
path: string;
startTime: number;
endTime: number;
sourcePath: string;
audioStreamIndex: number | null;
/** Input descriptor for FFmpeg reads; timestamps stay absolute so callers keep source times. */
media: MediaInput;
}
type WindowExecFile = (
file: string,
args: readonly string[],
options: { timeout: number },
callback: (error: ExecFileException | null) => void,
) => void;
export interface RemoteMediaWindowCacheOptions {
tempDir?: string;
execFile?: WindowExecFile;
idleTtlMs?: number;
logDebug?: (message: string) => void;
}
interface PendingFetch extends RemoteMediaWindowRange {
sourcePath: string;
audioStreamIndex: number | null;
promise: Promise<RemoteMediaWindow>;
}
export function isRemoteMediaWindowSourcePath(value: string): boolean {
return /^https?:\/\//i.test(value.trim());
}
function describeSourceForDebugLog(sourcePath: string): string {
try {
return `remote:${new URL(sourcePath).hostname.toLowerCase() || 'unknown'}`;
} catch {
return 'remote:unknown';
}
}
function isUsableRange(range: RemoteMediaWindowRange, allowEmpty: boolean): boolean {
return (
Number.isFinite(range.startTime) &&
Number.isFinite(range.endTime) &&
range.startTime >= 0 &&
(allowEmpty ? range.endTime >= range.startTime : range.endTime > range.startTime)
);
}
function audioStreamMatches(
windowIndex: number | null,
requested: number | null | undefined,
): boolean {
return requested == null || windowIndex === requested;
}
function covers(
candidate: RemoteMediaWindowRange & { sourcePath: string; audioStreamIndex: number | null },
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): boolean {
return (
candidate.sourcePath === source.path &&
audioStreamMatches(candidate.audioStreamIndex, source.audioStreamIndex) &&
candidate.startTime <= range.startTime + COVERAGE_EPSILON_SECONDS &&
candidate.endTime >= range.endTime - COVERAGE_EPSILON_SECONDS
);
}
/**
* Stream-copies `[startTime, endTime]` of a remote source into a local Matroska file.
* `-copyts -start_at_zero` keeps the source timestamps, so later reads seek with the
* original times via `-seek_timestamp 1` (see `MediaInput.absoluteTimestamps`).
*/
export function buildRemoteMediaWindowArgs(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
outputPath: string,
): string[] {
const input = normalizeMediaInput({ path: source.path, inputOptions: source.inputOptions });
const audioMap =
typeof source.audioStreamIndex === 'number' && Number.isInteger(source.audioStreamIndex)
? `0:${source.audioStreamIndex}`
: '0:a';
return [
'-hide_banner',
'-nostdin',
'-loglevel',
'error',
'-ss',
String(range.startTime),
'-t',
String(range.endTime - range.startTime),
...input.inputArgs,
'-i',
input.path,
'-map',
'0:v:0?',
'-map',
audioMap,
'-c',
'copy',
'-sn',
'-dn',
'-copyts',
'-start_at_zero',
'-f',
'matroska',
'-y',
outputPath,
];
}
/**
* Holds one downloaded window of the current remote stream so the timing review,
* audio extraction, and screenshot all read the same local bytes instead of each
* re-fetching the clip over HTTP. A new window replaces the old one; the file is
* deleted after `idleTtlMs` without use, on `clear()`, or on `cleanup()`.
*/
export class RemoteMediaWindowCache {
private readonly tempDir: string;
private readonly execFile: WindowExecFile;
private readonly idleTtlMs: number;
private readonly logDebug: (message: string) => void;
private current: RemoteMediaWindow | null = null;
private pending: PendingFetch | null = null;
private idleTimer: ReturnType<typeof setTimeout> | null = null;
private sequence = 0;
constructor(options: RemoteMediaWindowCacheOptions = {}) {
this.tempDir = options.tempDir ?? path.join(os.tmpdir(), 'subminer-media-windows');
this.execFile = options.execFile ?? nodeExecFile;
this.idleTtlMs = options.idleTtlMs ?? DEFAULT_IDLE_TTL_MS;
this.logDebug = options.logDebug ?? ((message) => log.debug(message));
}
get currentWindow(): RemoteMediaWindow | null {
return this.current;
}
/** Returns a ready or in-flight window covering the range; never starts a download. */
async lookup(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow | null> {
if (!isUsableRange(range, true)) return null;
if (this.current && covers(this.current, source, range)) {
this.touch();
return this.current;
}
const pending = this.pending;
if (pending && covers(pending, source, range)) {
try {
const window = await pending.promise;
this.touch();
return window;
} catch {
return null;
}
}
return null;
}
/** Returns a window covering the range, downloading (and widening) one when needed. */
async acquire(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow> {
if (!isUsableRange(range, false)) {
throw new Error('Media window range is invalid.');
}
if (range.endTime - range.startTime > REMOTE_MEDIA_WINDOW_MAX_SECONDS) {
throw new Error('Media window range is too long to download.');
}
for (;;) {
const hit = await this.lookup(source, range);
if (hit) return hit;
const pending = this.pending;
if (!pending) break;
// Another caller is already downloading; wait for it, then re-check coverage.
await pending.promise.catch(() => null);
}
return this.fetch(source, this.planFetchRange(source, range));
}
clear(): void {
this.cancelIdleTimer();
const current = this.current;
this.current = null;
if (current) this.removeFile(current.path);
}
cleanup(): void {
this.clear();
try {
fs.rmSync(this.tempDir, { recursive: true, force: true });
} catch (error) {
log.error('Failed to cleanup media window directory:', error);
}
}
private planFetchRange(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): RemoteMediaWindowRange {
let startTime = Math.max(0, range.startTime - HEAD_SLACK_SECONDS);
let endTime = range.endTime + TAIL_SLACK_SECONDS;
const current = this.current;
if (
current &&
current.sourcePath === source.path &&
audioStreamMatches(current.audioStreamIndex, source.audioStreamIndex)
) {
// Keep what was already downloaded when the review timeline grows in one direction.
const unionStart = Math.min(startTime, current.startTime);
const unionEnd = Math.max(endTime, current.endTime);
if (unionEnd - unionStart <= REMOTE_MEDIA_WINDOW_MAX_SECONDS) {
startTime = unionStart;
endTime = unionEnd;
}
}
return { startTime, endTime };
}
private fetch(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow> {
fs.mkdirSync(this.tempDir, { recursive: true });
this.sequence += 1;
const outputPath = path.join(this.tempDir, `window_${Date.now()}_${this.sequence}.mkv`);
const audioStreamIndex =
typeof source.audioStreamIndex === 'number' ? source.audioStreamIndex : null;
const description = describeSourceForDebugLog(source.path);
const startedAt = Date.now();
this.logDebug(
`[media-window] fetch start ${description} start=${range.startTime} end=${range.endTime} audioStream=${audioStreamIndex ?? 'all'}`,
);
const promise = new Promise<RemoteMediaWindow>((resolve, reject) => {
this.execFile(
'ffmpeg',
buildRemoteMediaWindowArgs(source, range, outputPath),
{ timeout: REMOTE_MEDIA_WINDOW_TIMEOUT_MS },
(error) => {
const elapsedMs = Math.max(0, Date.now() - startedAt);
const size = error ? 0 : this.fileSize(outputPath);
if (error || size === 0) {
this.removeFile(outputPath);
const reason = error
? error.code === 'ENOENT'
? 'FFmpeg not found. Install FFmpeg to enable media generation.'
: `FFmpeg media window failed: ${error.message}`
: 'FFmpeg exited without creating a media window.';
this.logDebug(`[media-window] fetch failed ${description} elapsedMs=${elapsedMs}`);
reject(new Error(reason));
return;
}
const window: RemoteMediaWindow = {
path: outputPath,
startTime: range.startTime,
endTime: range.endTime,
sourcePath: source.path,
audioStreamIndex,
media: {
path: outputPath,
source: 'remote-window',
singleResolvedStream: true,
absoluteTimestamps: true,
},
};
this.logDebug(
`[media-window] fetch complete ${description} elapsedMs=${elapsedMs} bytes=${size}`,
);
this.replaceCurrent(window);
resolve(window);
},
);
});
const pending: PendingFetch = {
sourcePath: source.path,
audioStreamIndex,
startTime: range.startTime,
endTime: range.endTime,
promise,
};
this.pending = pending;
promise
.catch(() => undefined)
.then(() => {
if (this.pending === pending) this.pending = null;
});
return promise;
}
private replaceCurrent(window: RemoteMediaWindow): void {
const previous = this.current;
this.current = window;
if (previous && previous.path !== window.path) this.removeFile(previous.path);
this.touch();
}
private touch(): void {
this.cancelIdleTimer();
if (this.idleTtlMs <= 0 || !this.current) return;
const timer = setTimeout(() => {
if (this.idleTimer === timer) this.idleTimer = null;
this.clear();
}, this.idleTtlMs);
timer.unref?.();
this.idleTimer = timer;
}
private cancelIdleTimer(): void {
if (this.idleTimer) clearTimeout(this.idleTimer);
this.idleTimer = null;
}
private fileSize(filePath: string): number {
try {
return fs.statSync(filePath).size;
} catch {
return 0;
}
}
private removeFile(filePath: string): void {
try {
fs.unlinkSync(filePath);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
log.debug(`Failed to remove media window ${filePath}:`, (error as Error).message);
}
}
}
}
let sharedCache: RemoteMediaWindowCache | null = null;
/** Process-wide cache so the review modal and card media generation share one download. */
export function getSharedRemoteMediaWindowCache(): RemoteMediaWindowCache {
sharedCache ??= new RemoteMediaWindowCache();
return sharedCache;
}
+18
View File
@@ -464,6 +464,8 @@ import { handleCliCommandRuntimeServiceWithContext } from './main/cli-runtime';
import { createOverlayModalRuntimeService } from './main/overlay-runtime';
import { createOverlayModalInputState } from './main/runtime/overlay-modal-input-state';
import { MediaTimingPreviewSession } from './core/services/media-timing-preview';
import { getSharedRemoteMediaWindowCache } from './core/services/remote-media-window-cache';
import { resolveMediaGenerationInput } from './anki-integration/media-source';
import { generateSpeechWaveform } from './core/services/media-timing-waveform';
import {
collectMediaTimingContextLines,
@@ -2896,6 +2898,21 @@ const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({
configService.getConfig().mpv.executablePath || process.env.SUBMINER_MPV_PATH?.trim() || '',
createPreviewSession: () => new MediaTimingPreviewSession(),
generateWaveform: (options) => generateSpeechWaveform(options),
resolveMediaSource: async () => {
const resolved = await resolveMediaGenerationInput(appState.mpvClient, 'audio', {
getCachedMediaPath: (currentVideoPath, kind) =>
getCachedYoutubeMediaPathForCurrentPlayback(currentVideoPath, kind),
remoteCacheMode: shouldRequireYoutubeMediaCacheForCurrentPlayback() ? 'required' : 'optional',
});
return resolved
? {
path: resolved.path,
...(resolved.inputOptions ? { inputOptions: resolved.inputOptions } : {}),
singleResolvedStream: resolved.singleResolvedStream,
}
: null;
},
acquireMediaWindow: (source, range) => getSharedRemoteMediaWindowCache().acquire(source, range),
getSubtitleContextLines: (range) =>
collectMediaTimingContextLines({
cues: appState.activeParsedSubtitleCues,
@@ -3973,6 +3990,7 @@ const {
cleanupInternalSubtitleTrackCache: () => cachedInternalSubtitleTrackExtractor.clear(),
cleanupYoutubeSubtitleTempDirs: () => youtubeFlowRuntime.cleanupSubtitleTempDirs(),
cleanupYoutubeMediaCache: () => youtubeMediaCache.cleanup(),
cleanupRemoteMediaWindows: () => getSharedRemoteMediaWindowCache().cleanup(),
cleanupJellyfinSubtitleCache: () => cleanupJellyfinSubtitleCache(),
stopDiscordPresenceService: () => {
void appState.discordPresenceService?.stop();
@@ -46,12 +46,13 @@ test('on will quit cleanup handler runs all cleanup steps', () => {
cleanupInternalSubtitleTrackCache: () => calls.push('cleanup-internal-subtitles'),
cleanupYoutubeSubtitleTempDirs: () => calls.push('cleanup-youtube-subtitles'),
cleanupYoutubeMediaCache: () => calls.push('cleanup-youtube-media'),
cleanupRemoteMediaWindows: () => calls.push('cleanup-remote-media-windows'),
cleanupJellyfinSubtitleCache: () => calls.push('cleanup-jellyfin-subtitles'),
stopDiscordPresenceService: () => calls.push('stop-discord-presence'),
});
cleanup();
assert.equal(calls.length, 35);
assert.equal(calls.length, 36);
assert.equal(calls[0], 'destroy-tray');
assert.equal(calls[calls.length - 1], 'stop-discord-presence');
assert.ok(calls.includes('cleanup-jellyfin-subtitles'));
@@ -60,6 +61,7 @@ test('on will quit cleanup handler runs all cleanup steps', () => {
assert.ok(calls.includes('clear-linux-mpv-fullscreen-overlay-refresh-timeouts'));
assert.ok(calls.includes('cleanup-youtube-subtitles'));
assert.ok(calls.includes('cleanup-youtube-media'));
assert.ok(calls.includes('cleanup-remote-media-windows'));
assert.ok(calls.indexOf('flush-mpv-log') < calls.indexOf('destroy-socket'));
});
@@ -102,6 +104,7 @@ test('on will quit cleanup handler cleans jellyfin subtitle cache when stopping
cleanupInternalSubtitleTrackCache: () => calls.push('cleanup-internal-subtitles'),
cleanupYoutubeSubtitleTempDirs: () => calls.push('cleanup-youtube-subtitles'),
cleanupYoutubeMediaCache: () => calls.push('cleanup-youtube-media'),
cleanupRemoteMediaWindows: () => calls.push('cleanup-remote-media-windows'),
cleanupJellyfinSubtitleCache: () => calls.push('cleanup-jellyfin-subtitles'),
stopDiscordPresenceService: () => calls.push('stop-discord-presence'),
});
@@ -32,6 +32,7 @@ export function createOnWillQuitCleanupHandler(deps: {
cleanupInternalSubtitleTrackCache: () => void;
cleanupYoutubeSubtitleTempDirs: () => void;
cleanupYoutubeMediaCache: () => void;
cleanupRemoteMediaWindows: () => void;
cleanupJellyfinSubtitleCache: () => void;
stopDiscordPresenceService: () => void;
}) {
@@ -76,6 +77,7 @@ export function createOnWillQuitCleanupHandler(deps: {
}
deps.cleanupYoutubeSubtitleTempDirs();
deps.cleanupYoutubeMediaCache();
deps.cleanupRemoteMediaWindows();
deps.stopDiscordPresenceService();
return Promise.resolve(stopSyncAutoScheduler);
};
@@ -75,6 +75,7 @@ test('cleanup deps builder returns handlers that guard optional runtime objects'
cleanupInternalSubtitleTrackCache: () => calls.push('cleanup-internal-subtitles'),
cleanupYoutubeSubtitleTempDirs: () => calls.push('cleanup-youtube-subtitles'),
cleanupYoutubeMediaCache: () => calls.push('cleanup-youtube-media'),
cleanupRemoteMediaWindows: () => calls.push('cleanup-remote-media-windows'),
cleanupJellyfinSubtitleCache: () => calls.push('cleanup-jellyfin-subtitles'),
stopDiscordPresenceService: () => calls.push('stop-discord-presence'),
});
@@ -157,6 +158,7 @@ test('cleanup deps builder skips destroyed yomitan window', () => {
cleanupInternalSubtitleTrackCache: () => {},
cleanupYoutubeSubtitleTempDirs: () => {},
cleanupYoutubeMediaCache: () => {},
cleanupRemoteMediaWindows: () => {},
cleanupJellyfinSubtitleCache: () => {},
stopDiscordPresenceService: () => {},
});
@@ -210,6 +212,7 @@ test('cleanup deps builder skips global shortcut cleanup before app ready', () =
cleanupInternalSubtitleTrackCache: () => {},
cleanupYoutubeSubtitleTempDirs: () => {},
cleanupYoutubeMediaCache: () => {},
cleanupRemoteMediaWindows: () => {},
cleanupJellyfinSubtitleCache: () => {},
stopDiscordPresenceService: () => {},
});
@@ -61,6 +61,7 @@ export function createBuildOnWillQuitCleanupDepsHandler(deps: {
cleanupInternalSubtitleTrackCache: () => void;
cleanupYoutubeSubtitleTempDirs: () => void;
cleanupYoutubeMediaCache: () => void;
cleanupRemoteMediaWindows: () => void;
cleanupJellyfinSubtitleCache: () => void;
stopDiscordPresenceService: () => void;
}) {
@@ -148,6 +149,7 @@ export function createBuildOnWillQuitCleanupDepsHandler(deps: {
cleanupInternalSubtitleTrackCache: () => deps.cleanupInternalSubtitleTrackCache(),
cleanupYoutubeSubtitleTempDirs: () => deps.cleanupYoutubeSubtitleTempDirs(),
cleanupYoutubeMediaCache: () => deps.cleanupYoutubeMediaCache(),
cleanupRemoteMediaWindows: () => deps.cleanupRemoteMediaWindows(),
cleanupJellyfinSubtitleCache: () => deps.cleanupJellyfinSubtitleCache(),
stopDiscordPresenceService: () => deps.stopDiscordPresenceService(),
});
@@ -52,6 +52,7 @@ test('composeStartupLifecycleHandlers returns callable startup lifecycle handler
cleanupInternalSubtitleTrackCache: () => {},
cleanupYoutubeSubtitleTempDirs: () => {},
cleanupYoutubeMediaCache: () => {},
cleanupRemoteMediaWindows: () => {},
cleanupJellyfinSubtitleCache: () => {},
stopDiscordPresenceService: () => {},
},
+312 -9
View File
@@ -1,6 +1,15 @@
import assert from 'node:assert/strict';
import { describe, test } from 'node:test';
import type { MediaTimingReviewOpenPayload } from '../../types/anki';
import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform';
import type {
RemoteMediaWindow,
RemoteMediaWindowRange,
RemoteMediaWindowSource,
} from '../../core/services/remote-media-window-cache';
import type { MediaTimingPreviewSession } from '../../core/services/media-timing-preview';
type MediaTimingPreviewSessionLike = Pick<MediaTimingPreviewSession, 'start'>;
import {
buildMediaTimingReviewPayload,
collectMediaTimingContextLines,
@@ -177,12 +186,7 @@ test('media timing review pauses playback, resolves exact timing, and restores p
});
test('media timing review analyzes the visible range on the selected audio stream', async () => {
const waveformCalls: Array<{
mediaPath: string;
startTime: number;
endTime: number;
audioStreamIndex?: number;
}> = [];
const waveformCalls: SpeechWaveformOptions[] = [];
let runtime: ReturnType<typeof createMediaTimingReviewRuntime>;
runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
@@ -239,6 +243,300 @@ test('media timing review analyzes the visible range on the selected audio strea
]);
});
const REMOTE_STREAM_URL = 'https://jellyfin.example/Videos/abc/stream?static=true';
function createWindowStub(options: { fail?: boolean } = {}) {
const calls: Array<{ source: RemoteMediaWindowSource; range: RemoteMediaWindowRange }> = [];
const acquireMediaWindow = async (
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow> => {
calls.push({ source, range });
if (options.fail) throw new Error('offline');
const windowPath = `/tmp/window-${range.startTime}-${range.endTime}.mkv`;
return {
path: windowPath,
startTime: range.startTime,
endTime: range.endTime,
sourcePath: source.path,
audioStreamIndex: source.audioStreamIndex ?? null,
media: {
path: windowPath,
source: 'remote-window',
singleResolvedStream: true,
absoluteTimestamps: true,
},
};
};
return { calls, acquireMediaWindow };
}
function createRemoteReviewRuntime(options: {
windowStub: ReturnType<typeof createWindowStub>;
waveformCalls: SpeechWaveformOptions[];
previewStarts: Array<Parameters<MediaTimingPreviewSessionLike['start']>[0]>;
previewPlays: Array<[string, number, number]>;
disposed: string[];
openModal: (
runtime: ReturnType<typeof createMediaTimingReviewRuntime>,
payload: MediaTimingReviewOpenPayload,
) => Promise<void>;
}) {
let runtime!: ReturnType<typeof createMediaTimingReviewRuntime>;
runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
connected: true,
currentVideoPath: REMOTE_STREAM_URL,
currentAudioStreamIndex: 2,
requestProperty: async (name) =>
({ pause: true, duration: 100, aid: 3, volume: 60 })[
name as 'pause' | 'duration' | 'aid' | 'volume'
] ?? null,
send: () => undefined,
}),
getCurrentMediaPath: () => REMOTE_STREAM_URL,
getMpvExecutablePath: () => 'mpv',
resolveMediaSource: async () => ({
path: REMOTE_STREAM_URL,
inputOptions: { reconnect: true },
}),
acquireMediaWindow: options.windowStub.acquireMediaWindow,
generateWaveform: async (waveformOptions) => {
options.waveformCalls.push(waveformOptions);
return [0.1, 0.8, 0.2];
},
createPreviewSession: () => {
let mediaPath = '';
return {
start: async (startOptions) => {
mediaPath = startOptions.mediaPath;
options.previewStarts.push(startOptions);
},
play: async (startTime, endTime) => {
options.previewPlays.push([mediaPath, startTime, endTime]);
},
stop: async () => undefined,
dispose: () => {
options.disposed.push(mediaPath);
},
};
},
openModal: async (payload) => {
await options.openModal(runtime, payload);
return true;
},
showStatus: () => undefined,
});
return runtime;
}
test('media timing review downloads one window of a remote stream for the waveform and preview', async () => {
const windowStub = createWindowStub();
const waveformCalls: SpeechWaveformOptions[] = [];
const previewStarts: Array<Parameters<MediaTimingPreviewSessionLike['start']>[0]> = [];
const previewPlays: Array<[string, number, number]> = [];
const disposed: string[] = [];
const runtime = createRemoteReviewRuntime({
windowStub,
waveformCalls,
previewStarts,
previewPlays,
disposed,
openModal: async (active, payload) => {
const waveform = await active.getWaveform({
reviewId: payload.reviewId,
startTime: payload.timelineStartTime,
endTime: payload.timelineEndTime,
});
assert.deepEqual(waveform, { ok: true, peaks: [0.1, 0.8, 0.2] });
assert.deepEqual(
await active.previewRange({ reviewId: payload.reviewId, startTime: 9.5, endTime: 12.5 }),
{ ok: true },
);
active.resolveReview({
reviewId: payload.reviewId,
decision: { action: 'confirm', startTime: 9.5, endTime: 12.5 },
});
},
});
const decision = await runtime.requestReview({
kind: 'word',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0.5,
maxMediaDuration: 30,
});
assert.deepEqual(decision, { action: 'confirm', startTime: 9.5, endTime: 12.5 });
assert.deepEqual(windowStub.calls, [
{
source: { path: REMOTE_STREAM_URL, inputOptions: { reconnect: true }, audioStreamIndex: 2 },
range: { startTime: 7.5, endTime: 14.5 },
},
]);
assert.deepEqual(waveformCalls, [
{
mediaPath: {
path: '/tmp/window-7.5-14.5.mkv',
source: 'remote-window',
singleResolvedStream: true,
absoluteTimestamps: true,
},
startTime: 7.5,
endTime: 14.5,
},
]);
assert.deepEqual(previewStarts, [
{
mediaPath: '/tmp/window-7.5-14.5.mkv',
executablePath: 'mpv',
volume: 60,
absoluteTimestamps: true,
},
]);
assert.deepEqual(previewPlays, [['/tmp/window-7.5-14.5.mkv', 9.5, 12.5]]);
assert.deepEqual(disposed, ['/tmp/window-7.5-14.5.mkv']);
});
test('media timing review restarts the preview on a wider window when the timeline grows', async () => {
const windowStub = createWindowStub();
const waveformCalls: SpeechWaveformOptions[] = [];
const previewStarts: Array<Parameters<MediaTimingPreviewSessionLike['start']>[0]> = [];
const previewPlays: Array<[string, number, number]> = [];
const disposed: string[] = [];
const runtime = createRemoteReviewRuntime({
windowStub,
waveformCalls,
previewStarts,
previewPlays,
disposed,
openModal: async (active, payload) => {
await active.previewRange({ reviewId: payload.reviewId, startTime: 9.5, endTime: 12.5 });
// The user revealed two more seconds before the clip.
await active.getWaveform({ reviewId: payload.reviewId, startTime: 5.5, endTime: 14.5 });
await active.previewRange({ reviewId: payload.reviewId, startTime: 6, endTime: 12.5 });
active.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } });
},
});
await runtime.requestReview({
kind: 'sentence',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0.5,
maxMediaDuration: 30,
});
assert.deepEqual(
windowStub.calls.map((call) => call.range),
[
{ startTime: 7.5, endTime: 14.5 },
{ startTime: 5.5, endTime: 14.5 },
],
);
assert.deepEqual(
previewStarts.map((start) => start.mediaPath),
['/tmp/window-7.5-14.5.mkv', '/tmp/window-5.5-14.5.mkv'],
);
assert.deepEqual(previewPlays, [
['/tmp/window-7.5-14.5.mkv', 9.5, 12.5],
['/tmp/window-5.5-14.5.mkv', 6, 12.5],
]);
assert.deepEqual(disposed, ['/tmp/window-7.5-14.5.mkv', '/tmp/window-5.5-14.5.mkv']);
assert.equal(waveformCalls[0]?.startTime, 5.5);
});
test('media timing review falls back to the remote stream after one failed window download', async () => {
const windowStub = createWindowStub({ fail: true });
const waveformCalls: SpeechWaveformOptions[] = [];
const previewStarts: Array<Parameters<MediaTimingPreviewSessionLike['start']>[0]> = [];
const previewPlays: Array<[string, number, number]> = [];
const disposed: string[] = [];
const runtime = createRemoteReviewRuntime({
windowStub,
waveformCalls,
previewStarts,
previewPlays,
disposed,
openModal: async (active, payload) => {
await active.getWaveform({
reviewId: payload.reviewId,
startTime: payload.timelineStartTime,
endTime: payload.timelineEndTime,
});
await active.previewRange({ reviewId: payload.reviewId, startTime: 9.5, endTime: 12.5 });
active.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } });
},
});
await runtime.requestReview({
kind: 'word',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0.5,
maxMediaDuration: 30,
});
assert.equal(windowStub.calls.length, 1);
assert.deepEqual(waveformCalls, [
{
mediaPath: { path: REMOTE_STREAM_URL, inputOptions: { reconnect: true } },
startTime: 7.5,
endTime: 14.5,
audioStreamIndex: 2,
},
]);
assert.deepEqual(previewStarts, [
{ mediaPath: REMOTE_STREAM_URL, executablePath: 'mpv', volume: 60, audioTrackId: 3 },
]);
assert.deepEqual(previewPlays, [[REMOTE_STREAM_URL, 9.5, 12.5]]);
});
test('media timing review never downloads windows for local media', async () => {
const windowStub = createWindowStub();
let runtime!: ReturnType<typeof createMediaTimingReviewRuntime>;
runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
connected: true,
currentVideoPath: '/video/show.mkv',
requestProperty: async (name) => (name === 'duration' ? 100 : name === 'pause' ? true : null),
send: () => undefined,
}),
getCurrentMediaPath: () => '/video/show.mkv',
getMpvExecutablePath: () => 'mpv',
resolveMediaSource: async () => ({ path: '/video/show.mkv' }),
acquireMediaWindow: windowStub.acquireMediaWindow,
generateWaveform: async () => [0.1, 0.8, 0.2],
createPreviewSession: () => ({
start: async () => undefined,
play: async () => undefined,
stop: async () => undefined,
dispose: () => undefined,
}),
openModal: async (payload) => {
await runtime.getWaveform({ reviewId: payload.reviewId, startTime: 7.5, endTime: 14.5 });
runtime.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } });
return true;
},
showStatus: () => undefined,
});
await runtime.requestReview({
kind: 'sentence',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0.5,
maxMediaDuration: 30,
});
assert.equal(windowStub.calls.length, 0);
});
test('media timing review rejects stale and out-of-range actions before allowing discard', async () => {
const { runtime, payload, pendingDecision, previewCalls } = await startActiveMediaTimingReview({
maxMediaDuration: 3,
@@ -384,11 +682,16 @@ test('media timing review restores playback when setup fails after pausing', asy
send: ({ command }) => commands.push(command),
}),
getCurrentMediaPath: () => '/video/show.mkv',
getMpvExecutablePath: () => 'mpv',
generateWaveform: async () => [],
createPreviewSession: () => {
getMpvExecutablePath: () => {
throw new Error('preview setup failed');
},
generateWaveform: async () => [],
createPreviewSession: () => ({
start: async () => undefined,
play: async () => undefined,
stop: async () => undefined,
dispose: () => undefined,
}),
openModal: async () => true,
showStatus: () => undefined,
});
+158 -28
View File
@@ -11,6 +11,13 @@ import type {
MediaTimingReviewWaveformResult,
} from '../../types/anki';
import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform';
import {
isRemoteMediaWindowSourcePath,
type RemoteMediaWindow,
type RemoteMediaWindowRange,
type RemoteMediaWindowSource,
} from '../../core/services/remote-media-window-cache';
import type { MediaInput, MediaInputOptions } from '../../media-input';
const INITIAL_TIMELINE_MARGIN_SECONDS = 2;
const REVIEW_DECISION_TIMEOUT_MS = 5 * 60_000;
@@ -31,19 +38,36 @@ interface PreviewSession {
executablePath?: string;
audioTrackId?: number;
volume?: number;
absoluteTimestamps?: boolean;
}): Promise<void>;
play(startTime: number, endTime: number): Promise<void>;
stop(): Promise<void>;
dispose(): void;
}
interface ReviewMediaSource {
path: string;
inputOptions?: MediaInputOptions;
singleResolvedStream?: boolean;
}
interface ActiveReview {
payload: MediaTimingReviewOpenPayload;
/** What the hidden mpv preview plays when no cached window is available. */
mediaPath: string;
/** What the waveform reads when no cached window is available. */
waveformMedia: MediaInput;
audioStreamIndex?: number;
/** Remote source to download windows of; null for local media or without a cache. */
windowSource: RemoteMediaWindowSource | null;
/** Latest window returned for this review; reused while it still covers the request. */
window: RemoteMediaWindow | null;
windowRequest: (RemoteMediaWindowRange & { promise: Promise<RemoteMediaWindow | null> }) | null;
windowFailed: boolean;
previewOptions: { executablePath?: string; audioTrackId?: number; volume?: number };
preview: { path: string; session: Promise<PreviewSession> } | null;
mpvClient: ReviewMpvClient;
restorePlayback: boolean;
preview: Promise<PreviewSession>;
resolve: (decision: MediaTimingReviewDecision) => void;
}
@@ -53,6 +77,13 @@ export interface MediaTimingReviewRuntimeDeps {
getMpvExecutablePath: () => string;
createPreviewSession: () => PreviewSession;
generateWaveform: (options: SpeechWaveformOptions) => Promise<number[]>;
/** Resolves the FFmpeg-readable stream URL and headers behind the current media path. */
resolveMediaSource?: () => Promise<ReviewMediaSource | null>;
/** Downloads (or reuses) a local window of a remote source covering the range. */
acquireMediaWindow?: (
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
) => Promise<RemoteMediaWindow>;
getSubtitleContextLines?: (range: { startTime: number; endTime: number }) => {
previous: MediaTimingReviewContextLine[];
next: MediaTimingReviewContextLine[];
@@ -199,6 +230,84 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
}
}
function ensureWindow(
review: ActiveReview,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow | null> {
const { windowSource } = review;
if (!windowSource || review.windowFailed || !deps.acquireMediaWindow) {
return Promise.resolve(null);
}
const coversRange = (candidate: RemoteMediaWindowRange): boolean =>
candidate.startTime <= range.startTime && candidate.endTime >= range.endTime;
if (review.window && coversRange(review.window)) return Promise.resolve(review.window);
const inFlight = review.windowRequest;
if (inFlight && coversRange(inFlight)) return inFlight.promise;
const request = {
startTime: range.startTime,
endTime: range.endTime,
promise: Promise.resolve<RemoteMediaWindow | null>(null),
};
request.promise = deps
.acquireMediaWindow(windowSource, { startTime: range.startTime, endTime: range.endTime })
.then((window) => {
review.window = window;
return window;
})
.catch(() => {
// Fall back to the remote source for the rest of this review instead of retrying.
review.windowFailed = true;
return null;
})
.finally(() => {
if (review.windowRequest === request) review.windowRequest = null;
});
review.windowRequest = request;
return request.promise;
}
/**
* Returns the preview player for the range, restarting it when the range needs a
* different file (the first cached window, or a wider one after the timeline grew).
*/
async function previewFor(
review: ActiveReview,
range: RemoteMediaWindowRange,
): Promise<PreviewSession> {
const window = await ensureWindow(review, range);
if (active !== review) {
// The review ended during the download; do not start a player nobody will dispose.
throw new Error('This timing review is no longer active.');
}
const mediaPath = window?.path ?? review.mediaPath;
if (review.preview?.path === mediaPath) return review.preview.session;
const previous = review.preview;
const session = deps.createPreviewSession();
const { audioTrackId, ...previewOptions } = review.previewOptions;
const started = session
.start({
mediaPath,
...previewOptions,
// A cached window keeps one audio stream, so mpv's track id from the source no longer applies.
...(window
? { absoluteTimestamps: true }
: audioTrackId !== undefined
? { audioTrackId }
: {}),
})
.then(() => session)
.catch((error) => {
session.dispose();
throw error;
});
review.preview = { path: mediaPath, session: started };
void started.catch(() => {});
if (previous) void previous.session.then((old) => old.dispose()).catch(() => {});
return started;
}
async function runReview(request: MediaTimingReviewRequest): Promise<MediaTimingReviewDecision> {
const mpvClient = deps.getMpvClient();
const mediaPath =
@@ -208,11 +317,12 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
return { action: 'use-original' };
}
const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw] = await Promise.all([
const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw, resolvedSource] = await Promise.all([
mpvClient.requestProperty?.('pause').catch(() => null) ?? null,
mpvClient.requestProperty?.('duration').catch(() => null) ?? null,
mpvClient.requestProperty?.('aid').catch(() => null) ?? null,
mpvClient.requestProperty?.('volume').catch(() => null) ?? null,
deps.resolveMediaSource?.().catch(() => null) ?? null,
]);
const pauseState = booleanProperty(pauseRaw);
mpvClient.send({ command: ['set_property', 'pause', 'yes'] });
@@ -232,38 +342,51 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
mediaDuration: finiteNumber(durationRaw) ?? undefined,
...(contextLines ? { contextLines } : {}),
});
const previewSession = deps.createPreviewSession();
const preview = previewSession
.start({
mediaPath,
executablePath: deps.getMpvExecutablePath(),
audioTrackId: finiteNumber(audioTrackRaw) ?? undefined,
volume: finiteNumber(volumeRaw) ?? undefined,
})
.then(() => previewSession)
.catch((error) => {
previewSession.dispose();
throw error;
});
void preview.catch(() => {});
const sourcePath = resolvedSource?.path.trim() || mediaPath;
const inputOptions = resolvedSource?.inputOptions;
const audioStreamIndex =
resolvedSource?.singleResolvedStream || mpvClient.currentAudioStreamIndex == null
? undefined
: mpvClient.currentAudioStreamIndex;
const windowSource: RemoteMediaWindowSource | null =
deps.acquireMediaWindow && isRemoteMediaWindowSourcePath(sourcePath)
? {
path: sourcePath,
...(inputOptions ? { inputOptions } : {}),
audioStreamIndex: audioStreamIndex ?? null,
}
: null;
let resolveDecision!: (decision: MediaTimingReviewDecision) => void;
const decisionPromise = new Promise<MediaTimingReviewDecision>((resolve) => {
resolveDecision = resolve;
});
active = {
const review: ActiveReview = {
payload,
mediaPath,
...(mpvClient.currentAudioStreamIndex !== null &&
mpvClient.currentAudioStreamIndex !== undefined
? { audioStreamIndex: mpvClient.currentAudioStreamIndex }
: {}),
waveformMedia: inputOptions ? { path: sourcePath, inputOptions } : sourcePath,
...(audioStreamIndex !== undefined ? { audioStreamIndex } : {}),
windowSource,
window: null,
windowRequest: null,
windowFailed: false,
previewOptions: {
executablePath: deps.getMpvExecutablePath(),
audioTrackId: finiteNumber(audioTrackRaw) ?? undefined,
volume: finiteNumber(volumeRaw) ?? undefined,
},
preview: null,
mpvClient,
restorePlayback: pendingPauseRestore === mpvClient,
preview,
resolve: resolveDecision,
};
active = review;
pendingPauseRestore = null;
// Download the visible timeline once now; the waveform and preview both wait on it.
void previewFor(review, {
startTime: payload.timelineStartTime,
endTime: payload.timelineEndTime,
}).catch(() => {});
const opened = await deps.openModal(payload).catch(() => false);
if (!opened) {
@@ -317,7 +440,10 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
return { ok: false, message: 'The selected preview range is invalid.' };
}
try {
const previewSession = await current.preview;
const previewSession = await previewFor(current, request);
if (active !== current) {
return { ok: false, message: 'This timing review is no longer active.' };
}
await previewSession.play(request.startTime, request.endTime);
return { ok: true };
} catch (error) {
@@ -347,11 +473,15 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
}
try {
const window = await ensureWindow(current, request);
if (active !== current) {
return { ok: false, message: 'This timing review is no longer active.' };
}
const peaks = await deps.generateWaveform({
mediaPath: current.mediaPath,
mediaPath: window?.media ?? current.waveformMedia,
startTime: request.startTime,
endTime: request.endTime,
...(current.audioStreamIndex !== undefined
...(!window && current.audioStreamIndex !== undefined
? { audioStreamIndex: current.audioStreamIndex }
: {}),
});
@@ -370,8 +500,8 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
return { ok: false, message: 'This timing review is no longer active.' };
}
try {
const previewSession = await current.preview;
await previewSession.stop();
const previewSession = current.preview ? await current.preview.session : null;
await previewSession?.stop();
return { ok: true };
} catch (error) {
return {
@@ -403,7 +533,7 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
const current = active;
active = null;
if (!current) return;
void current.preview.then((session) => session.dispose()).catch(() => {});
void current.preview?.session.then((session) => session.dispose()).catch(() => {});
if (current.restorePlayback && current.mpvClient.connected) {
current.mpvClient.send({ command: ['set_property', 'pause', 'no'] });
}
+141 -31
View File
@@ -10,6 +10,9 @@ import {
MediaGenerator,
type MediaGeneratorOptions,
} from './media-generator';
import { RemoteMediaWindowCache } from './core/services/remote-media-window-cache';
const REMOTE_STREAM_URL = 'https://jellyfin.example/Videos/abc/stream?static=true';
async function withStubbedFfmpeg(
run: (generator: MediaGenerator, argsPath: string) => Promise<void>,
@@ -21,6 +24,7 @@ async function withStubbedFfmpeg(
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-media-generator-test-'));
const binDir = path.join(root, 'bin');
const tempDir = path.join(root, 'media');
const windowsDir = path.join(root, 'windows');
const argsPath = path.join(root, 'ffmpeg-args.txt');
fs.mkdirSync(binDir, { recursive: true });
const ffmpegStubPath = path.join(binDir, 'ffmpeg-stub.cjs');
@@ -34,7 +38,7 @@ async function withStubbedFfmpeg(
" console.log(' V..... libaom-av1');",
' process.exit(0);',
'}',
"fs.writeFileSync(process.env.SUBMINER_TEST_FFMPEG_ARGS, JSON.stringify(args), 'utf8');",
"fs.appendFileSync(process.env.SUBMINER_TEST_FFMPEG_ARGS, JSON.stringify(args) + '\\n', 'utf8');",
'const outputPath = args.at(-1);',
"if (process.env.SUBMINER_TEST_FFMPEG_SKIP_OUTPUT !== '1') {",
" fs.writeFileSync(outputPath, 'avif', 'utf8');",
@@ -61,12 +65,18 @@ async function withStubbedFfmpeg(
} else {
delete process.env.SUBMINER_TEST_FFMPEG_SKIP_OUTPUT;
}
const generator = new MediaGenerator(tempDir, options);
// Each test gets its own window cache so remote inputs never leak windows between tests.
const remoteMediaWindows = new RemoteMediaWindowCache({ tempDir: windowsDir, idleTtlMs: 0 });
const generator = new MediaGenerator(tempDir, {
remoteMediaWindows,
...options,
});
try {
await run(generator, argsPath);
} finally {
generator.cleanup();
remoteMediaWindows.cleanup();
process.env.PATH = originalPath;
if (originalArgsPath === undefined) {
delete process.env.SUBMINER_TEST_FFMPEG_ARGS;
@@ -82,8 +92,17 @@ async function withStubbedFfmpeg(
}
}
function readAllFfmpegArgs(argsPath: string): string[][] {
return fs
.readFileSync(argsPath, 'utf8')
.split('\n')
.filter((line) => line.trim().length > 0)
.map((line) => JSON.parse(line) as string[]);
}
/** Arguments of the most recent ffmpeg invocation. */
function readFfmpegArgs(argsPath: string): string[] {
return JSON.parse(fs.readFileSync(argsPath, 'utf8')) as string[];
return readAllFfmpegArgs(argsPath).at(-1) ?? [];
}
test('buildAnimatedImageVideoFilter holds lead-in until the next frame after the audio boundary', () => {
@@ -272,41 +291,131 @@ test('generateAudio recreates missing temp directory before invoking ffmpeg', as
});
test('generateAudio adds remote input options before the ffmpeg input', async () => {
await withStubbedFfmpeg(async (generator, argsPath) => {
await generator.generateAudio(
{
path: 'https://rr1---sn.example.googlevideo.com/videoplayback?mime=audio%2Fwebm',
inputOptions: {
reconnect: true,
userAgent: 'Mozilla/5.0',
headers: {
Referer: 'https://www.youtube.com/',
Origin: 'https://www.youtube.com',
await withStubbedFfmpeg(
async (generator, argsPath) => {
await generator.generateAudio(
{
path: 'https://rr1---sn.example.googlevideo.com/videoplayback?mime=audio%2Fwebm',
inputOptions: {
reconnect: true,
userAgent: 'Mozilla/5.0',
headers: {
Referer: 'https://www.youtube.com/',
Origin: 'https://www.youtube.com',
},
},
},
},
10,
12,
);
const args = readFfmpegArgs(argsPath);
const inputIndex = args.indexOf('-i');
assert.ok(inputIndex > 0);
assert.ok(args.indexOf('-reconnect') > -1);
assert.ok(args.indexOf('-reconnect') < inputIndex);
assert.equal(args[args.indexOf('-reconnect') + 1], '1');
assert.equal(args[args.indexOf('-reconnect_streamed') + 1], '1');
assert.equal(args[args.indexOf('-reconnect_on_network_error') + 1], '1');
assert.equal(args[args.indexOf('-reconnect_on_http_error') + 1], '403,5xx');
assert.equal(args[args.indexOf('-reconnect_delay_max') + 1], '5');
assert.equal(args[args.indexOf('-user_agent') + 1], 'Mozilla/5.0');
assert.equal(
args[args.indexOf('-headers') + 1],
'Referer: https://www.youtube.com/\r\nOrigin: https://www.youtube.com\r\n',
);
},
{ remoteMediaWindows: null },
);
});
test('generateAudio downloads a remote window once and extracts from it with absolute seeks', async () => {
await withStubbedFfmpeg(async (generator, argsPath) => {
await generator.generateAudio(
{ path: REMOTE_STREAM_URL, inputOptions: { reconnect: true } },
10,
12,
0.5,
2,
);
const args = readFfmpegArgs(argsPath);
const inputIndex = args.indexOf('-i');
assert.ok(inputIndex > 0);
assert.ok(args.indexOf('-reconnect') > -1);
assert.ok(args.indexOf('-reconnect') < inputIndex);
assert.equal(args[args.indexOf('-reconnect') + 1], '1');
assert.equal(args[args.indexOf('-reconnect_streamed') + 1], '1');
assert.equal(args[args.indexOf('-reconnect_on_network_error') + 1], '1');
assert.equal(args[args.indexOf('-reconnect_on_http_error') + 1], '403,5xx');
assert.equal(args[args.indexOf('-reconnect_delay_max') + 1], '5');
assert.equal(args[args.indexOf('-user_agent') + 1], 'Mozilla/5.0');
assert.equal(
args[args.indexOf('-headers') + 1],
'Referer: https://www.youtube.com/\r\nOrigin: https://www.youtube.com\r\n',
);
const calls = readAllFfmpegArgs(argsPath);
assert.equal(calls.length, 2);
const [fetchArgs, audioArgs] = calls as [string[], string[]];
assert.equal(fetchArgs[fetchArgs.indexOf('-i') + 1], REMOTE_STREAM_URL);
assert.ok(fetchArgs.indexOf('-reconnect') < fetchArgs.indexOf('-i'));
assert.equal(fetchArgs[fetchArgs.indexOf('-ss') + 1], '9.25');
assert.equal(fetchArgs[fetchArgs.lastIndexOf('-map') + 1], '0:2');
assert.ok(fetchArgs.includes('-copyts'));
const windowPath = audioArgs[audioArgs.indexOf('-i') + 1];
assert.ok(windowPath?.endsWith('.mkv'));
assert.notEqual(windowPath, REMOTE_STREAM_URL);
assert.equal(audioArgs[audioArgs.indexOf('-ss') + 1], '9.5');
assert.equal(audioArgs[audioArgs.indexOf('-seek_timestamp') + 1], '1');
assert.ok(audioArgs.indexOf('-seek_timestamp') < audioArgs.indexOf('-i'));
assert.equal(audioArgs.includes('-reconnect'), false);
assert.equal(audioArgs.includes('-map'), false);
assert.equal(audioArgs.includes('-probesize'), false);
assert.ok(audioArgs.includes('loudnorm=I=-23:TP=-2:LRA=11'));
});
});
test('generateScreenshot reuses a downloaded window but never downloads one itself', async () => {
await withStubbedFfmpeg(async (generator, argsPath) => {
await generator.generateScreenshot(REMOTE_STREAM_URL, 11, { format: 'jpg' });
let calls = readAllFfmpegArgs(argsPath);
assert.equal(calls.length, 1);
assert.equal(calls[0]![calls[0]!.indexOf('-i') + 1], REMOTE_STREAM_URL);
await generator.generateAudio(REMOTE_STREAM_URL, 10, 12);
await generator.generateScreenshot(REMOTE_STREAM_URL, 11, { format: 'jpg' });
await generator.generateScreenshot(REMOTE_STREAM_URL, 40, { format: 'jpg' });
calls = readAllFfmpegArgs(argsPath);
assert.equal(calls.length, 5);
const insideWindow = calls[3]!;
assert.ok(insideWindow[insideWindow.indexOf('-i') + 1]?.endsWith('.mkv'));
assert.equal(insideWindow[insideWindow.indexOf('-seek_timestamp') + 1], '1');
const outsideWindow = calls[4]!;
assert.equal(outsideWindow[outsideWindow.indexOf('-i') + 1], REMOTE_STREAM_URL);
});
});
test('generateAnimatedImage downloads the clip window before encoding', async () => {
await withStubbedFfmpeg(async (generator, argsPath) => {
await generator.generateAnimatedImage(REMOTE_STREAM_URL, 10, 12, 0, { fps: 10 });
const calls = readAllFfmpegArgs(argsPath).filter(
(args) => args[0] !== '-hide_banner' || args[1] !== '-encoders',
);
assert.equal(calls.length, 2);
assert.equal(calls[0]![calls[0]!.indexOf('-i') + 1], REMOTE_STREAM_URL);
assert.ok(calls[1]![calls[1]!.indexOf('-i') + 1]?.endsWith('.mkv'));
assert.equal(calls[1]![calls[1]!.indexOf('-seek_timestamp') + 1], '1');
});
});
test('generateAudio reads the remote source directly when the window download fails', async () => {
await withStubbedFfmpeg(
async (generator, argsPath) => {
await generator.generateAudio(REMOTE_STREAM_URL, 10, 12);
const args = readFfmpegArgs(argsPath);
assert.equal(args[args.indexOf('-i') + 1], REMOTE_STREAM_URL);
assert.equal(args.includes('-seek_timestamp'), false);
},
{
remoteMediaWindows: new RemoteMediaWindowCache({
execFile: (_file, _args, _options, callback) =>
queueMicrotask(() => callback(Object.assign(new Error('offline'), { code: 1 }))),
idleTtlMs: 0,
logDebug: () => undefined,
}),
},
);
});
test('generateAudio skips stale audio stream maps for single resolved streams', async () => {
await withStubbedFfmpeg(async (generator, argsPath) => {
await generator.generateAudio(
@@ -320,8 +429,9 @@ test('generateAudio skips stale audio stream maps for single resolved streams',
22,
);
const args = readFfmpegArgs(argsPath);
assert.equal(args.includes('-map'), false);
const [fetchArgs, audioArgs] = readAllFfmpegArgs(argsPath) as [string[], string[]];
assert.equal(fetchArgs[fetchArgs.lastIndexOf('-map') + 1], '0:a');
assert.equal(audioArgs.includes('-map'), false);
});
});
+84 -3
View File
@@ -22,6 +22,12 @@ import * as path from 'path';
import * as os from 'os';
import { createLogger } from './logger';
import { normalizeMediaInput, type MediaInput } from './media-input';
import {
getSharedRemoteMediaWindowCache,
isRemoteMediaWindowSourcePath,
type RemoteMediaWindowCache,
type RemoteMediaWindowRange,
} from './core/services/remote-media-window-cache';
const log = createLogger('media');
const AUDIO_NORMALIZATION_FILTER = 'loudnorm=I=-23:TP=-2:LRA=11';
@@ -86,6 +92,11 @@ export interface MediaGeneratorOptions {
logDebug?: (message: string) => void;
now?: () => number;
execFile?: MediaGeneratorExecFile;
/**
* Local window cache for http(s) sources. Defaults to the process-wide cache shared
* with the timing review; pass `null` to always read remote sources directly.
*/
remoteMediaWindows?: RemoteMediaWindowCache | null;
}
function sanitizeDebugToken(value: string, fallback: string): string {
@@ -232,6 +243,54 @@ export class MediaGenerator {
}, delayMs);
}
/**
* Swaps an http(s) input for the locally cached window that covers `range`, so the
* clip is downloaded once instead of per FFmpeg run. `acquire` downloads on a miss;
* `lookup` only reuses a window that another step already fetched. Any failure falls
* back to reading the remote source directly.
*/
private async resolveRemoteWindowInput(
input: MediaInput,
range: RemoteMediaWindowRange,
audioStreamIndex: number | null | undefined,
mode: 'acquire' | 'lookup',
): Promise<MediaInput> {
const cache =
this.options.remoteMediaWindows === undefined
? getSharedRemoteMediaWindowCache()
: this.options.remoteMediaWindows;
const sourcePath = typeof input === 'string' ? input : input.path;
if (!cache || !isRemoteMediaWindowSourcePath(sourcePath)) {
return input;
}
const source = {
path: sourcePath,
...(typeof input === 'object' && input.inputOptions
? { inputOptions: input.inputOptions }
: {}),
audioStreamIndex:
typeof input === 'object' && input.singleResolvedStream ? null : (audioStreamIndex ?? null),
};
const description = describeMediaInputForDebugLog(input);
try {
const window =
mode === 'acquire' ? await cache.acquire(source, range) : await cache.lookup(source, range);
if (!window) {
this.logMediaDebug(`window miss ${description} mode=${mode}`);
return input;
}
this.logMediaDebug(
`window hit ${description} mode=${mode} start=${window.startTime} end=${window.endTime}`,
);
return window.media;
} catch (error) {
this.logMediaDebug(
`window failed ${description} mode=${mode} reason=${sanitizeDebugToken((error as Error).message, 'error')}`,
);
return input;
}
}
private ffmpegError(label: string, error: ExecFileException): Error {
if (error.code === 'ENOENT') {
return new Error('FFmpeg not found. Install FFmpeg to enable media generation.');
@@ -281,7 +340,13 @@ export class MediaGenerator {
const safePadding = Number.isFinite(padding) ? Math.max(0, padding) : 0;
const start = Math.max(0, startTime - safePadding);
const duration = endTime - start + safePadding;
const mediaInput = normalizeMediaInput(videoPath);
const sourceInput = await this.resolveRemoteWindowInput(
videoPath,
{ startTime: start, endTime: start + duration },
audioStreamIndex,
'acquire',
);
const mediaInput = normalizeMediaInput(sourceInput);
const inputDescription = describeMediaInputForDebugLog(videoPath);
const hasSelectedAudioStream =
!mediaInput.singleResolvedStream &&
@@ -385,7 +450,15 @@ export class MediaGenerator {
png: 'png',
webp: 'webp',
};
const mediaInput = normalizeMediaInput(videoPath);
// A single frame is cheap to fetch remotely, so only reuse a window another step downloaded.
const mediaInput = normalizeMediaInput(
await this.resolveRemoteWindowInput(
videoPath,
{ startTime: timestamp, endTime: timestamp },
null,
'lookup',
),
);
const inputDescription = describeMediaInputForDebugLog(videoPath);
const args: string[] = [
@@ -533,9 +606,17 @@ export class MediaGenerator {
);
}
const mediaInput = normalizeMediaInput(
await this.resolveRemoteWindowInput(
videoPath,
{ startTime: start, endTime: start + duration },
null,
'acquire',
),
);
return new Promise((resolve, reject) => {
const outputPath = this.createTempOutputPath('animation', 'avif');
const mediaInput = normalizeMediaInput(videoPath);
const startedAt = this.nowMs();
const encoderArgs: string[] = ['-c:v', av1Encoder];
+10
View File
@@ -11,6 +11,12 @@ export type MediaInput =
source?: string;
inputOptions?: MediaInputOptions;
singleResolvedStream?: boolean;
/**
* The file keeps the original media timestamps instead of starting at zero (a
* stream-copied window of a longer source). Seek with `-ss` against those
* absolute timestamps rather than relative to the file's own start time.
*/
absoluteTimestamps?: boolean;
};
export type NormalizedMediaInput = {
@@ -89,6 +95,10 @@ export function normalizeMediaInput(input: MediaInput): NormalizedMediaInput {
inputArgs.push('-headers', headers);
}
if (input.absoluteTimestamps) {
inputArgs.push('-seek_timestamp', '1');
}
return {
path: input.path,
inputArgs,