mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-09-11 17:16:18 -07:00
feat(mining): cache remote media windows during card creation
- Reuse one temporary download for timing review, audio, and screenshots - Expire cached windows after inactivity and clean them up on exit
This commit is contained in:
@@ -23,6 +23,21 @@ describe('buildMediaTimingPreviewArgs', () => {
|
||||
assert.equal(args.at(-1), '/video/show.mkv');
|
||||
});
|
||||
|
||||
test('keeps source timestamps for cached remote windows', () => {
|
||||
const args = buildMediaTimingPreviewArgs('/tmp/review.sock', {
|
||||
mediaPath: '/tmp/window.mkv',
|
||||
absoluteTimestamps: true,
|
||||
});
|
||||
|
||||
assert.ok(args.includes('--rebase-start-time=no'));
|
||||
assert.equal(
|
||||
buildMediaTimingPreviewArgs('/tmp/review.sock', { mediaPath: '/video/show.mkv' }).includes(
|
||||
'--rebase-start-time=no',
|
||||
),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('separates an option-like media path without adding optional audio arguments', () => {
|
||||
const args = buildMediaTimingPreviewArgs('/tmp/review.sock', {
|
||||
mediaPath: '--fullscreen',
|
||||
|
||||
@@ -14,6 +14,8 @@ export interface MediaTimingPreviewStartOptions {
|
||||
executablePath?: string;
|
||||
audioTrackId?: number;
|
||||
volume?: number;
|
||||
/** The file keeps source timestamps (a cached remote window); seek with the original times. */
|
||||
absoluteTimestamps?: boolean;
|
||||
}
|
||||
|
||||
type PreviewProcess = Pick<ChildProcess, 'kill' | 'once'>;
|
||||
@@ -51,6 +53,9 @@ export function buildMediaTimingPreviewArgs(
|
||||
if (typeof options.volume === 'number' && Number.isFinite(options.volume)) {
|
||||
args.push(`--volume=${Math.max(0, options.volume)}`);
|
||||
}
|
||||
if (options.absoluteTimestamps) {
|
||||
args.push('--rebase-start-time=no');
|
||||
}
|
||||
args.push('--', options.mediaPath);
|
||||
return args;
|
||||
}
|
||||
|
||||
@@ -35,6 +35,29 @@ test('speech waveform maps the selected FFmpeg stream and visible range', () =>
|
||||
assert.match(args[args.indexOf('-af') + 1] ?? '', /c0=FC/);
|
||||
});
|
||||
|
||||
test('speech waveform seeks cached windows by source timestamps', () => {
|
||||
const args = buildSpeechWaveformArgs(
|
||||
{
|
||||
mediaPath: { path: '/tmp/window.mkv', absoluteTimestamps: true, singleResolvedStream: true },
|
||||
startTime: 8,
|
||||
endTime: 15,
|
||||
},
|
||||
'downmix',
|
||||
);
|
||||
|
||||
assert.deepEqual(args.slice(args.indexOf('-ss'), args.indexOf('-t') + 2), [
|
||||
'-ss',
|
||||
'8',
|
||||
'-seek_timestamp',
|
||||
'1',
|
||||
'-i',
|
||||
'/tmp/window.mkv',
|
||||
'-t',
|
||||
'7',
|
||||
]);
|
||||
assert.equal(args.includes('-map'), false);
|
||||
});
|
||||
|
||||
test('waveform peaks are normalized without flattening quieter sections', () => {
|
||||
const peaks = computeWaveformPeaks(pcm([0, 1_000, -2_000, 4_000, -8_000, 16_000]), 3);
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { normalizeMediaInput, type MediaInput } from '../../media-input';
|
||||
|
||||
const WAVEFORM_SAMPLE_RATE = 8_000;
|
||||
const WAVEFORM_POINT_COUNT = 480;
|
||||
@@ -9,7 +10,7 @@ const CENTER_CHANNEL_FILTER = `pan=mono|c0=FC,${SPEECH_FILTER}`;
|
||||
const DOWNMIX_FILTER = `aformat=channel_layouts=mono,${SPEECH_FILTER}`;
|
||||
|
||||
export interface SpeechWaveformOptions {
|
||||
mediaPath: string;
|
||||
mediaPath: MediaInput;
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
audioStreamIndex?: number;
|
||||
@@ -22,6 +23,7 @@ export function buildSpeechWaveformArgs(
|
||||
mode: 'center' | 'downmix',
|
||||
): string[] {
|
||||
const duration = options.endTime - options.startTime;
|
||||
const input = normalizeMediaInput(options.mediaPath);
|
||||
const args = [
|
||||
'-hide_banner',
|
||||
'-nostdin',
|
||||
@@ -29,8 +31,9 @@ export function buildSpeechWaveformArgs(
|
||||
'error',
|
||||
'-ss',
|
||||
String(options.startTime),
|
||||
...input.inputArgs,
|
||||
'-i',
|
||||
options.mediaPath,
|
||||
input.path,
|
||||
'-t',
|
||||
String(duration),
|
||||
];
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
buildRemoteMediaWindowArgs,
|
||||
RemoteMediaWindowCache,
|
||||
REMOTE_MEDIA_WINDOW_MAX_SECONDS,
|
||||
type RemoteMediaWindowCacheOptions,
|
||||
} from './remote-media-window-cache';
|
||||
|
||||
const SOURCE = {
|
||||
path: 'https://jellyfin.example/Videos/abc/stream?static=true',
|
||||
audioStreamIndex: 2,
|
||||
};
|
||||
|
||||
type ExecFileStub = NonNullable<RemoteMediaWindowCacheOptions['execFile']>;
|
||||
|
||||
function createStub(options: { fail?: boolean; empty?: boolean; defer?: boolean } = {}) {
|
||||
const calls: string[][] = [];
|
||||
const pendingCallbacks: Array<() => void> = [];
|
||||
const execFile: ExecFileStub = (_file, args, _options, callback) => {
|
||||
calls.push([...args]);
|
||||
const finish = (): void => {
|
||||
const outputPath = args.at(-1);
|
||||
assert.ok(outputPath);
|
||||
if (options.fail) {
|
||||
callback(Object.assign(new Error('boom'), { code: 1 }));
|
||||
return;
|
||||
}
|
||||
if (!options.empty) {
|
||||
fs.writeFileSync(outputPath, 'mkv', 'utf8');
|
||||
}
|
||||
callback(null);
|
||||
};
|
||||
if (options.defer) {
|
||||
pendingCallbacks.push(finish);
|
||||
} else {
|
||||
queueMicrotask(finish);
|
||||
}
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
execFile,
|
||||
flush: () => {
|
||||
for (const finish of pendingCallbacks.splice(0)) finish();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function withCache(
|
||||
stubOptions: Parameters<typeof createStub>[0],
|
||||
cacheOptions: Omit<RemoteMediaWindowCacheOptions, 'execFile' | 'tempDir'>,
|
||||
run: (cache: RemoteMediaWindowCache, stub: ReturnType<typeof createStub>) => Promise<void>,
|
||||
): Promise<void> {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-media-window-test-'));
|
||||
const stub = createStub(stubOptions);
|
||||
const cache = new RemoteMediaWindowCache({
|
||||
tempDir,
|
||||
execFile: stub.execFile,
|
||||
idleTtlMs: 0,
|
||||
logDebug: () => undefined,
|
||||
...cacheOptions,
|
||||
});
|
||||
try {
|
||||
await run(cache, stub);
|
||||
} finally {
|
||||
cache.cleanup();
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function argValue(args: string[], flag: string): string | undefined {
|
||||
const index = args.indexOf(flag);
|
||||
return index === -1 ? undefined : args[index + 1];
|
||||
}
|
||||
|
||||
test('buildRemoteMediaWindowArgs stream-copies the window with source timestamps intact', () => {
|
||||
const args = buildRemoteMediaWindowArgs(
|
||||
{ ...SOURCE, inputOptions: { reconnect: true, headers: { Referer: 'https://a.example/' } } },
|
||||
{ startTime: 22.75, endTime: 33 },
|
||||
'/tmp/window.mkv',
|
||||
);
|
||||
|
||||
const inputIndex = args.indexOf('-i');
|
||||
assert.equal(args[inputIndex + 1], SOURCE.path);
|
||||
assert.ok(args.indexOf('-reconnect') < inputIndex);
|
||||
assert.ok(args.indexOf('-headers') < inputIndex);
|
||||
assert.equal(argValue(args, '-ss'), '22.75');
|
||||
assert.equal(argValue(args, '-t'), '10.25');
|
||||
assert.ok(args.indexOf('-t') < inputIndex);
|
||||
assert.deepEqual(args.slice(args.indexOf('-map'), args.indexOf('-map') + 4), [
|
||||
'-map',
|
||||
'0:v:0?',
|
||||
'-map',
|
||||
'0:2',
|
||||
]);
|
||||
assert.equal(argValue(args, '-c'), 'copy');
|
||||
assert.ok(args.includes('-copyts'));
|
||||
assert.ok(args.includes('-start_at_zero'));
|
||||
assert.equal(argValue(args, '-f'), 'matroska');
|
||||
assert.equal(args.at(-1), '/tmp/window.mkv');
|
||||
});
|
||||
|
||||
test('buildRemoteMediaWindowArgs keeps every audio stream when none is selected', () => {
|
||||
const args = buildRemoteMediaWindowArgs(
|
||||
{ path: SOURCE.path, audioStreamIndex: null },
|
||||
{ startTime: 0, endTime: 5 },
|
||||
'/tmp/window.mkv',
|
||||
);
|
||||
|
||||
assert.equal(args[args.lastIndexOf('-map') + 1], '0:a');
|
||||
});
|
||||
|
||||
test('acquire downloads once and reuses the window for covered ranges', async () => {
|
||||
await withCache({}, {}, async (cache, stub) => {
|
||||
const window = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
|
||||
|
||||
assert.equal(stub.calls.length, 1);
|
||||
assert.equal(argValue(stub.calls[0]!, '-ss'), '9.75');
|
||||
assert.equal(argValue(stub.calls[0]!, '-t'), '5.25');
|
||||
assert.equal(window.startTime, 9.75);
|
||||
assert.equal(window.endTime, 15);
|
||||
assert.equal(window.audioStreamIndex, 2);
|
||||
assert.ok(fs.existsSync(window.path));
|
||||
assert.deepEqual(window.media, {
|
||||
path: window.path,
|
||||
source: 'remote-window',
|
||||
singleResolvedStream: true,
|
||||
absoluteTimestamps: true,
|
||||
});
|
||||
|
||||
assert.equal(await cache.acquire(SOURCE, { startTime: 11, endTime: 15 }), window);
|
||||
assert.equal(await cache.lookup(SOURCE, { startTime: 12, endTime: 12 }), window);
|
||||
assert.equal(
|
||||
await cache.lookup(
|
||||
{ path: SOURCE.path, audioStreamIndex: null },
|
||||
{ startTime: 12, endTime: 13 },
|
||||
),
|
||||
window,
|
||||
);
|
||||
assert.equal(stub.calls.length, 1);
|
||||
});
|
||||
});
|
||||
|
||||
test('lookup never downloads and misses on other ranges, sources, or audio streams', async () => {
|
||||
await withCache({}, {}, async (cache, stub) => {
|
||||
assert.equal(await cache.lookup(SOURCE, { startTime: 10, endTime: 14 }), null);
|
||||
assert.equal(stub.calls.length, 0);
|
||||
|
||||
await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
|
||||
assert.equal(await cache.lookup(SOURCE, { startTime: 14, endTime: 16 }), null);
|
||||
assert.equal(
|
||||
await cache.lookup(
|
||||
{ path: 'https://other.example/stream', audioStreamIndex: 2 },
|
||||
{
|
||||
startTime: 11,
|
||||
endTime: 12,
|
||||
},
|
||||
),
|
||||
null,
|
||||
);
|
||||
assert.equal(
|
||||
await cache.lookup(
|
||||
{ path: SOURCE.path, audioStreamIndex: 3 },
|
||||
{ startTime: 11, endTime: 12 },
|
||||
),
|
||||
null,
|
||||
);
|
||||
assert.equal(stub.calls.length, 1);
|
||||
});
|
||||
});
|
||||
|
||||
test('acquire widens to the union of the old window and replaces the old file', async () => {
|
||||
await withCache({}, {}, async (cache, stub) => {
|
||||
const first = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
|
||||
const second = await cache.acquire(SOURCE, { startTime: 8, endTime: 12 });
|
||||
|
||||
assert.equal(stub.calls.length, 2);
|
||||
assert.equal(argValue(stub.calls[1]!, '-ss'), '7.75');
|
||||
assert.equal(second.startTime, 7.75);
|
||||
assert.equal(second.endTime, 15);
|
||||
assert.notEqual(second.path, first.path);
|
||||
assert.equal(fs.existsSync(first.path), false);
|
||||
assert.ok(fs.existsSync(second.path));
|
||||
assert.equal(cache.currentWindow, second);
|
||||
});
|
||||
});
|
||||
|
||||
test('acquire shares an in-flight download between concurrent callers', async () => {
|
||||
await withCache({ defer: true }, {}, async (cache, stub) => {
|
||||
const first = cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
|
||||
await Promise.resolve();
|
||||
const second = cache.acquire(SOURCE, { startTime: 11, endTime: 13 });
|
||||
const lookup = cache.lookup(SOURCE, { startTime: 12, endTime: 12 });
|
||||
await Promise.resolve();
|
||||
assert.equal(stub.calls.length, 1);
|
||||
|
||||
stub.flush();
|
||||
const [a, b, c] = await Promise.all([first, second, lookup]);
|
||||
assert.equal(a, b);
|
||||
assert.equal(a, c);
|
||||
assert.equal(stub.calls.length, 1);
|
||||
});
|
||||
});
|
||||
|
||||
test('acquire rejects on ffmpeg failure, leaves no file, and can retry', async () => {
|
||||
await withCache({ fail: true }, {}, async (cache, stub) => {
|
||||
await assert.rejects(
|
||||
cache.acquire(SOURCE, { startTime: 10, endTime: 14 }),
|
||||
/FFmpeg media window failed: boom/,
|
||||
);
|
||||
assert.equal(cache.currentWindow, null);
|
||||
assert.equal(await cache.lookup(SOURCE, { startTime: 10, endTime: 14 }), null);
|
||||
|
||||
await assert.rejects(cache.acquire(SOURCE, { startTime: 10, endTime: 14 }));
|
||||
assert.equal(stub.calls.length, 2);
|
||||
});
|
||||
await withCache({ empty: true }, {}, async (cache) => {
|
||||
await assert.rejects(
|
||||
cache.acquire(SOURCE, { startTime: 10, endTime: 14 }),
|
||||
/exited without creating a media window/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('acquire refuses invalid and oversized ranges without spawning ffmpeg', async () => {
|
||||
await withCache({}, {}, async (cache, stub) => {
|
||||
await assert.rejects(cache.acquire(SOURCE, { startTime: 10, endTime: 10 }), /invalid/);
|
||||
await assert.rejects(cache.acquire(SOURCE, { startTime: -1, endTime: 10 }), /invalid/);
|
||||
await assert.rejects(
|
||||
cache.acquire(SOURCE, { startTime: 0, endTime: REMOTE_MEDIA_WINDOW_MAX_SECONDS + 1 }),
|
||||
/too long/,
|
||||
);
|
||||
assert.equal(stub.calls.length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
test('the window is deleted after the idle timeout and on cleanup', async () => {
|
||||
await withCache({}, { idleTtlMs: 20 }, async (cache) => {
|
||||
const window = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
|
||||
await new Promise((resolve) => setTimeout(resolve, 60));
|
||||
|
||||
assert.equal(cache.currentWindow, null);
|
||||
assert.equal(fs.existsSync(window.path), false);
|
||||
|
||||
const again = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
|
||||
cache.cleanup();
|
||||
assert.equal(fs.existsSync(again.path), false);
|
||||
assert.equal(fs.existsSync(path.dirname(again.path)), false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,377 @@
|
||||
import { execFile as nodeExecFile, type ExecFileException } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { createLogger } from '../../logger';
|
||||
import { normalizeMediaInput, type MediaInput, type MediaInputOptions } from '../../media-input';
|
||||
|
||||
const log = createLogger('media-window');
|
||||
|
||||
export const REMOTE_MEDIA_WINDOW_TIMEOUT_MS = 120_000;
|
||||
export const REMOTE_MEDIA_WINDOW_MAX_SECONDS = 180;
|
||||
const HEAD_SLACK_SECONDS = 0.25;
|
||||
const TAIL_SLACK_SECONDS = 1;
|
||||
const DEFAULT_IDLE_TTL_MS = 10 * 60_000;
|
||||
const COVERAGE_EPSILON_SECONDS = 0.01;
|
||||
|
||||
export interface RemoteMediaWindowSource {
|
||||
path: string;
|
||||
inputOptions?: MediaInputOptions;
|
||||
/** FFmpeg stream index to keep; `null`/undefined keeps every audio stream. */
|
||||
audioStreamIndex?: number | null;
|
||||
}
|
||||
|
||||
export interface RemoteMediaWindowRange {
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
}
|
||||
|
||||
export interface RemoteMediaWindow {
|
||||
path: string;
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
sourcePath: string;
|
||||
audioStreamIndex: number | null;
|
||||
/** Input descriptor for FFmpeg reads; timestamps stay absolute so callers keep source times. */
|
||||
media: MediaInput;
|
||||
}
|
||||
|
||||
type WindowExecFile = (
|
||||
file: string,
|
||||
args: readonly string[],
|
||||
options: { timeout: number },
|
||||
callback: (error: ExecFileException | null) => void,
|
||||
) => void;
|
||||
|
||||
export interface RemoteMediaWindowCacheOptions {
|
||||
tempDir?: string;
|
||||
execFile?: WindowExecFile;
|
||||
idleTtlMs?: number;
|
||||
logDebug?: (message: string) => void;
|
||||
}
|
||||
|
||||
interface PendingFetch extends RemoteMediaWindowRange {
|
||||
sourcePath: string;
|
||||
audioStreamIndex: number | null;
|
||||
promise: Promise<RemoteMediaWindow>;
|
||||
}
|
||||
|
||||
export function isRemoteMediaWindowSourcePath(value: string): boolean {
|
||||
return /^https?:\/\//i.test(value.trim());
|
||||
}
|
||||
|
||||
function describeSourceForDebugLog(sourcePath: string): string {
|
||||
try {
|
||||
return `remote:${new URL(sourcePath).hostname.toLowerCase() || 'unknown'}`;
|
||||
} catch {
|
||||
return 'remote:unknown';
|
||||
}
|
||||
}
|
||||
|
||||
function isUsableRange(range: RemoteMediaWindowRange, allowEmpty: boolean): boolean {
|
||||
return (
|
||||
Number.isFinite(range.startTime) &&
|
||||
Number.isFinite(range.endTime) &&
|
||||
range.startTime >= 0 &&
|
||||
(allowEmpty ? range.endTime >= range.startTime : range.endTime > range.startTime)
|
||||
);
|
||||
}
|
||||
|
||||
function audioStreamMatches(
|
||||
windowIndex: number | null,
|
||||
requested: number | null | undefined,
|
||||
): boolean {
|
||||
return requested == null || windowIndex === requested;
|
||||
}
|
||||
|
||||
function covers(
|
||||
candidate: RemoteMediaWindowRange & { sourcePath: string; audioStreamIndex: number | null },
|
||||
source: RemoteMediaWindowSource,
|
||||
range: RemoteMediaWindowRange,
|
||||
): boolean {
|
||||
return (
|
||||
candidate.sourcePath === source.path &&
|
||||
audioStreamMatches(candidate.audioStreamIndex, source.audioStreamIndex) &&
|
||||
candidate.startTime <= range.startTime + COVERAGE_EPSILON_SECONDS &&
|
||||
candidate.endTime >= range.endTime - COVERAGE_EPSILON_SECONDS
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream-copies `[startTime, endTime]` of a remote source into a local Matroska file.
|
||||
* `-copyts -start_at_zero` keeps the source timestamps, so later reads seek with the
|
||||
* original times via `-seek_timestamp 1` (see `MediaInput.absoluteTimestamps`).
|
||||
*/
|
||||
export function buildRemoteMediaWindowArgs(
|
||||
source: RemoteMediaWindowSource,
|
||||
range: RemoteMediaWindowRange,
|
||||
outputPath: string,
|
||||
): string[] {
|
||||
const input = normalizeMediaInput({ path: source.path, inputOptions: source.inputOptions });
|
||||
const audioMap =
|
||||
typeof source.audioStreamIndex === 'number' && Number.isInteger(source.audioStreamIndex)
|
||||
? `0:${source.audioStreamIndex}`
|
||||
: '0:a';
|
||||
return [
|
||||
'-hide_banner',
|
||||
'-nostdin',
|
||||
'-loglevel',
|
||||
'error',
|
||||
'-ss',
|
||||
String(range.startTime),
|
||||
'-t',
|
||||
String(range.endTime - range.startTime),
|
||||
...input.inputArgs,
|
||||
'-i',
|
||||
input.path,
|
||||
'-map',
|
||||
'0:v:0?',
|
||||
'-map',
|
||||
audioMap,
|
||||
'-c',
|
||||
'copy',
|
||||
'-sn',
|
||||
'-dn',
|
||||
'-copyts',
|
||||
'-start_at_zero',
|
||||
'-f',
|
||||
'matroska',
|
||||
'-y',
|
||||
outputPath,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Holds one downloaded window of the current remote stream so the timing review,
|
||||
* audio extraction, and screenshot all read the same local bytes instead of each
|
||||
* re-fetching the clip over HTTP. A new window replaces the old one; the file is
|
||||
* deleted after `idleTtlMs` without use, on `clear()`, or on `cleanup()`.
|
||||
*/
|
||||
export class RemoteMediaWindowCache {
|
||||
private readonly tempDir: string;
|
||||
private readonly execFile: WindowExecFile;
|
||||
private readonly idleTtlMs: number;
|
||||
private readonly logDebug: (message: string) => void;
|
||||
private current: RemoteMediaWindow | null = null;
|
||||
private pending: PendingFetch | null = null;
|
||||
private idleTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private sequence = 0;
|
||||
|
||||
constructor(options: RemoteMediaWindowCacheOptions = {}) {
|
||||
this.tempDir = options.tempDir ?? path.join(os.tmpdir(), 'subminer-media-windows');
|
||||
this.execFile = options.execFile ?? nodeExecFile;
|
||||
this.idleTtlMs = options.idleTtlMs ?? DEFAULT_IDLE_TTL_MS;
|
||||
this.logDebug = options.logDebug ?? ((message) => log.debug(message));
|
||||
}
|
||||
|
||||
get currentWindow(): RemoteMediaWindow | null {
|
||||
return this.current;
|
||||
}
|
||||
|
||||
/** Returns a ready or in-flight window covering the range; never starts a download. */
|
||||
async lookup(
|
||||
source: RemoteMediaWindowSource,
|
||||
range: RemoteMediaWindowRange,
|
||||
): Promise<RemoteMediaWindow | null> {
|
||||
if (!isUsableRange(range, true)) return null;
|
||||
if (this.current && covers(this.current, source, range)) {
|
||||
this.touch();
|
||||
return this.current;
|
||||
}
|
||||
const pending = this.pending;
|
||||
if (pending && covers(pending, source, range)) {
|
||||
try {
|
||||
const window = await pending.promise;
|
||||
this.touch();
|
||||
return window;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Returns a window covering the range, downloading (and widening) one when needed. */
|
||||
async acquire(
|
||||
source: RemoteMediaWindowSource,
|
||||
range: RemoteMediaWindowRange,
|
||||
): Promise<RemoteMediaWindow> {
|
||||
if (!isUsableRange(range, false)) {
|
||||
throw new Error('Media window range is invalid.');
|
||||
}
|
||||
if (range.endTime - range.startTime > REMOTE_MEDIA_WINDOW_MAX_SECONDS) {
|
||||
throw new Error('Media window range is too long to download.');
|
||||
}
|
||||
|
||||
for (;;) {
|
||||
const hit = await this.lookup(source, range);
|
||||
if (hit) return hit;
|
||||
const pending = this.pending;
|
||||
if (!pending) break;
|
||||
// Another caller is already downloading; wait for it, then re-check coverage.
|
||||
await pending.promise.catch(() => null);
|
||||
}
|
||||
|
||||
return this.fetch(source, this.planFetchRange(source, range));
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.cancelIdleTimer();
|
||||
const current = this.current;
|
||||
this.current = null;
|
||||
if (current) this.removeFile(current.path);
|
||||
}
|
||||
|
||||
cleanup(): void {
|
||||
this.clear();
|
||||
try {
|
||||
fs.rmSync(this.tempDir, { recursive: true, force: true });
|
||||
} catch (error) {
|
||||
log.error('Failed to cleanup media window directory:', error);
|
||||
}
|
||||
}
|
||||
|
||||
private planFetchRange(
|
||||
source: RemoteMediaWindowSource,
|
||||
range: RemoteMediaWindowRange,
|
||||
): RemoteMediaWindowRange {
|
||||
let startTime = Math.max(0, range.startTime - HEAD_SLACK_SECONDS);
|
||||
let endTime = range.endTime + TAIL_SLACK_SECONDS;
|
||||
const current = this.current;
|
||||
if (
|
||||
current &&
|
||||
current.sourcePath === source.path &&
|
||||
audioStreamMatches(current.audioStreamIndex, source.audioStreamIndex)
|
||||
) {
|
||||
// Keep what was already downloaded when the review timeline grows in one direction.
|
||||
const unionStart = Math.min(startTime, current.startTime);
|
||||
const unionEnd = Math.max(endTime, current.endTime);
|
||||
if (unionEnd - unionStart <= REMOTE_MEDIA_WINDOW_MAX_SECONDS) {
|
||||
startTime = unionStart;
|
||||
endTime = unionEnd;
|
||||
}
|
||||
}
|
||||
return { startTime, endTime };
|
||||
}
|
||||
|
||||
private fetch(
|
||||
source: RemoteMediaWindowSource,
|
||||
range: RemoteMediaWindowRange,
|
||||
): Promise<RemoteMediaWindow> {
|
||||
fs.mkdirSync(this.tempDir, { recursive: true });
|
||||
this.sequence += 1;
|
||||
const outputPath = path.join(this.tempDir, `window_${Date.now()}_${this.sequence}.mkv`);
|
||||
const audioStreamIndex =
|
||||
typeof source.audioStreamIndex === 'number' ? source.audioStreamIndex : null;
|
||||
const description = describeSourceForDebugLog(source.path);
|
||||
const startedAt = Date.now();
|
||||
this.logDebug(
|
||||
`[media-window] fetch start ${description} start=${range.startTime} end=${range.endTime} audioStream=${audioStreamIndex ?? 'all'}`,
|
||||
);
|
||||
|
||||
const promise = new Promise<RemoteMediaWindow>((resolve, reject) => {
|
||||
this.execFile(
|
||||
'ffmpeg',
|
||||
buildRemoteMediaWindowArgs(source, range, outputPath),
|
||||
{ timeout: REMOTE_MEDIA_WINDOW_TIMEOUT_MS },
|
||||
(error) => {
|
||||
const elapsedMs = Math.max(0, Date.now() - startedAt);
|
||||
const size = error ? 0 : this.fileSize(outputPath);
|
||||
if (error || size === 0) {
|
||||
this.removeFile(outputPath);
|
||||
const reason = error
|
||||
? error.code === 'ENOENT'
|
||||
? 'FFmpeg not found. Install FFmpeg to enable media generation.'
|
||||
: `FFmpeg media window failed: ${error.message}`
|
||||
: 'FFmpeg exited without creating a media window.';
|
||||
this.logDebug(`[media-window] fetch failed ${description} elapsedMs=${elapsedMs}`);
|
||||
reject(new Error(reason));
|
||||
return;
|
||||
}
|
||||
const window: RemoteMediaWindow = {
|
||||
path: outputPath,
|
||||
startTime: range.startTime,
|
||||
endTime: range.endTime,
|
||||
sourcePath: source.path,
|
||||
audioStreamIndex,
|
||||
media: {
|
||||
path: outputPath,
|
||||
source: 'remote-window',
|
||||
singleResolvedStream: true,
|
||||
absoluteTimestamps: true,
|
||||
},
|
||||
};
|
||||
this.logDebug(
|
||||
`[media-window] fetch complete ${description} elapsedMs=${elapsedMs} bytes=${size}`,
|
||||
);
|
||||
this.replaceCurrent(window);
|
||||
resolve(window);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
const pending: PendingFetch = {
|
||||
sourcePath: source.path,
|
||||
audioStreamIndex,
|
||||
startTime: range.startTime,
|
||||
endTime: range.endTime,
|
||||
promise,
|
||||
};
|
||||
this.pending = pending;
|
||||
promise
|
||||
.catch(() => undefined)
|
||||
.then(() => {
|
||||
if (this.pending === pending) this.pending = null;
|
||||
});
|
||||
return promise;
|
||||
}
|
||||
|
||||
private replaceCurrent(window: RemoteMediaWindow): void {
|
||||
const previous = this.current;
|
||||
this.current = window;
|
||||
if (previous && previous.path !== window.path) this.removeFile(previous.path);
|
||||
this.touch();
|
||||
}
|
||||
|
||||
private touch(): void {
|
||||
this.cancelIdleTimer();
|
||||
if (this.idleTtlMs <= 0 || !this.current) return;
|
||||
const timer = setTimeout(() => {
|
||||
if (this.idleTimer === timer) this.idleTimer = null;
|
||||
this.clear();
|
||||
}, this.idleTtlMs);
|
||||
timer.unref?.();
|
||||
this.idleTimer = timer;
|
||||
}
|
||||
|
||||
private cancelIdleTimer(): void {
|
||||
if (this.idleTimer) clearTimeout(this.idleTimer);
|
||||
this.idleTimer = null;
|
||||
}
|
||||
|
||||
private fileSize(filePath: string): number {
|
||||
try {
|
||||
return fs.statSync(filePath).size;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private removeFile(filePath: string): void {
|
||||
try {
|
||||
fs.unlinkSync(filePath);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
log.debug(`Failed to remove media window ${filePath}:`, (error as Error).message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let sharedCache: RemoteMediaWindowCache | null = null;
|
||||
|
||||
/** Process-wide cache so the review modal and card media generation share one download. */
|
||||
export function getSharedRemoteMediaWindowCache(): RemoteMediaWindowCache {
|
||||
sharedCache ??= new RemoteMediaWindowCache();
|
||||
return sharedCache;
|
||||
}
|
||||
+18
@@ -464,6 +464,8 @@ import { handleCliCommandRuntimeServiceWithContext } from './main/cli-runtime';
|
||||
import { createOverlayModalRuntimeService } from './main/overlay-runtime';
|
||||
import { createOverlayModalInputState } from './main/runtime/overlay-modal-input-state';
|
||||
import { MediaTimingPreviewSession } from './core/services/media-timing-preview';
|
||||
import { getSharedRemoteMediaWindowCache } from './core/services/remote-media-window-cache';
|
||||
import { resolveMediaGenerationInput } from './anki-integration/media-source';
|
||||
import { generateSpeechWaveform } from './core/services/media-timing-waveform';
|
||||
import {
|
||||
collectMediaTimingContextLines,
|
||||
@@ -2896,6 +2898,21 @@ const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({
|
||||
configService.getConfig().mpv.executablePath || process.env.SUBMINER_MPV_PATH?.trim() || '',
|
||||
createPreviewSession: () => new MediaTimingPreviewSession(),
|
||||
generateWaveform: (options) => generateSpeechWaveform(options),
|
||||
resolveMediaSource: async () => {
|
||||
const resolved = await resolveMediaGenerationInput(appState.mpvClient, 'audio', {
|
||||
getCachedMediaPath: (currentVideoPath, kind) =>
|
||||
getCachedYoutubeMediaPathForCurrentPlayback(currentVideoPath, kind),
|
||||
remoteCacheMode: shouldRequireYoutubeMediaCacheForCurrentPlayback() ? 'required' : 'optional',
|
||||
});
|
||||
return resolved
|
||||
? {
|
||||
path: resolved.path,
|
||||
...(resolved.inputOptions ? { inputOptions: resolved.inputOptions } : {}),
|
||||
singleResolvedStream: resolved.singleResolvedStream,
|
||||
}
|
||||
: null;
|
||||
},
|
||||
acquireMediaWindow: (source, range) => getSharedRemoteMediaWindowCache().acquire(source, range),
|
||||
getSubtitleContextLines: (range) =>
|
||||
collectMediaTimingContextLines({
|
||||
cues: appState.activeParsedSubtitleCues,
|
||||
@@ -3973,6 +3990,7 @@ const {
|
||||
cleanupInternalSubtitleTrackCache: () => cachedInternalSubtitleTrackExtractor.clear(),
|
||||
cleanupYoutubeSubtitleTempDirs: () => youtubeFlowRuntime.cleanupSubtitleTempDirs(),
|
||||
cleanupYoutubeMediaCache: () => youtubeMediaCache.cleanup(),
|
||||
cleanupRemoteMediaWindows: () => getSharedRemoteMediaWindowCache().cleanup(),
|
||||
cleanupJellyfinSubtitleCache: () => cleanupJellyfinSubtitleCache(),
|
||||
stopDiscordPresenceService: () => {
|
||||
void appState.discordPresenceService?.stop();
|
||||
|
||||
@@ -46,12 +46,13 @@ test('on will quit cleanup handler runs all cleanup steps', () => {
|
||||
cleanupInternalSubtitleTrackCache: () => calls.push('cleanup-internal-subtitles'),
|
||||
cleanupYoutubeSubtitleTempDirs: () => calls.push('cleanup-youtube-subtitles'),
|
||||
cleanupYoutubeMediaCache: () => calls.push('cleanup-youtube-media'),
|
||||
cleanupRemoteMediaWindows: () => calls.push('cleanup-remote-media-windows'),
|
||||
cleanupJellyfinSubtitleCache: () => calls.push('cleanup-jellyfin-subtitles'),
|
||||
stopDiscordPresenceService: () => calls.push('stop-discord-presence'),
|
||||
});
|
||||
|
||||
cleanup();
|
||||
assert.equal(calls.length, 35);
|
||||
assert.equal(calls.length, 36);
|
||||
assert.equal(calls[0], 'destroy-tray');
|
||||
assert.equal(calls[calls.length - 1], 'stop-discord-presence');
|
||||
assert.ok(calls.includes('cleanup-jellyfin-subtitles'));
|
||||
@@ -60,6 +61,7 @@ test('on will quit cleanup handler runs all cleanup steps', () => {
|
||||
assert.ok(calls.includes('clear-linux-mpv-fullscreen-overlay-refresh-timeouts'));
|
||||
assert.ok(calls.includes('cleanup-youtube-subtitles'));
|
||||
assert.ok(calls.includes('cleanup-youtube-media'));
|
||||
assert.ok(calls.includes('cleanup-remote-media-windows'));
|
||||
assert.ok(calls.indexOf('flush-mpv-log') < calls.indexOf('destroy-socket'));
|
||||
});
|
||||
|
||||
@@ -102,6 +104,7 @@ test('on will quit cleanup handler cleans jellyfin subtitle cache when stopping
|
||||
cleanupInternalSubtitleTrackCache: () => calls.push('cleanup-internal-subtitles'),
|
||||
cleanupYoutubeSubtitleTempDirs: () => calls.push('cleanup-youtube-subtitles'),
|
||||
cleanupYoutubeMediaCache: () => calls.push('cleanup-youtube-media'),
|
||||
cleanupRemoteMediaWindows: () => calls.push('cleanup-remote-media-windows'),
|
||||
cleanupJellyfinSubtitleCache: () => calls.push('cleanup-jellyfin-subtitles'),
|
||||
stopDiscordPresenceService: () => calls.push('stop-discord-presence'),
|
||||
});
|
||||
|
||||
@@ -32,6 +32,7 @@ export function createOnWillQuitCleanupHandler(deps: {
|
||||
cleanupInternalSubtitleTrackCache: () => void;
|
||||
cleanupYoutubeSubtitleTempDirs: () => void;
|
||||
cleanupYoutubeMediaCache: () => void;
|
||||
cleanupRemoteMediaWindows: () => void;
|
||||
cleanupJellyfinSubtitleCache: () => void;
|
||||
stopDiscordPresenceService: () => void;
|
||||
}) {
|
||||
@@ -76,6 +77,7 @@ export function createOnWillQuitCleanupHandler(deps: {
|
||||
}
|
||||
deps.cleanupYoutubeSubtitleTempDirs();
|
||||
deps.cleanupYoutubeMediaCache();
|
||||
deps.cleanupRemoteMediaWindows();
|
||||
deps.stopDiscordPresenceService();
|
||||
return Promise.resolve(stopSyncAutoScheduler);
|
||||
};
|
||||
|
||||
@@ -75,6 +75,7 @@ test('cleanup deps builder returns handlers that guard optional runtime objects'
|
||||
cleanupInternalSubtitleTrackCache: () => calls.push('cleanup-internal-subtitles'),
|
||||
cleanupYoutubeSubtitleTempDirs: () => calls.push('cleanup-youtube-subtitles'),
|
||||
cleanupYoutubeMediaCache: () => calls.push('cleanup-youtube-media'),
|
||||
cleanupRemoteMediaWindows: () => calls.push('cleanup-remote-media-windows'),
|
||||
cleanupJellyfinSubtitleCache: () => calls.push('cleanup-jellyfin-subtitles'),
|
||||
stopDiscordPresenceService: () => calls.push('stop-discord-presence'),
|
||||
});
|
||||
@@ -157,6 +158,7 @@ test('cleanup deps builder skips destroyed yomitan window', () => {
|
||||
cleanupInternalSubtitleTrackCache: () => {},
|
||||
cleanupYoutubeSubtitleTempDirs: () => {},
|
||||
cleanupYoutubeMediaCache: () => {},
|
||||
cleanupRemoteMediaWindows: () => {},
|
||||
cleanupJellyfinSubtitleCache: () => {},
|
||||
stopDiscordPresenceService: () => {},
|
||||
});
|
||||
@@ -210,6 +212,7 @@ test('cleanup deps builder skips global shortcut cleanup before app ready', () =
|
||||
cleanupInternalSubtitleTrackCache: () => {},
|
||||
cleanupYoutubeSubtitleTempDirs: () => {},
|
||||
cleanupYoutubeMediaCache: () => {},
|
||||
cleanupRemoteMediaWindows: () => {},
|
||||
cleanupJellyfinSubtitleCache: () => {},
|
||||
stopDiscordPresenceService: () => {},
|
||||
});
|
||||
|
||||
@@ -61,6 +61,7 @@ export function createBuildOnWillQuitCleanupDepsHandler(deps: {
|
||||
cleanupInternalSubtitleTrackCache: () => void;
|
||||
cleanupYoutubeSubtitleTempDirs: () => void;
|
||||
cleanupYoutubeMediaCache: () => void;
|
||||
cleanupRemoteMediaWindows: () => void;
|
||||
cleanupJellyfinSubtitleCache: () => void;
|
||||
stopDiscordPresenceService: () => void;
|
||||
}) {
|
||||
@@ -148,6 +149,7 @@ export function createBuildOnWillQuitCleanupDepsHandler(deps: {
|
||||
cleanupInternalSubtitleTrackCache: () => deps.cleanupInternalSubtitleTrackCache(),
|
||||
cleanupYoutubeSubtitleTempDirs: () => deps.cleanupYoutubeSubtitleTempDirs(),
|
||||
cleanupYoutubeMediaCache: () => deps.cleanupYoutubeMediaCache(),
|
||||
cleanupRemoteMediaWindows: () => deps.cleanupRemoteMediaWindows(),
|
||||
cleanupJellyfinSubtitleCache: () => deps.cleanupJellyfinSubtitleCache(),
|
||||
stopDiscordPresenceService: () => deps.stopDiscordPresenceService(),
|
||||
});
|
||||
|
||||
@@ -52,6 +52,7 @@ test('composeStartupLifecycleHandlers returns callable startup lifecycle handler
|
||||
cleanupInternalSubtitleTrackCache: () => {},
|
||||
cleanupYoutubeSubtitleTempDirs: () => {},
|
||||
cleanupYoutubeMediaCache: () => {},
|
||||
cleanupRemoteMediaWindows: () => {},
|
||||
cleanupJellyfinSubtitleCache: () => {},
|
||||
stopDiscordPresenceService: () => {},
|
||||
},
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, test } from 'node:test';
|
||||
import type { MediaTimingReviewOpenPayload } from '../../types/anki';
|
||||
import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform';
|
||||
import type {
|
||||
RemoteMediaWindow,
|
||||
RemoteMediaWindowRange,
|
||||
RemoteMediaWindowSource,
|
||||
} from '../../core/services/remote-media-window-cache';
|
||||
import type { MediaTimingPreviewSession } from '../../core/services/media-timing-preview';
|
||||
|
||||
type MediaTimingPreviewSessionLike = Pick<MediaTimingPreviewSession, 'start'>;
|
||||
import {
|
||||
buildMediaTimingReviewPayload,
|
||||
collectMediaTimingContextLines,
|
||||
@@ -177,12 +186,7 @@ test('media timing review pauses playback, resolves exact timing, and restores p
|
||||
});
|
||||
|
||||
test('media timing review analyzes the visible range on the selected audio stream', async () => {
|
||||
const waveformCalls: Array<{
|
||||
mediaPath: string;
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
audioStreamIndex?: number;
|
||||
}> = [];
|
||||
const waveformCalls: SpeechWaveformOptions[] = [];
|
||||
let runtime: ReturnType<typeof createMediaTimingReviewRuntime>;
|
||||
runtime = createMediaTimingReviewRuntime({
|
||||
getMpvClient: () => ({
|
||||
@@ -239,6 +243,300 @@ test('media timing review analyzes the visible range on the selected audio strea
|
||||
]);
|
||||
});
|
||||
|
||||
const REMOTE_STREAM_URL = 'https://jellyfin.example/Videos/abc/stream?static=true';
|
||||
|
||||
function createWindowStub(options: { fail?: boolean } = {}) {
|
||||
const calls: Array<{ source: RemoteMediaWindowSource; range: RemoteMediaWindowRange }> = [];
|
||||
const acquireMediaWindow = async (
|
||||
source: RemoteMediaWindowSource,
|
||||
range: RemoteMediaWindowRange,
|
||||
): Promise<RemoteMediaWindow> => {
|
||||
calls.push({ source, range });
|
||||
if (options.fail) throw new Error('offline');
|
||||
const windowPath = `/tmp/window-${range.startTime}-${range.endTime}.mkv`;
|
||||
return {
|
||||
path: windowPath,
|
||||
startTime: range.startTime,
|
||||
endTime: range.endTime,
|
||||
sourcePath: source.path,
|
||||
audioStreamIndex: source.audioStreamIndex ?? null,
|
||||
media: {
|
||||
path: windowPath,
|
||||
source: 'remote-window',
|
||||
singleResolvedStream: true,
|
||||
absoluteTimestamps: true,
|
||||
},
|
||||
};
|
||||
};
|
||||
return { calls, acquireMediaWindow };
|
||||
}
|
||||
|
||||
function createRemoteReviewRuntime(options: {
|
||||
windowStub: ReturnType<typeof createWindowStub>;
|
||||
waveformCalls: SpeechWaveformOptions[];
|
||||
previewStarts: Array<Parameters<MediaTimingPreviewSessionLike['start']>[0]>;
|
||||
previewPlays: Array<[string, number, number]>;
|
||||
disposed: string[];
|
||||
openModal: (
|
||||
runtime: ReturnType<typeof createMediaTimingReviewRuntime>,
|
||||
payload: MediaTimingReviewOpenPayload,
|
||||
) => Promise<void>;
|
||||
}) {
|
||||
let runtime!: ReturnType<typeof createMediaTimingReviewRuntime>;
|
||||
runtime = createMediaTimingReviewRuntime({
|
||||
getMpvClient: () => ({
|
||||
connected: true,
|
||||
currentVideoPath: REMOTE_STREAM_URL,
|
||||
currentAudioStreamIndex: 2,
|
||||
requestProperty: async (name) =>
|
||||
({ pause: true, duration: 100, aid: 3, volume: 60 })[
|
||||
name as 'pause' | 'duration' | 'aid' | 'volume'
|
||||
] ?? null,
|
||||
send: () => undefined,
|
||||
}),
|
||||
getCurrentMediaPath: () => REMOTE_STREAM_URL,
|
||||
getMpvExecutablePath: () => 'mpv',
|
||||
resolveMediaSource: async () => ({
|
||||
path: REMOTE_STREAM_URL,
|
||||
inputOptions: { reconnect: true },
|
||||
}),
|
||||
acquireMediaWindow: options.windowStub.acquireMediaWindow,
|
||||
generateWaveform: async (waveformOptions) => {
|
||||
options.waveformCalls.push(waveformOptions);
|
||||
return [0.1, 0.8, 0.2];
|
||||
},
|
||||
createPreviewSession: () => {
|
||||
let mediaPath = '';
|
||||
return {
|
||||
start: async (startOptions) => {
|
||||
mediaPath = startOptions.mediaPath;
|
||||
options.previewStarts.push(startOptions);
|
||||
},
|
||||
play: async (startTime, endTime) => {
|
||||
options.previewPlays.push([mediaPath, startTime, endTime]);
|
||||
},
|
||||
stop: async () => undefined,
|
||||
dispose: () => {
|
||||
options.disposed.push(mediaPath);
|
||||
},
|
||||
};
|
||||
},
|
||||
openModal: async (payload) => {
|
||||
await options.openModal(runtime, payload);
|
||||
return true;
|
||||
},
|
||||
showStatus: () => undefined,
|
||||
});
|
||||
return runtime;
|
||||
}
|
||||
|
||||
test('media timing review downloads one window of a remote stream for the waveform and preview', async () => {
|
||||
const windowStub = createWindowStub();
|
||||
const waveformCalls: SpeechWaveformOptions[] = [];
|
||||
const previewStarts: Array<Parameters<MediaTimingPreviewSessionLike['start']>[0]> = [];
|
||||
const previewPlays: Array<[string, number, number]> = [];
|
||||
const disposed: string[] = [];
|
||||
const runtime = createRemoteReviewRuntime({
|
||||
windowStub,
|
||||
waveformCalls,
|
||||
previewStarts,
|
||||
previewPlays,
|
||||
disposed,
|
||||
openModal: async (active, payload) => {
|
||||
const waveform = await active.getWaveform({
|
||||
reviewId: payload.reviewId,
|
||||
startTime: payload.timelineStartTime,
|
||||
endTime: payload.timelineEndTime,
|
||||
});
|
||||
assert.deepEqual(waveform, { ok: true, peaks: [0.1, 0.8, 0.2] });
|
||||
assert.deepEqual(
|
||||
await active.previewRange({ reviewId: payload.reviewId, startTime: 9.5, endTime: 12.5 }),
|
||||
{ ok: true },
|
||||
);
|
||||
active.resolveReview({
|
||||
reviewId: payload.reviewId,
|
||||
decision: { action: 'confirm', startTime: 9.5, endTime: 12.5 },
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const decision = await runtime.requestReview({
|
||||
kind: 'word',
|
||||
text: '字幕',
|
||||
startTime: 10,
|
||||
endTime: 12,
|
||||
audioPadding: 0.5,
|
||||
maxMediaDuration: 30,
|
||||
});
|
||||
|
||||
assert.deepEqual(decision, { action: 'confirm', startTime: 9.5, endTime: 12.5 });
|
||||
assert.deepEqual(windowStub.calls, [
|
||||
{
|
||||
source: { path: REMOTE_STREAM_URL, inputOptions: { reconnect: true }, audioStreamIndex: 2 },
|
||||
range: { startTime: 7.5, endTime: 14.5 },
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(waveformCalls, [
|
||||
{
|
||||
mediaPath: {
|
||||
path: '/tmp/window-7.5-14.5.mkv',
|
||||
source: 'remote-window',
|
||||
singleResolvedStream: true,
|
||||
absoluteTimestamps: true,
|
||||
},
|
||||
startTime: 7.5,
|
||||
endTime: 14.5,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(previewStarts, [
|
||||
{
|
||||
mediaPath: '/tmp/window-7.5-14.5.mkv',
|
||||
executablePath: 'mpv',
|
||||
volume: 60,
|
||||
absoluteTimestamps: true,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(previewPlays, [['/tmp/window-7.5-14.5.mkv', 9.5, 12.5]]);
|
||||
assert.deepEqual(disposed, ['/tmp/window-7.5-14.5.mkv']);
|
||||
});
|
||||
|
||||
test('media timing review restarts the preview on a wider window when the timeline grows', async () => {
|
||||
const windowStub = createWindowStub();
|
||||
const waveformCalls: SpeechWaveformOptions[] = [];
|
||||
const previewStarts: Array<Parameters<MediaTimingPreviewSessionLike['start']>[0]> = [];
|
||||
const previewPlays: Array<[string, number, number]> = [];
|
||||
const disposed: string[] = [];
|
||||
const runtime = createRemoteReviewRuntime({
|
||||
windowStub,
|
||||
waveformCalls,
|
||||
previewStarts,
|
||||
previewPlays,
|
||||
disposed,
|
||||
openModal: async (active, payload) => {
|
||||
await active.previewRange({ reviewId: payload.reviewId, startTime: 9.5, endTime: 12.5 });
|
||||
// The user revealed two more seconds before the clip.
|
||||
await active.getWaveform({ reviewId: payload.reviewId, startTime: 5.5, endTime: 14.5 });
|
||||
await active.previewRange({ reviewId: payload.reviewId, startTime: 6, endTime: 12.5 });
|
||||
active.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } });
|
||||
},
|
||||
});
|
||||
|
||||
await runtime.requestReview({
|
||||
kind: 'sentence',
|
||||
text: '字幕',
|
||||
startTime: 10,
|
||||
endTime: 12,
|
||||
audioPadding: 0.5,
|
||||
maxMediaDuration: 30,
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
windowStub.calls.map((call) => call.range),
|
||||
[
|
||||
{ startTime: 7.5, endTime: 14.5 },
|
||||
{ startTime: 5.5, endTime: 14.5 },
|
||||
],
|
||||
);
|
||||
assert.deepEqual(
|
||||
previewStarts.map((start) => start.mediaPath),
|
||||
['/tmp/window-7.5-14.5.mkv', '/tmp/window-5.5-14.5.mkv'],
|
||||
);
|
||||
assert.deepEqual(previewPlays, [
|
||||
['/tmp/window-7.5-14.5.mkv', 9.5, 12.5],
|
||||
['/tmp/window-5.5-14.5.mkv', 6, 12.5],
|
||||
]);
|
||||
assert.deepEqual(disposed, ['/tmp/window-7.5-14.5.mkv', '/tmp/window-5.5-14.5.mkv']);
|
||||
assert.equal(waveformCalls[0]?.startTime, 5.5);
|
||||
});
|
||||
|
||||
test('media timing review falls back to the remote stream after one failed window download', async () => {
|
||||
const windowStub = createWindowStub({ fail: true });
|
||||
const waveformCalls: SpeechWaveformOptions[] = [];
|
||||
const previewStarts: Array<Parameters<MediaTimingPreviewSessionLike['start']>[0]> = [];
|
||||
const previewPlays: Array<[string, number, number]> = [];
|
||||
const disposed: string[] = [];
|
||||
const runtime = createRemoteReviewRuntime({
|
||||
windowStub,
|
||||
waveformCalls,
|
||||
previewStarts,
|
||||
previewPlays,
|
||||
disposed,
|
||||
openModal: async (active, payload) => {
|
||||
await active.getWaveform({
|
||||
reviewId: payload.reviewId,
|
||||
startTime: payload.timelineStartTime,
|
||||
endTime: payload.timelineEndTime,
|
||||
});
|
||||
await active.previewRange({ reviewId: payload.reviewId, startTime: 9.5, endTime: 12.5 });
|
||||
active.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } });
|
||||
},
|
||||
});
|
||||
|
||||
await runtime.requestReview({
|
||||
kind: 'word',
|
||||
text: '字幕',
|
||||
startTime: 10,
|
||||
endTime: 12,
|
||||
audioPadding: 0.5,
|
||||
maxMediaDuration: 30,
|
||||
});
|
||||
|
||||
assert.equal(windowStub.calls.length, 1);
|
||||
assert.deepEqual(waveformCalls, [
|
||||
{
|
||||
mediaPath: { path: REMOTE_STREAM_URL, inputOptions: { reconnect: true } },
|
||||
startTime: 7.5,
|
||||
endTime: 14.5,
|
||||
audioStreamIndex: 2,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(previewStarts, [
|
||||
{ mediaPath: REMOTE_STREAM_URL, executablePath: 'mpv', volume: 60, audioTrackId: 3 },
|
||||
]);
|
||||
assert.deepEqual(previewPlays, [[REMOTE_STREAM_URL, 9.5, 12.5]]);
|
||||
});
|
||||
|
||||
test('media timing review never downloads windows for local media', async () => {
|
||||
const windowStub = createWindowStub();
|
||||
let runtime!: ReturnType<typeof createMediaTimingReviewRuntime>;
|
||||
runtime = createMediaTimingReviewRuntime({
|
||||
getMpvClient: () => ({
|
||||
connected: true,
|
||||
currentVideoPath: '/video/show.mkv',
|
||||
requestProperty: async (name) => (name === 'duration' ? 100 : name === 'pause' ? true : null),
|
||||
send: () => undefined,
|
||||
}),
|
||||
getCurrentMediaPath: () => '/video/show.mkv',
|
||||
getMpvExecutablePath: () => 'mpv',
|
||||
resolveMediaSource: async () => ({ path: '/video/show.mkv' }),
|
||||
acquireMediaWindow: windowStub.acquireMediaWindow,
|
||||
generateWaveform: async () => [0.1, 0.8, 0.2],
|
||||
createPreviewSession: () => ({
|
||||
start: async () => undefined,
|
||||
play: async () => undefined,
|
||||
stop: async () => undefined,
|
||||
dispose: () => undefined,
|
||||
}),
|
||||
openModal: async (payload) => {
|
||||
await runtime.getWaveform({ reviewId: payload.reviewId, startTime: 7.5, endTime: 14.5 });
|
||||
runtime.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } });
|
||||
return true;
|
||||
},
|
||||
showStatus: () => undefined,
|
||||
});
|
||||
|
||||
await runtime.requestReview({
|
||||
kind: 'sentence',
|
||||
text: '字幕',
|
||||
startTime: 10,
|
||||
endTime: 12,
|
||||
audioPadding: 0.5,
|
||||
maxMediaDuration: 30,
|
||||
});
|
||||
|
||||
assert.equal(windowStub.calls.length, 0);
|
||||
});
|
||||
|
||||
test('media timing review rejects stale and out-of-range actions before allowing discard', async () => {
|
||||
const { runtime, payload, pendingDecision, previewCalls } = await startActiveMediaTimingReview({
|
||||
maxMediaDuration: 3,
|
||||
@@ -384,11 +682,16 @@ test('media timing review restores playback when setup fails after pausing', asy
|
||||
send: ({ command }) => commands.push(command),
|
||||
}),
|
||||
getCurrentMediaPath: () => '/video/show.mkv',
|
||||
getMpvExecutablePath: () => 'mpv',
|
||||
generateWaveform: async () => [],
|
||||
createPreviewSession: () => {
|
||||
getMpvExecutablePath: () => {
|
||||
throw new Error('preview setup failed');
|
||||
},
|
||||
generateWaveform: async () => [],
|
||||
createPreviewSession: () => ({
|
||||
start: async () => undefined,
|
||||
play: async () => undefined,
|
||||
stop: async () => undefined,
|
||||
dispose: () => undefined,
|
||||
}),
|
||||
openModal: async () => true,
|
||||
showStatus: () => undefined,
|
||||
});
|
||||
|
||||
@@ -11,6 +11,13 @@ import type {
|
||||
MediaTimingReviewWaveformResult,
|
||||
} from '../../types/anki';
|
||||
import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform';
|
||||
import {
|
||||
isRemoteMediaWindowSourcePath,
|
||||
type RemoteMediaWindow,
|
||||
type RemoteMediaWindowRange,
|
||||
type RemoteMediaWindowSource,
|
||||
} from '../../core/services/remote-media-window-cache';
|
||||
import type { MediaInput, MediaInputOptions } from '../../media-input';
|
||||
|
||||
const INITIAL_TIMELINE_MARGIN_SECONDS = 2;
|
||||
const REVIEW_DECISION_TIMEOUT_MS = 5 * 60_000;
|
||||
@@ -31,19 +38,36 @@ interface PreviewSession {
|
||||
executablePath?: string;
|
||||
audioTrackId?: number;
|
||||
volume?: number;
|
||||
absoluteTimestamps?: boolean;
|
||||
}): Promise<void>;
|
||||
play(startTime: number, endTime: number): Promise<void>;
|
||||
stop(): Promise<void>;
|
||||
dispose(): void;
|
||||
}
|
||||
|
||||
interface ReviewMediaSource {
|
||||
path: string;
|
||||
inputOptions?: MediaInputOptions;
|
||||
singleResolvedStream?: boolean;
|
||||
}
|
||||
|
||||
interface ActiveReview {
|
||||
payload: MediaTimingReviewOpenPayload;
|
||||
/** What the hidden mpv preview plays when no cached window is available. */
|
||||
mediaPath: string;
|
||||
/** What the waveform reads when no cached window is available. */
|
||||
waveformMedia: MediaInput;
|
||||
audioStreamIndex?: number;
|
||||
/** Remote source to download windows of; null for local media or without a cache. */
|
||||
windowSource: RemoteMediaWindowSource | null;
|
||||
/** Latest window returned for this review; reused while it still covers the request. */
|
||||
window: RemoteMediaWindow | null;
|
||||
windowRequest: (RemoteMediaWindowRange & { promise: Promise<RemoteMediaWindow | null> }) | null;
|
||||
windowFailed: boolean;
|
||||
previewOptions: { executablePath?: string; audioTrackId?: number; volume?: number };
|
||||
preview: { path: string; session: Promise<PreviewSession> } | null;
|
||||
mpvClient: ReviewMpvClient;
|
||||
restorePlayback: boolean;
|
||||
preview: Promise<PreviewSession>;
|
||||
resolve: (decision: MediaTimingReviewDecision) => void;
|
||||
}
|
||||
|
||||
@@ -53,6 +77,13 @@ export interface MediaTimingReviewRuntimeDeps {
|
||||
getMpvExecutablePath: () => string;
|
||||
createPreviewSession: () => PreviewSession;
|
||||
generateWaveform: (options: SpeechWaveformOptions) => Promise<number[]>;
|
||||
/** Resolves the FFmpeg-readable stream URL and headers behind the current media path. */
|
||||
resolveMediaSource?: () => Promise<ReviewMediaSource | null>;
|
||||
/** Downloads (or reuses) a local window of a remote source covering the range. */
|
||||
acquireMediaWindow?: (
|
||||
source: RemoteMediaWindowSource,
|
||||
range: RemoteMediaWindowRange,
|
||||
) => Promise<RemoteMediaWindow>;
|
||||
getSubtitleContextLines?: (range: { startTime: number; endTime: number }) => {
|
||||
previous: MediaTimingReviewContextLine[];
|
||||
next: MediaTimingReviewContextLine[];
|
||||
@@ -199,6 +230,84 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
}
|
||||
}
|
||||
|
||||
function ensureWindow(
|
||||
review: ActiveReview,
|
||||
range: RemoteMediaWindowRange,
|
||||
): Promise<RemoteMediaWindow | null> {
|
||||
const { windowSource } = review;
|
||||
if (!windowSource || review.windowFailed || !deps.acquireMediaWindow) {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
const coversRange = (candidate: RemoteMediaWindowRange): boolean =>
|
||||
candidate.startTime <= range.startTime && candidate.endTime >= range.endTime;
|
||||
if (review.window && coversRange(review.window)) return Promise.resolve(review.window);
|
||||
const inFlight = review.windowRequest;
|
||||
if (inFlight && coversRange(inFlight)) return inFlight.promise;
|
||||
|
||||
const request = {
|
||||
startTime: range.startTime,
|
||||
endTime: range.endTime,
|
||||
promise: Promise.resolve<RemoteMediaWindow | null>(null),
|
||||
};
|
||||
request.promise = deps
|
||||
.acquireMediaWindow(windowSource, { startTime: range.startTime, endTime: range.endTime })
|
||||
.then((window) => {
|
||||
review.window = window;
|
||||
return window;
|
||||
})
|
||||
.catch(() => {
|
||||
// Fall back to the remote source for the rest of this review instead of retrying.
|
||||
review.windowFailed = true;
|
||||
return null;
|
||||
})
|
||||
.finally(() => {
|
||||
if (review.windowRequest === request) review.windowRequest = null;
|
||||
});
|
||||
review.windowRequest = request;
|
||||
return request.promise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the preview player for the range, restarting it when the range needs a
|
||||
* different file (the first cached window, or a wider one after the timeline grew).
|
||||
*/
|
||||
async function previewFor(
|
||||
review: ActiveReview,
|
||||
range: RemoteMediaWindowRange,
|
||||
): Promise<PreviewSession> {
|
||||
const window = await ensureWindow(review, range);
|
||||
if (active !== review) {
|
||||
// The review ended during the download; do not start a player nobody will dispose.
|
||||
throw new Error('This timing review is no longer active.');
|
||||
}
|
||||
const mediaPath = window?.path ?? review.mediaPath;
|
||||
if (review.preview?.path === mediaPath) return review.preview.session;
|
||||
|
||||
const previous = review.preview;
|
||||
const session = deps.createPreviewSession();
|
||||
const { audioTrackId, ...previewOptions } = review.previewOptions;
|
||||
const started = session
|
||||
.start({
|
||||
mediaPath,
|
||||
...previewOptions,
|
||||
// A cached window keeps one audio stream, so mpv's track id from the source no longer applies.
|
||||
...(window
|
||||
? { absoluteTimestamps: true }
|
||||
: audioTrackId !== undefined
|
||||
? { audioTrackId }
|
||||
: {}),
|
||||
})
|
||||
.then(() => session)
|
||||
.catch((error) => {
|
||||
session.dispose();
|
||||
throw error;
|
||||
});
|
||||
review.preview = { path: mediaPath, session: started };
|
||||
void started.catch(() => {});
|
||||
if (previous) void previous.session.then((old) => old.dispose()).catch(() => {});
|
||||
return started;
|
||||
}
|
||||
|
||||
async function runReview(request: MediaTimingReviewRequest): Promise<MediaTimingReviewDecision> {
|
||||
const mpvClient = deps.getMpvClient();
|
||||
const mediaPath =
|
||||
@@ -208,11 +317,12 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
return { action: 'use-original' };
|
||||
}
|
||||
|
||||
const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw] = await Promise.all([
|
||||
const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw, resolvedSource] = await Promise.all([
|
||||
mpvClient.requestProperty?.('pause').catch(() => null) ?? null,
|
||||
mpvClient.requestProperty?.('duration').catch(() => null) ?? null,
|
||||
mpvClient.requestProperty?.('aid').catch(() => null) ?? null,
|
||||
mpvClient.requestProperty?.('volume').catch(() => null) ?? null,
|
||||
deps.resolveMediaSource?.().catch(() => null) ?? null,
|
||||
]);
|
||||
const pauseState = booleanProperty(pauseRaw);
|
||||
mpvClient.send({ command: ['set_property', 'pause', 'yes'] });
|
||||
@@ -232,38 +342,51 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
mediaDuration: finiteNumber(durationRaw) ?? undefined,
|
||||
...(contextLines ? { contextLines } : {}),
|
||||
});
|
||||
const previewSession = deps.createPreviewSession();
|
||||
const preview = previewSession
|
||||
.start({
|
||||
mediaPath,
|
||||
executablePath: deps.getMpvExecutablePath(),
|
||||
audioTrackId: finiteNumber(audioTrackRaw) ?? undefined,
|
||||
volume: finiteNumber(volumeRaw) ?? undefined,
|
||||
})
|
||||
.then(() => previewSession)
|
||||
.catch((error) => {
|
||||
previewSession.dispose();
|
||||
throw error;
|
||||
});
|
||||
void preview.catch(() => {});
|
||||
const sourcePath = resolvedSource?.path.trim() || mediaPath;
|
||||
const inputOptions = resolvedSource?.inputOptions;
|
||||
const audioStreamIndex =
|
||||
resolvedSource?.singleResolvedStream || mpvClient.currentAudioStreamIndex == null
|
||||
? undefined
|
||||
: mpvClient.currentAudioStreamIndex;
|
||||
const windowSource: RemoteMediaWindowSource | null =
|
||||
deps.acquireMediaWindow && isRemoteMediaWindowSourcePath(sourcePath)
|
||||
? {
|
||||
path: sourcePath,
|
||||
...(inputOptions ? { inputOptions } : {}),
|
||||
audioStreamIndex: audioStreamIndex ?? null,
|
||||
}
|
||||
: null;
|
||||
|
||||
let resolveDecision!: (decision: MediaTimingReviewDecision) => void;
|
||||
const decisionPromise = new Promise<MediaTimingReviewDecision>((resolve) => {
|
||||
resolveDecision = resolve;
|
||||
});
|
||||
active = {
|
||||
const review: ActiveReview = {
|
||||
payload,
|
||||
mediaPath,
|
||||
...(mpvClient.currentAudioStreamIndex !== null &&
|
||||
mpvClient.currentAudioStreamIndex !== undefined
|
||||
? { audioStreamIndex: mpvClient.currentAudioStreamIndex }
|
||||
: {}),
|
||||
waveformMedia: inputOptions ? { path: sourcePath, inputOptions } : sourcePath,
|
||||
...(audioStreamIndex !== undefined ? { audioStreamIndex } : {}),
|
||||
windowSource,
|
||||
window: null,
|
||||
windowRequest: null,
|
||||
windowFailed: false,
|
||||
previewOptions: {
|
||||
executablePath: deps.getMpvExecutablePath(),
|
||||
audioTrackId: finiteNumber(audioTrackRaw) ?? undefined,
|
||||
volume: finiteNumber(volumeRaw) ?? undefined,
|
||||
},
|
||||
preview: null,
|
||||
mpvClient,
|
||||
restorePlayback: pendingPauseRestore === mpvClient,
|
||||
preview,
|
||||
resolve: resolveDecision,
|
||||
};
|
||||
active = review;
|
||||
pendingPauseRestore = null;
|
||||
// Download the visible timeline once now; the waveform and preview both wait on it.
|
||||
void previewFor(review, {
|
||||
startTime: payload.timelineStartTime,
|
||||
endTime: payload.timelineEndTime,
|
||||
}).catch(() => {});
|
||||
|
||||
const opened = await deps.openModal(payload).catch(() => false);
|
||||
if (!opened) {
|
||||
@@ -317,7 +440,10 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
return { ok: false, message: 'The selected preview range is invalid.' };
|
||||
}
|
||||
try {
|
||||
const previewSession = await current.preview;
|
||||
const previewSession = await previewFor(current, request);
|
||||
if (active !== current) {
|
||||
return { ok: false, message: 'This timing review is no longer active.' };
|
||||
}
|
||||
await previewSession.play(request.startTime, request.endTime);
|
||||
return { ok: true };
|
||||
} catch (error) {
|
||||
@@ -347,11 +473,15 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
}
|
||||
|
||||
try {
|
||||
const window = await ensureWindow(current, request);
|
||||
if (active !== current) {
|
||||
return { ok: false, message: 'This timing review is no longer active.' };
|
||||
}
|
||||
const peaks = await deps.generateWaveform({
|
||||
mediaPath: current.mediaPath,
|
||||
mediaPath: window?.media ?? current.waveformMedia,
|
||||
startTime: request.startTime,
|
||||
endTime: request.endTime,
|
||||
...(current.audioStreamIndex !== undefined
|
||||
...(!window && current.audioStreamIndex !== undefined
|
||||
? { audioStreamIndex: current.audioStreamIndex }
|
||||
: {}),
|
||||
});
|
||||
@@ -370,8 +500,8 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
return { ok: false, message: 'This timing review is no longer active.' };
|
||||
}
|
||||
try {
|
||||
const previewSession = await current.preview;
|
||||
await previewSession.stop();
|
||||
const previewSession = current.preview ? await current.preview.session : null;
|
||||
await previewSession?.stop();
|
||||
return { ok: true };
|
||||
} catch (error) {
|
||||
return {
|
||||
@@ -403,7 +533,7 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
const current = active;
|
||||
active = null;
|
||||
if (!current) return;
|
||||
void current.preview.then((session) => session.dispose()).catch(() => {});
|
||||
void current.preview?.session.then((session) => session.dispose()).catch(() => {});
|
||||
if (current.restorePlayback && current.mpvClient.connected) {
|
||||
current.mpvClient.send({ command: ['set_property', 'pause', 'no'] });
|
||||
}
|
||||
|
||||
+141
-31
@@ -10,6 +10,9 @@ import {
|
||||
MediaGenerator,
|
||||
type MediaGeneratorOptions,
|
||||
} from './media-generator';
|
||||
import { RemoteMediaWindowCache } from './core/services/remote-media-window-cache';
|
||||
|
||||
const REMOTE_STREAM_URL = 'https://jellyfin.example/Videos/abc/stream?static=true';
|
||||
|
||||
async function withStubbedFfmpeg(
|
||||
run: (generator: MediaGenerator, argsPath: string) => Promise<void>,
|
||||
@@ -21,6 +24,7 @@ async function withStubbedFfmpeg(
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-media-generator-test-'));
|
||||
const binDir = path.join(root, 'bin');
|
||||
const tempDir = path.join(root, 'media');
|
||||
const windowsDir = path.join(root, 'windows');
|
||||
const argsPath = path.join(root, 'ffmpeg-args.txt');
|
||||
fs.mkdirSync(binDir, { recursive: true });
|
||||
const ffmpegStubPath = path.join(binDir, 'ffmpeg-stub.cjs');
|
||||
@@ -34,7 +38,7 @@ async function withStubbedFfmpeg(
|
||||
" console.log(' V..... libaom-av1');",
|
||||
' process.exit(0);',
|
||||
'}',
|
||||
"fs.writeFileSync(process.env.SUBMINER_TEST_FFMPEG_ARGS, JSON.stringify(args), 'utf8');",
|
||||
"fs.appendFileSync(process.env.SUBMINER_TEST_FFMPEG_ARGS, JSON.stringify(args) + '\\n', 'utf8');",
|
||||
'const outputPath = args.at(-1);',
|
||||
"if (process.env.SUBMINER_TEST_FFMPEG_SKIP_OUTPUT !== '1') {",
|
||||
" fs.writeFileSync(outputPath, 'avif', 'utf8');",
|
||||
@@ -61,12 +65,18 @@ async function withStubbedFfmpeg(
|
||||
} else {
|
||||
delete process.env.SUBMINER_TEST_FFMPEG_SKIP_OUTPUT;
|
||||
}
|
||||
const generator = new MediaGenerator(tempDir, options);
|
||||
// Each test gets its own window cache so remote inputs never leak windows between tests.
|
||||
const remoteMediaWindows = new RemoteMediaWindowCache({ tempDir: windowsDir, idleTtlMs: 0 });
|
||||
const generator = new MediaGenerator(tempDir, {
|
||||
remoteMediaWindows,
|
||||
...options,
|
||||
});
|
||||
|
||||
try {
|
||||
await run(generator, argsPath);
|
||||
} finally {
|
||||
generator.cleanup();
|
||||
remoteMediaWindows.cleanup();
|
||||
process.env.PATH = originalPath;
|
||||
if (originalArgsPath === undefined) {
|
||||
delete process.env.SUBMINER_TEST_FFMPEG_ARGS;
|
||||
@@ -82,8 +92,17 @@ async function withStubbedFfmpeg(
|
||||
}
|
||||
}
|
||||
|
||||
function readAllFfmpegArgs(argsPath: string): string[][] {
|
||||
return fs
|
||||
.readFileSync(argsPath, 'utf8')
|
||||
.split('\n')
|
||||
.filter((line) => line.trim().length > 0)
|
||||
.map((line) => JSON.parse(line) as string[]);
|
||||
}
|
||||
|
||||
/** Arguments of the most recent ffmpeg invocation. */
|
||||
function readFfmpegArgs(argsPath: string): string[] {
|
||||
return JSON.parse(fs.readFileSync(argsPath, 'utf8')) as string[];
|
||||
return readAllFfmpegArgs(argsPath).at(-1) ?? [];
|
||||
}
|
||||
|
||||
test('buildAnimatedImageVideoFilter holds lead-in until the next frame after the audio boundary', () => {
|
||||
@@ -272,41 +291,131 @@ test('generateAudio recreates missing temp directory before invoking ffmpeg', as
|
||||
});
|
||||
|
||||
test('generateAudio adds remote input options before the ffmpeg input', async () => {
|
||||
await withStubbedFfmpeg(async (generator, argsPath) => {
|
||||
await generator.generateAudio(
|
||||
{
|
||||
path: 'https://rr1---sn.example.googlevideo.com/videoplayback?mime=audio%2Fwebm',
|
||||
inputOptions: {
|
||||
reconnect: true,
|
||||
userAgent: 'Mozilla/5.0',
|
||||
headers: {
|
||||
Referer: 'https://www.youtube.com/',
|
||||
Origin: 'https://www.youtube.com',
|
||||
await withStubbedFfmpeg(
|
||||
async (generator, argsPath) => {
|
||||
await generator.generateAudio(
|
||||
{
|
||||
path: 'https://rr1---sn.example.googlevideo.com/videoplayback?mime=audio%2Fwebm',
|
||||
inputOptions: {
|
||||
reconnect: true,
|
||||
userAgent: 'Mozilla/5.0',
|
||||
headers: {
|
||||
Referer: 'https://www.youtube.com/',
|
||||
Origin: 'https://www.youtube.com',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
10,
|
||||
12,
|
||||
);
|
||||
|
||||
const args = readFfmpegArgs(argsPath);
|
||||
const inputIndex = args.indexOf('-i');
|
||||
assert.ok(inputIndex > 0);
|
||||
assert.ok(args.indexOf('-reconnect') > -1);
|
||||
assert.ok(args.indexOf('-reconnect') < inputIndex);
|
||||
assert.equal(args[args.indexOf('-reconnect') + 1], '1');
|
||||
assert.equal(args[args.indexOf('-reconnect_streamed') + 1], '1');
|
||||
assert.equal(args[args.indexOf('-reconnect_on_network_error') + 1], '1');
|
||||
assert.equal(args[args.indexOf('-reconnect_on_http_error') + 1], '403,5xx');
|
||||
assert.equal(args[args.indexOf('-reconnect_delay_max') + 1], '5');
|
||||
assert.equal(args[args.indexOf('-user_agent') + 1], 'Mozilla/5.0');
|
||||
assert.equal(
|
||||
args[args.indexOf('-headers') + 1],
|
||||
'Referer: https://www.youtube.com/\r\nOrigin: https://www.youtube.com\r\n',
|
||||
);
|
||||
},
|
||||
{ remoteMediaWindows: null },
|
||||
);
|
||||
});
|
||||
|
||||
test('generateAudio downloads a remote window once and extracts from it with absolute seeks', async () => {
|
||||
await withStubbedFfmpeg(async (generator, argsPath) => {
|
||||
await generator.generateAudio(
|
||||
{ path: REMOTE_STREAM_URL, inputOptions: { reconnect: true } },
|
||||
10,
|
||||
12,
|
||||
0.5,
|
||||
2,
|
||||
);
|
||||
|
||||
const args = readFfmpegArgs(argsPath);
|
||||
const inputIndex = args.indexOf('-i');
|
||||
assert.ok(inputIndex > 0);
|
||||
assert.ok(args.indexOf('-reconnect') > -1);
|
||||
assert.ok(args.indexOf('-reconnect') < inputIndex);
|
||||
assert.equal(args[args.indexOf('-reconnect') + 1], '1');
|
||||
assert.equal(args[args.indexOf('-reconnect_streamed') + 1], '1');
|
||||
assert.equal(args[args.indexOf('-reconnect_on_network_error') + 1], '1');
|
||||
assert.equal(args[args.indexOf('-reconnect_on_http_error') + 1], '403,5xx');
|
||||
assert.equal(args[args.indexOf('-reconnect_delay_max') + 1], '5');
|
||||
assert.equal(args[args.indexOf('-user_agent') + 1], 'Mozilla/5.0');
|
||||
assert.equal(
|
||||
args[args.indexOf('-headers') + 1],
|
||||
'Referer: https://www.youtube.com/\r\nOrigin: https://www.youtube.com\r\n',
|
||||
);
|
||||
const calls = readAllFfmpegArgs(argsPath);
|
||||
assert.equal(calls.length, 2);
|
||||
const [fetchArgs, audioArgs] = calls as [string[], string[]];
|
||||
assert.equal(fetchArgs[fetchArgs.indexOf('-i') + 1], REMOTE_STREAM_URL);
|
||||
assert.ok(fetchArgs.indexOf('-reconnect') < fetchArgs.indexOf('-i'));
|
||||
assert.equal(fetchArgs[fetchArgs.indexOf('-ss') + 1], '9.25');
|
||||
assert.equal(fetchArgs[fetchArgs.lastIndexOf('-map') + 1], '0:2');
|
||||
assert.ok(fetchArgs.includes('-copyts'));
|
||||
|
||||
const windowPath = audioArgs[audioArgs.indexOf('-i') + 1];
|
||||
assert.ok(windowPath?.endsWith('.mkv'));
|
||||
assert.notEqual(windowPath, REMOTE_STREAM_URL);
|
||||
assert.equal(audioArgs[audioArgs.indexOf('-ss') + 1], '9.5');
|
||||
assert.equal(audioArgs[audioArgs.indexOf('-seek_timestamp') + 1], '1');
|
||||
assert.ok(audioArgs.indexOf('-seek_timestamp') < audioArgs.indexOf('-i'));
|
||||
assert.equal(audioArgs.includes('-reconnect'), false);
|
||||
assert.equal(audioArgs.includes('-map'), false);
|
||||
assert.equal(audioArgs.includes('-probesize'), false);
|
||||
assert.ok(audioArgs.includes('loudnorm=I=-23:TP=-2:LRA=11'));
|
||||
});
|
||||
});
|
||||
|
||||
test('generateScreenshot reuses a downloaded window but never downloads one itself', async () => {
|
||||
await withStubbedFfmpeg(async (generator, argsPath) => {
|
||||
await generator.generateScreenshot(REMOTE_STREAM_URL, 11, { format: 'jpg' });
|
||||
let calls = readAllFfmpegArgs(argsPath);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0]![calls[0]!.indexOf('-i') + 1], REMOTE_STREAM_URL);
|
||||
|
||||
await generator.generateAudio(REMOTE_STREAM_URL, 10, 12);
|
||||
await generator.generateScreenshot(REMOTE_STREAM_URL, 11, { format: 'jpg' });
|
||||
await generator.generateScreenshot(REMOTE_STREAM_URL, 40, { format: 'jpg' });
|
||||
|
||||
calls = readAllFfmpegArgs(argsPath);
|
||||
assert.equal(calls.length, 5);
|
||||
const insideWindow = calls[3]!;
|
||||
assert.ok(insideWindow[insideWindow.indexOf('-i') + 1]?.endsWith('.mkv'));
|
||||
assert.equal(insideWindow[insideWindow.indexOf('-seek_timestamp') + 1], '1');
|
||||
const outsideWindow = calls[4]!;
|
||||
assert.equal(outsideWindow[outsideWindow.indexOf('-i') + 1], REMOTE_STREAM_URL);
|
||||
});
|
||||
});
|
||||
|
||||
test('generateAnimatedImage downloads the clip window before encoding', async () => {
|
||||
await withStubbedFfmpeg(async (generator, argsPath) => {
|
||||
await generator.generateAnimatedImage(REMOTE_STREAM_URL, 10, 12, 0, { fps: 10 });
|
||||
|
||||
const calls = readAllFfmpegArgs(argsPath).filter(
|
||||
(args) => args[0] !== '-hide_banner' || args[1] !== '-encoders',
|
||||
);
|
||||
assert.equal(calls.length, 2);
|
||||
assert.equal(calls[0]![calls[0]!.indexOf('-i') + 1], REMOTE_STREAM_URL);
|
||||
assert.ok(calls[1]![calls[1]!.indexOf('-i') + 1]?.endsWith('.mkv'));
|
||||
assert.equal(calls[1]![calls[1]!.indexOf('-seek_timestamp') + 1], '1');
|
||||
});
|
||||
});
|
||||
|
||||
test('generateAudio reads the remote source directly when the window download fails', async () => {
|
||||
await withStubbedFfmpeg(
|
||||
async (generator, argsPath) => {
|
||||
await generator.generateAudio(REMOTE_STREAM_URL, 10, 12);
|
||||
|
||||
const args = readFfmpegArgs(argsPath);
|
||||
assert.equal(args[args.indexOf('-i') + 1], REMOTE_STREAM_URL);
|
||||
assert.equal(args.includes('-seek_timestamp'), false);
|
||||
},
|
||||
{
|
||||
remoteMediaWindows: new RemoteMediaWindowCache({
|
||||
execFile: (_file, _args, _options, callback) =>
|
||||
queueMicrotask(() => callback(Object.assign(new Error('offline'), { code: 1 }))),
|
||||
idleTtlMs: 0,
|
||||
logDebug: () => undefined,
|
||||
}),
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('generateAudio skips stale audio stream maps for single resolved streams', async () => {
|
||||
await withStubbedFfmpeg(async (generator, argsPath) => {
|
||||
await generator.generateAudio(
|
||||
@@ -320,8 +429,9 @@ test('generateAudio skips stale audio stream maps for single resolved streams',
|
||||
22,
|
||||
);
|
||||
|
||||
const args = readFfmpegArgs(argsPath);
|
||||
assert.equal(args.includes('-map'), false);
|
||||
const [fetchArgs, audioArgs] = readAllFfmpegArgs(argsPath) as [string[], string[]];
|
||||
assert.equal(fetchArgs[fetchArgs.lastIndexOf('-map') + 1], '0:a');
|
||||
assert.equal(audioArgs.includes('-map'), false);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+84
-3
@@ -22,6 +22,12 @@ import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { createLogger } from './logger';
|
||||
import { normalizeMediaInput, type MediaInput } from './media-input';
|
||||
import {
|
||||
getSharedRemoteMediaWindowCache,
|
||||
isRemoteMediaWindowSourcePath,
|
||||
type RemoteMediaWindowCache,
|
||||
type RemoteMediaWindowRange,
|
||||
} from './core/services/remote-media-window-cache';
|
||||
|
||||
const log = createLogger('media');
|
||||
const AUDIO_NORMALIZATION_FILTER = 'loudnorm=I=-23:TP=-2:LRA=11';
|
||||
@@ -86,6 +92,11 @@ export interface MediaGeneratorOptions {
|
||||
logDebug?: (message: string) => void;
|
||||
now?: () => number;
|
||||
execFile?: MediaGeneratorExecFile;
|
||||
/**
|
||||
* Local window cache for http(s) sources. Defaults to the process-wide cache shared
|
||||
* with the timing review; pass `null` to always read remote sources directly.
|
||||
*/
|
||||
remoteMediaWindows?: RemoteMediaWindowCache | null;
|
||||
}
|
||||
|
||||
function sanitizeDebugToken(value: string, fallback: string): string {
|
||||
@@ -232,6 +243,54 @@ export class MediaGenerator {
|
||||
}, delayMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Swaps an http(s) input for the locally cached window that covers `range`, so the
|
||||
* clip is downloaded once instead of per FFmpeg run. `acquire` downloads on a miss;
|
||||
* `lookup` only reuses a window that another step already fetched. Any failure falls
|
||||
* back to reading the remote source directly.
|
||||
*/
|
||||
private async resolveRemoteWindowInput(
|
||||
input: MediaInput,
|
||||
range: RemoteMediaWindowRange,
|
||||
audioStreamIndex: number | null | undefined,
|
||||
mode: 'acquire' | 'lookup',
|
||||
): Promise<MediaInput> {
|
||||
const cache =
|
||||
this.options.remoteMediaWindows === undefined
|
||||
? getSharedRemoteMediaWindowCache()
|
||||
: this.options.remoteMediaWindows;
|
||||
const sourcePath = typeof input === 'string' ? input : input.path;
|
||||
if (!cache || !isRemoteMediaWindowSourcePath(sourcePath)) {
|
||||
return input;
|
||||
}
|
||||
const source = {
|
||||
path: sourcePath,
|
||||
...(typeof input === 'object' && input.inputOptions
|
||||
? { inputOptions: input.inputOptions }
|
||||
: {}),
|
||||
audioStreamIndex:
|
||||
typeof input === 'object' && input.singleResolvedStream ? null : (audioStreamIndex ?? null),
|
||||
};
|
||||
const description = describeMediaInputForDebugLog(input);
|
||||
try {
|
||||
const window =
|
||||
mode === 'acquire' ? await cache.acquire(source, range) : await cache.lookup(source, range);
|
||||
if (!window) {
|
||||
this.logMediaDebug(`window miss ${description} mode=${mode}`);
|
||||
return input;
|
||||
}
|
||||
this.logMediaDebug(
|
||||
`window hit ${description} mode=${mode} start=${window.startTime} end=${window.endTime}`,
|
||||
);
|
||||
return window.media;
|
||||
} catch (error) {
|
||||
this.logMediaDebug(
|
||||
`window failed ${description} mode=${mode} reason=${sanitizeDebugToken((error as Error).message, 'error')}`,
|
||||
);
|
||||
return input;
|
||||
}
|
||||
}
|
||||
|
||||
private ffmpegError(label: string, error: ExecFileException): Error {
|
||||
if (error.code === 'ENOENT') {
|
||||
return new Error('FFmpeg not found. Install FFmpeg to enable media generation.');
|
||||
@@ -281,7 +340,13 @@ export class MediaGenerator {
|
||||
const safePadding = Number.isFinite(padding) ? Math.max(0, padding) : 0;
|
||||
const start = Math.max(0, startTime - safePadding);
|
||||
const duration = endTime - start + safePadding;
|
||||
const mediaInput = normalizeMediaInput(videoPath);
|
||||
const sourceInput = await this.resolveRemoteWindowInput(
|
||||
videoPath,
|
||||
{ startTime: start, endTime: start + duration },
|
||||
audioStreamIndex,
|
||||
'acquire',
|
||||
);
|
||||
const mediaInput = normalizeMediaInput(sourceInput);
|
||||
const inputDescription = describeMediaInputForDebugLog(videoPath);
|
||||
const hasSelectedAudioStream =
|
||||
!mediaInput.singleResolvedStream &&
|
||||
@@ -385,7 +450,15 @@ export class MediaGenerator {
|
||||
png: 'png',
|
||||
webp: 'webp',
|
||||
};
|
||||
const mediaInput = normalizeMediaInput(videoPath);
|
||||
// A single frame is cheap to fetch remotely, so only reuse a window another step downloaded.
|
||||
const mediaInput = normalizeMediaInput(
|
||||
await this.resolveRemoteWindowInput(
|
||||
videoPath,
|
||||
{ startTime: timestamp, endTime: timestamp },
|
||||
null,
|
||||
'lookup',
|
||||
),
|
||||
);
|
||||
const inputDescription = describeMediaInputForDebugLog(videoPath);
|
||||
|
||||
const args: string[] = [
|
||||
@@ -533,9 +606,17 @@ export class MediaGenerator {
|
||||
);
|
||||
}
|
||||
|
||||
const mediaInput = normalizeMediaInput(
|
||||
await this.resolveRemoteWindowInput(
|
||||
videoPath,
|
||||
{ startTime: start, endTime: start + duration },
|
||||
null,
|
||||
'acquire',
|
||||
),
|
||||
);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const outputPath = this.createTempOutputPath('animation', 'avif');
|
||||
const mediaInput = normalizeMediaInput(videoPath);
|
||||
const startedAt = this.nowMs();
|
||||
|
||||
const encoderArgs: string[] = ['-c:v', av1Encoder];
|
||||
|
||||
@@ -11,6 +11,12 @@ export type MediaInput =
|
||||
source?: string;
|
||||
inputOptions?: MediaInputOptions;
|
||||
singleResolvedStream?: boolean;
|
||||
/**
|
||||
* The file keeps the original media timestamps instead of starting at zero (a
|
||||
* stream-copied window of a longer source). Seek with `-ss` against those
|
||||
* absolute timestamps rather than relative to the file's own start time.
|
||||
*/
|
||||
absoluteTimestamps?: boolean;
|
||||
};
|
||||
|
||||
export type NormalizedMediaInput = {
|
||||
@@ -89,6 +95,10 @@ export function normalizeMediaInput(input: MediaInput): NormalizedMediaInput {
|
||||
inputArgs.push('-headers', headers);
|
||||
}
|
||||
|
||||
if (input.absoluteTimestamps) {
|
||||
inputArgs.push('-seek_timestamp', '1');
|
||||
}
|
||||
|
||||
return {
|
||||
path: input.path,
|
||||
inputArgs,
|
||||
|
||||
Reference in New Issue
Block a user