diff --git a/changes/remote-media-window-cache.md b/changes/remote-media-window-cache.md new file mode 100644 index 00000000..25b3192f --- /dev/null +++ b/changes/remote-media-window-cache.md @@ -0,0 +1,4 @@ +type: changed +area: mining + +- Mining a card from a remote stream (Jellyfin and other HTTP sources) now downloads the clip window once into a temporary file and reuses it for the timing review waveform, audio preview, audio extraction, and screenshot, instead of re-fetching the stream for every step. The temporary window is removed after ten minutes without use or on exit. diff --git a/docs-site/anki-integration.md b/docs-site/anki-integration.md index c6a78960..f573a81a 100644 --- a/docs-site/anki-integration.md +++ b/docs-site/anki-integration.md @@ -158,6 +158,8 @@ If you only want sentence and audio on your cards: SubMiner uses FFmpeg to generate audio and image media from the video. FFmpeg must be installed and on `PATH`. +For remote streams such as Jellyfin playback, SubMiner downloads the clip's time window once into a temporary Matroska file (a stream copy, no re-encoding) and reads the timing review waveform, audio preview, audio, and image from that file instead of fetching the stream again for each step. The window covers the clip plus padding, plus the visible timeline in timing review, and grows when you reveal more of the timeline. It is deleted when a different window replaces it, after ten minutes without use, or when SubMiner exits. If the download fails, media generation reads the remote stream directly as before. + ### Audio Audio is extracted from the video file using the subtitle's start and end timestamps. Padding is opt-in; keep it at `0` when you want sentence audio to start exactly at the mined sentence. @@ -351,11 +353,11 @@ For Senren note types, enable `isSenren` instead. Kiku and Senren write incompat ### What Gets Merged -| Field | Merge behavior | -| -------- | --------------------------------------------- | -| Sentence | Both cards' sentences kept as grouped entries | -| Audio | Both cards' `[sound:...]` entries kept | -| Image | Both cards' images kept | +| Field | Merge behavior | +| -------- | ----------------------------------------------- | +| Sentence | Both cards' sentences kept as grouped entries | +| Audio | Both cards' `[sound:...]` entries kept | +| Image | Both cards' images kept | | MiscInfo | Both cards' source info kept as grouped entries | Identical values from both cards are kept as separate grouped entries; the merge does not deduplicate. diff --git a/src/core/services/media-timing-preview.test.ts b/src/core/services/media-timing-preview.test.ts index b972a8df..bb9d1168 100644 --- a/src/core/services/media-timing-preview.test.ts +++ b/src/core/services/media-timing-preview.test.ts @@ -23,6 +23,21 @@ describe('buildMediaTimingPreviewArgs', () => { assert.equal(args.at(-1), '/video/show.mkv'); }); + test('keeps source timestamps for cached remote windows', () => { + const args = buildMediaTimingPreviewArgs('/tmp/review.sock', { + mediaPath: '/tmp/window.mkv', + absoluteTimestamps: true, + }); + + assert.ok(args.includes('--rebase-start-time=no')); + assert.equal( + buildMediaTimingPreviewArgs('/tmp/review.sock', { mediaPath: '/video/show.mkv' }).includes( + '--rebase-start-time=no', + ), + false, + ); + }); + test('separates an option-like media path without adding optional audio arguments', () => { const args = buildMediaTimingPreviewArgs('/tmp/review.sock', { mediaPath: '--fullscreen', diff --git a/src/core/services/media-timing-preview.ts b/src/core/services/media-timing-preview.ts index 693721e3..578a3a2c 100644 --- a/src/core/services/media-timing-preview.ts +++ b/src/core/services/media-timing-preview.ts @@ -14,6 +14,8 @@ export interface MediaTimingPreviewStartOptions { executablePath?: string; audioTrackId?: number; volume?: number; + /** The file keeps source timestamps (a cached remote window); seek with the original times. */ + absoluteTimestamps?: boolean; } type PreviewProcess = Pick; @@ -51,6 +53,9 @@ export function buildMediaTimingPreviewArgs( if (typeof options.volume === 'number' && Number.isFinite(options.volume)) { args.push(`--volume=${Math.max(0, options.volume)}`); } + if (options.absoluteTimestamps) { + args.push('--rebase-start-time=no'); + } args.push('--', options.mediaPath); return args; } diff --git a/src/core/services/media-timing-waveform.test.ts b/src/core/services/media-timing-waveform.test.ts index 38376ba1..bdeff0e0 100644 --- a/src/core/services/media-timing-waveform.test.ts +++ b/src/core/services/media-timing-waveform.test.ts @@ -35,6 +35,29 @@ test('speech waveform maps the selected FFmpeg stream and visible range', () => assert.match(args[args.indexOf('-af') + 1] ?? '', /c0=FC/); }); +test('speech waveform seeks cached windows by source timestamps', () => { + const args = buildSpeechWaveformArgs( + { + mediaPath: { path: '/tmp/window.mkv', absoluteTimestamps: true, singleResolvedStream: true }, + startTime: 8, + endTime: 15, + }, + 'downmix', + ); + + assert.deepEqual(args.slice(args.indexOf('-ss'), args.indexOf('-t') + 2), [ + '-ss', + '8', + '-seek_timestamp', + '1', + '-i', + '/tmp/window.mkv', + '-t', + '7', + ]); + assert.equal(args.includes('-map'), false); +}); + test('waveform peaks are normalized without flattening quieter sections', () => { const peaks = computeWaveformPeaks(pcm([0, 1_000, -2_000, 4_000, -8_000, 16_000]), 3); diff --git a/src/core/services/media-timing-waveform.ts b/src/core/services/media-timing-waveform.ts index 8fbd1a63..157a76fd 100644 --- a/src/core/services/media-timing-waveform.ts +++ b/src/core/services/media-timing-waveform.ts @@ -1,4 +1,5 @@ import { spawn } from 'node:child_process'; +import { normalizeMediaInput, type MediaInput } from '../../media-input'; const WAVEFORM_SAMPLE_RATE = 8_000; const WAVEFORM_POINT_COUNT = 480; @@ -9,7 +10,7 @@ const CENTER_CHANNEL_FILTER = `pan=mono|c0=FC,${SPEECH_FILTER}`; const DOWNMIX_FILTER = `aformat=channel_layouts=mono,${SPEECH_FILTER}`; export interface SpeechWaveformOptions { - mediaPath: string; + mediaPath: MediaInput; startTime: number; endTime: number; audioStreamIndex?: number; @@ -22,6 +23,7 @@ export function buildSpeechWaveformArgs( mode: 'center' | 'downmix', ): string[] { const duration = options.endTime - options.startTime; + const input = normalizeMediaInput(options.mediaPath); const args = [ '-hide_banner', '-nostdin', @@ -29,8 +31,9 @@ export function buildSpeechWaveformArgs( 'error', '-ss', String(options.startTime), + ...input.inputArgs, '-i', - options.mediaPath, + input.path, '-t', String(duration), ]; diff --git a/src/core/services/remote-media-window-cache.test.ts b/src/core/services/remote-media-window-cache.test.ts new file mode 100644 index 00000000..836ca149 --- /dev/null +++ b/src/core/services/remote-media-window-cache.test.ts @@ -0,0 +1,253 @@ +import assert from 'node:assert/strict'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import test from 'node:test'; +import { + buildRemoteMediaWindowArgs, + RemoteMediaWindowCache, + REMOTE_MEDIA_WINDOW_MAX_SECONDS, + type RemoteMediaWindowCacheOptions, +} from './remote-media-window-cache'; + +const SOURCE = { + path: 'https://jellyfin.example/Videos/abc/stream?static=true', + audioStreamIndex: 2, +}; + +type ExecFileStub = NonNullable; + +function createStub(options: { fail?: boolean; empty?: boolean; defer?: boolean } = {}) { + const calls: string[][] = []; + const pendingCallbacks: Array<() => void> = []; + const execFile: ExecFileStub = (_file, args, _options, callback) => { + calls.push([...args]); + const finish = (): void => { + const outputPath = args.at(-1); + assert.ok(outputPath); + if (options.fail) { + callback(Object.assign(new Error('boom'), { code: 1 })); + return; + } + if (!options.empty) { + fs.writeFileSync(outputPath, 'mkv', 'utf8'); + } + callback(null); + }; + if (options.defer) { + pendingCallbacks.push(finish); + } else { + queueMicrotask(finish); + } + }; + return { + calls, + execFile, + flush: () => { + for (const finish of pendingCallbacks.splice(0)) finish(); + }, + }; +} + +async function withCache( + stubOptions: Parameters[0], + cacheOptions: Omit, + run: (cache: RemoteMediaWindowCache, stub: ReturnType) => Promise, +): Promise { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-media-window-test-')); + const stub = createStub(stubOptions); + const cache = new RemoteMediaWindowCache({ + tempDir, + execFile: stub.execFile, + idleTtlMs: 0, + logDebug: () => undefined, + ...cacheOptions, + }); + try { + await run(cache, stub); + } finally { + cache.cleanup(); + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +function argValue(args: string[], flag: string): string | undefined { + const index = args.indexOf(flag); + return index === -1 ? undefined : args[index + 1]; +} + +test('buildRemoteMediaWindowArgs stream-copies the window with source timestamps intact', () => { + const args = buildRemoteMediaWindowArgs( + { ...SOURCE, inputOptions: { reconnect: true, headers: { Referer: 'https://a.example/' } } }, + { startTime: 22.75, endTime: 33 }, + '/tmp/window.mkv', + ); + + const inputIndex = args.indexOf('-i'); + assert.equal(args[inputIndex + 1], SOURCE.path); + assert.ok(args.indexOf('-reconnect') < inputIndex); + assert.ok(args.indexOf('-headers') < inputIndex); + assert.equal(argValue(args, '-ss'), '22.75'); + assert.equal(argValue(args, '-t'), '10.25'); + assert.ok(args.indexOf('-t') < inputIndex); + assert.deepEqual(args.slice(args.indexOf('-map'), args.indexOf('-map') + 4), [ + '-map', + '0:v:0?', + '-map', + '0:2', + ]); + assert.equal(argValue(args, '-c'), 'copy'); + assert.ok(args.includes('-copyts')); + assert.ok(args.includes('-start_at_zero')); + assert.equal(argValue(args, '-f'), 'matroska'); + assert.equal(args.at(-1), '/tmp/window.mkv'); +}); + +test('buildRemoteMediaWindowArgs keeps every audio stream when none is selected', () => { + const args = buildRemoteMediaWindowArgs( + { path: SOURCE.path, audioStreamIndex: null }, + { startTime: 0, endTime: 5 }, + '/tmp/window.mkv', + ); + + assert.equal(args[args.lastIndexOf('-map') + 1], '0:a'); +}); + +test('acquire downloads once and reuses the window for covered ranges', async () => { + await withCache({}, {}, async (cache, stub) => { + const window = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 }); + + assert.equal(stub.calls.length, 1); + assert.equal(argValue(stub.calls[0]!, '-ss'), '9.75'); + assert.equal(argValue(stub.calls[0]!, '-t'), '5.25'); + assert.equal(window.startTime, 9.75); + assert.equal(window.endTime, 15); + assert.equal(window.audioStreamIndex, 2); + assert.ok(fs.existsSync(window.path)); + assert.deepEqual(window.media, { + path: window.path, + source: 'remote-window', + singleResolvedStream: true, + absoluteTimestamps: true, + }); + + assert.equal(await cache.acquire(SOURCE, { startTime: 11, endTime: 15 }), window); + assert.equal(await cache.lookup(SOURCE, { startTime: 12, endTime: 12 }), window); + assert.equal( + await cache.lookup( + { path: SOURCE.path, audioStreamIndex: null }, + { startTime: 12, endTime: 13 }, + ), + window, + ); + assert.equal(stub.calls.length, 1); + }); +}); + +test('lookup never downloads and misses on other ranges, sources, or audio streams', async () => { + await withCache({}, {}, async (cache, stub) => { + assert.equal(await cache.lookup(SOURCE, { startTime: 10, endTime: 14 }), null); + assert.equal(stub.calls.length, 0); + + await cache.acquire(SOURCE, { startTime: 10, endTime: 14 }); + assert.equal(await cache.lookup(SOURCE, { startTime: 14, endTime: 16 }), null); + assert.equal( + await cache.lookup( + { path: 'https://other.example/stream', audioStreamIndex: 2 }, + { + startTime: 11, + endTime: 12, + }, + ), + null, + ); + assert.equal( + await cache.lookup( + { path: SOURCE.path, audioStreamIndex: 3 }, + { startTime: 11, endTime: 12 }, + ), + null, + ); + assert.equal(stub.calls.length, 1); + }); +}); + +test('acquire widens to the union of the old window and replaces the old file', async () => { + await withCache({}, {}, async (cache, stub) => { + const first = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 }); + const second = await cache.acquire(SOURCE, { startTime: 8, endTime: 12 }); + + assert.equal(stub.calls.length, 2); + assert.equal(argValue(stub.calls[1]!, '-ss'), '7.75'); + assert.equal(second.startTime, 7.75); + assert.equal(second.endTime, 15); + assert.notEqual(second.path, first.path); + assert.equal(fs.existsSync(first.path), false); + assert.ok(fs.existsSync(second.path)); + assert.equal(cache.currentWindow, second); + }); +}); + +test('acquire shares an in-flight download between concurrent callers', async () => { + await withCache({ defer: true }, {}, async (cache, stub) => { + const first = cache.acquire(SOURCE, { startTime: 10, endTime: 14 }); + await Promise.resolve(); + const second = cache.acquire(SOURCE, { startTime: 11, endTime: 13 }); + const lookup = cache.lookup(SOURCE, { startTime: 12, endTime: 12 }); + await Promise.resolve(); + assert.equal(stub.calls.length, 1); + + stub.flush(); + const [a, b, c] = await Promise.all([first, second, lookup]); + assert.equal(a, b); + assert.equal(a, c); + assert.equal(stub.calls.length, 1); + }); +}); + +test('acquire rejects on ffmpeg failure, leaves no file, and can retry', async () => { + await withCache({ fail: true }, {}, async (cache, stub) => { + await assert.rejects( + cache.acquire(SOURCE, { startTime: 10, endTime: 14 }), + /FFmpeg media window failed: boom/, + ); + assert.equal(cache.currentWindow, null); + assert.equal(await cache.lookup(SOURCE, { startTime: 10, endTime: 14 }), null); + + await assert.rejects(cache.acquire(SOURCE, { startTime: 10, endTime: 14 })); + assert.equal(stub.calls.length, 2); + }); + await withCache({ empty: true }, {}, async (cache) => { + await assert.rejects( + cache.acquire(SOURCE, { startTime: 10, endTime: 14 }), + /exited without creating a media window/, + ); + }); +}); + +test('acquire refuses invalid and oversized ranges without spawning ffmpeg', async () => { + await withCache({}, {}, async (cache, stub) => { + await assert.rejects(cache.acquire(SOURCE, { startTime: 10, endTime: 10 }), /invalid/); + await assert.rejects(cache.acquire(SOURCE, { startTime: -1, endTime: 10 }), /invalid/); + await assert.rejects( + cache.acquire(SOURCE, { startTime: 0, endTime: REMOTE_MEDIA_WINDOW_MAX_SECONDS + 1 }), + /too long/, + ); + assert.equal(stub.calls.length, 0); + }); +}); + +test('the window is deleted after the idle timeout and on cleanup', async () => { + await withCache({}, { idleTtlMs: 20 }, async (cache) => { + const window = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 }); + await new Promise((resolve) => setTimeout(resolve, 60)); + + assert.equal(cache.currentWindow, null); + assert.equal(fs.existsSync(window.path), false); + + const again = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 }); + cache.cleanup(); + assert.equal(fs.existsSync(again.path), false); + assert.equal(fs.existsSync(path.dirname(again.path)), false); + }); +}); diff --git a/src/core/services/remote-media-window-cache.ts b/src/core/services/remote-media-window-cache.ts new file mode 100644 index 00000000..ca486a66 --- /dev/null +++ b/src/core/services/remote-media-window-cache.ts @@ -0,0 +1,377 @@ +import { execFile as nodeExecFile, type ExecFileException } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { createLogger } from '../../logger'; +import { normalizeMediaInput, type MediaInput, type MediaInputOptions } from '../../media-input'; + +const log = createLogger('media-window'); + +export const REMOTE_MEDIA_WINDOW_TIMEOUT_MS = 120_000; +export const REMOTE_MEDIA_WINDOW_MAX_SECONDS = 180; +const HEAD_SLACK_SECONDS = 0.25; +const TAIL_SLACK_SECONDS = 1; +const DEFAULT_IDLE_TTL_MS = 10 * 60_000; +const COVERAGE_EPSILON_SECONDS = 0.01; + +export interface RemoteMediaWindowSource { + path: string; + inputOptions?: MediaInputOptions; + /** FFmpeg stream index to keep; `null`/undefined keeps every audio stream. */ + audioStreamIndex?: number | null; +} + +export interface RemoteMediaWindowRange { + startTime: number; + endTime: number; +} + +export interface RemoteMediaWindow { + path: string; + startTime: number; + endTime: number; + sourcePath: string; + audioStreamIndex: number | null; + /** Input descriptor for FFmpeg reads; timestamps stay absolute so callers keep source times. */ + media: MediaInput; +} + +type WindowExecFile = ( + file: string, + args: readonly string[], + options: { timeout: number }, + callback: (error: ExecFileException | null) => void, +) => void; + +export interface RemoteMediaWindowCacheOptions { + tempDir?: string; + execFile?: WindowExecFile; + idleTtlMs?: number; + logDebug?: (message: string) => void; +} + +interface PendingFetch extends RemoteMediaWindowRange { + sourcePath: string; + audioStreamIndex: number | null; + promise: Promise; +} + +export function isRemoteMediaWindowSourcePath(value: string): boolean { + return /^https?:\/\//i.test(value.trim()); +} + +function describeSourceForDebugLog(sourcePath: string): string { + try { + return `remote:${new URL(sourcePath).hostname.toLowerCase() || 'unknown'}`; + } catch { + return 'remote:unknown'; + } +} + +function isUsableRange(range: RemoteMediaWindowRange, allowEmpty: boolean): boolean { + return ( + Number.isFinite(range.startTime) && + Number.isFinite(range.endTime) && + range.startTime >= 0 && + (allowEmpty ? range.endTime >= range.startTime : range.endTime > range.startTime) + ); +} + +function audioStreamMatches( + windowIndex: number | null, + requested: number | null | undefined, +): boolean { + return requested == null || windowIndex === requested; +} + +function covers( + candidate: RemoteMediaWindowRange & { sourcePath: string; audioStreamIndex: number | null }, + source: RemoteMediaWindowSource, + range: RemoteMediaWindowRange, +): boolean { + return ( + candidate.sourcePath === source.path && + audioStreamMatches(candidate.audioStreamIndex, source.audioStreamIndex) && + candidate.startTime <= range.startTime + COVERAGE_EPSILON_SECONDS && + candidate.endTime >= range.endTime - COVERAGE_EPSILON_SECONDS + ); +} + +/** + * Stream-copies `[startTime, endTime]` of a remote source into a local Matroska file. + * `-copyts -start_at_zero` keeps the source timestamps, so later reads seek with the + * original times via `-seek_timestamp 1` (see `MediaInput.absoluteTimestamps`). + */ +export function buildRemoteMediaWindowArgs( + source: RemoteMediaWindowSource, + range: RemoteMediaWindowRange, + outputPath: string, +): string[] { + const input = normalizeMediaInput({ path: source.path, inputOptions: source.inputOptions }); + const audioMap = + typeof source.audioStreamIndex === 'number' && Number.isInteger(source.audioStreamIndex) + ? `0:${source.audioStreamIndex}` + : '0:a'; + return [ + '-hide_banner', + '-nostdin', + '-loglevel', + 'error', + '-ss', + String(range.startTime), + '-t', + String(range.endTime - range.startTime), + ...input.inputArgs, + '-i', + input.path, + '-map', + '0:v:0?', + '-map', + audioMap, + '-c', + 'copy', + '-sn', + '-dn', + '-copyts', + '-start_at_zero', + '-f', + 'matroska', + '-y', + outputPath, + ]; +} + +/** + * Holds one downloaded window of the current remote stream so the timing review, + * audio extraction, and screenshot all read the same local bytes instead of each + * re-fetching the clip over HTTP. A new window replaces the old one; the file is + * deleted after `idleTtlMs` without use, on `clear()`, or on `cleanup()`. + */ +export class RemoteMediaWindowCache { + private readonly tempDir: string; + private readonly execFile: WindowExecFile; + private readonly idleTtlMs: number; + private readonly logDebug: (message: string) => void; + private current: RemoteMediaWindow | null = null; + private pending: PendingFetch | null = null; + private idleTimer: ReturnType | null = null; + private sequence = 0; + + constructor(options: RemoteMediaWindowCacheOptions = {}) { + this.tempDir = options.tempDir ?? path.join(os.tmpdir(), 'subminer-media-windows'); + this.execFile = options.execFile ?? nodeExecFile; + this.idleTtlMs = options.idleTtlMs ?? DEFAULT_IDLE_TTL_MS; + this.logDebug = options.logDebug ?? ((message) => log.debug(message)); + } + + get currentWindow(): RemoteMediaWindow | null { + return this.current; + } + + /** Returns a ready or in-flight window covering the range; never starts a download. */ + async lookup( + source: RemoteMediaWindowSource, + range: RemoteMediaWindowRange, + ): Promise { + if (!isUsableRange(range, true)) return null; + if (this.current && covers(this.current, source, range)) { + this.touch(); + return this.current; + } + const pending = this.pending; + if (pending && covers(pending, source, range)) { + try { + const window = await pending.promise; + this.touch(); + return window; + } catch { + return null; + } + } + return null; + } + + /** Returns a window covering the range, downloading (and widening) one when needed. */ + async acquire( + source: RemoteMediaWindowSource, + range: RemoteMediaWindowRange, + ): Promise { + if (!isUsableRange(range, false)) { + throw new Error('Media window range is invalid.'); + } + if (range.endTime - range.startTime > REMOTE_MEDIA_WINDOW_MAX_SECONDS) { + throw new Error('Media window range is too long to download.'); + } + + for (;;) { + const hit = await this.lookup(source, range); + if (hit) return hit; + const pending = this.pending; + if (!pending) break; + // Another caller is already downloading; wait for it, then re-check coverage. + await pending.promise.catch(() => null); + } + + return this.fetch(source, this.planFetchRange(source, range)); + } + + clear(): void { + this.cancelIdleTimer(); + const current = this.current; + this.current = null; + if (current) this.removeFile(current.path); + } + + cleanup(): void { + this.clear(); + try { + fs.rmSync(this.tempDir, { recursive: true, force: true }); + } catch (error) { + log.error('Failed to cleanup media window directory:', error); + } + } + + private planFetchRange( + source: RemoteMediaWindowSource, + range: RemoteMediaWindowRange, + ): RemoteMediaWindowRange { + let startTime = Math.max(0, range.startTime - HEAD_SLACK_SECONDS); + let endTime = range.endTime + TAIL_SLACK_SECONDS; + const current = this.current; + if ( + current && + current.sourcePath === source.path && + audioStreamMatches(current.audioStreamIndex, source.audioStreamIndex) + ) { + // Keep what was already downloaded when the review timeline grows in one direction. + const unionStart = Math.min(startTime, current.startTime); + const unionEnd = Math.max(endTime, current.endTime); + if (unionEnd - unionStart <= REMOTE_MEDIA_WINDOW_MAX_SECONDS) { + startTime = unionStart; + endTime = unionEnd; + } + } + return { startTime, endTime }; + } + + private fetch( + source: RemoteMediaWindowSource, + range: RemoteMediaWindowRange, + ): Promise { + fs.mkdirSync(this.tempDir, { recursive: true }); + this.sequence += 1; + const outputPath = path.join(this.tempDir, `window_${Date.now()}_${this.sequence}.mkv`); + const audioStreamIndex = + typeof source.audioStreamIndex === 'number' ? source.audioStreamIndex : null; + const description = describeSourceForDebugLog(source.path); + const startedAt = Date.now(); + this.logDebug( + `[media-window] fetch start ${description} start=${range.startTime} end=${range.endTime} audioStream=${audioStreamIndex ?? 'all'}`, + ); + + const promise = new Promise((resolve, reject) => { + this.execFile( + 'ffmpeg', + buildRemoteMediaWindowArgs(source, range, outputPath), + { timeout: REMOTE_MEDIA_WINDOW_TIMEOUT_MS }, + (error) => { + const elapsedMs = Math.max(0, Date.now() - startedAt); + const size = error ? 0 : this.fileSize(outputPath); + if (error || size === 0) { + this.removeFile(outputPath); + const reason = error + ? error.code === 'ENOENT' + ? 'FFmpeg not found. Install FFmpeg to enable media generation.' + : `FFmpeg media window failed: ${error.message}` + : 'FFmpeg exited without creating a media window.'; + this.logDebug(`[media-window] fetch failed ${description} elapsedMs=${elapsedMs}`); + reject(new Error(reason)); + return; + } + const window: RemoteMediaWindow = { + path: outputPath, + startTime: range.startTime, + endTime: range.endTime, + sourcePath: source.path, + audioStreamIndex, + media: { + path: outputPath, + source: 'remote-window', + singleResolvedStream: true, + absoluteTimestamps: true, + }, + }; + this.logDebug( + `[media-window] fetch complete ${description} elapsedMs=${elapsedMs} bytes=${size}`, + ); + this.replaceCurrent(window); + resolve(window); + }, + ); + }); + + const pending: PendingFetch = { + sourcePath: source.path, + audioStreamIndex, + startTime: range.startTime, + endTime: range.endTime, + promise, + }; + this.pending = pending; + promise + .catch(() => undefined) + .then(() => { + if (this.pending === pending) this.pending = null; + }); + return promise; + } + + private replaceCurrent(window: RemoteMediaWindow): void { + const previous = this.current; + this.current = window; + if (previous && previous.path !== window.path) this.removeFile(previous.path); + this.touch(); + } + + private touch(): void { + this.cancelIdleTimer(); + if (this.idleTtlMs <= 0 || !this.current) return; + const timer = setTimeout(() => { + if (this.idleTimer === timer) this.idleTimer = null; + this.clear(); + }, this.idleTtlMs); + timer.unref?.(); + this.idleTimer = timer; + } + + private cancelIdleTimer(): void { + if (this.idleTimer) clearTimeout(this.idleTimer); + this.idleTimer = null; + } + + private fileSize(filePath: string): number { + try { + return fs.statSync(filePath).size; + } catch { + return 0; + } + } + + private removeFile(filePath: string): void { + try { + fs.unlinkSync(filePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + log.debug(`Failed to remove media window ${filePath}:`, (error as Error).message); + } + } + } +} + +let sharedCache: RemoteMediaWindowCache | null = null; + +/** Process-wide cache so the review modal and card media generation share one download. */ +export function getSharedRemoteMediaWindowCache(): RemoteMediaWindowCache { + sharedCache ??= new RemoteMediaWindowCache(); + return sharedCache; +} diff --git a/src/main.ts b/src/main.ts index cca162b0..e51a0924 100644 --- a/src/main.ts +++ b/src/main.ts @@ -464,6 +464,8 @@ import { handleCliCommandRuntimeServiceWithContext } from './main/cli-runtime'; import { createOverlayModalRuntimeService } from './main/overlay-runtime'; import { createOverlayModalInputState } from './main/runtime/overlay-modal-input-state'; import { MediaTimingPreviewSession } from './core/services/media-timing-preview'; +import { getSharedRemoteMediaWindowCache } from './core/services/remote-media-window-cache'; +import { resolveMediaGenerationInput } from './anki-integration/media-source'; import { generateSpeechWaveform } from './core/services/media-timing-waveform'; import { collectMediaTimingContextLines, @@ -2896,6 +2898,21 @@ const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({ configService.getConfig().mpv.executablePath || process.env.SUBMINER_MPV_PATH?.trim() || '', createPreviewSession: () => new MediaTimingPreviewSession(), generateWaveform: (options) => generateSpeechWaveform(options), + resolveMediaSource: async () => { + const resolved = await resolveMediaGenerationInput(appState.mpvClient, 'audio', { + getCachedMediaPath: (currentVideoPath, kind) => + getCachedYoutubeMediaPathForCurrentPlayback(currentVideoPath, kind), + remoteCacheMode: shouldRequireYoutubeMediaCacheForCurrentPlayback() ? 'required' : 'optional', + }); + return resolved + ? { + path: resolved.path, + ...(resolved.inputOptions ? { inputOptions: resolved.inputOptions } : {}), + singleResolvedStream: resolved.singleResolvedStream, + } + : null; + }, + acquireMediaWindow: (source, range) => getSharedRemoteMediaWindowCache().acquire(source, range), getSubtitleContextLines: (range) => collectMediaTimingContextLines({ cues: appState.activeParsedSubtitleCues, @@ -3973,6 +3990,7 @@ const { cleanupInternalSubtitleTrackCache: () => cachedInternalSubtitleTrackExtractor.clear(), cleanupYoutubeSubtitleTempDirs: () => youtubeFlowRuntime.cleanupSubtitleTempDirs(), cleanupYoutubeMediaCache: () => youtubeMediaCache.cleanup(), + cleanupRemoteMediaWindows: () => getSharedRemoteMediaWindowCache().cleanup(), cleanupJellyfinSubtitleCache: () => cleanupJellyfinSubtitleCache(), stopDiscordPresenceService: () => { void appState.discordPresenceService?.stop(); diff --git a/src/main/runtime/app-lifecycle-actions.test.ts b/src/main/runtime/app-lifecycle-actions.test.ts index 5ab6227c..e22f1774 100644 --- a/src/main/runtime/app-lifecycle-actions.test.ts +++ b/src/main/runtime/app-lifecycle-actions.test.ts @@ -46,12 +46,13 @@ test('on will quit cleanup handler runs all cleanup steps', () => { cleanupInternalSubtitleTrackCache: () => calls.push('cleanup-internal-subtitles'), cleanupYoutubeSubtitleTempDirs: () => calls.push('cleanup-youtube-subtitles'), cleanupYoutubeMediaCache: () => calls.push('cleanup-youtube-media'), + cleanupRemoteMediaWindows: () => calls.push('cleanup-remote-media-windows'), cleanupJellyfinSubtitleCache: () => calls.push('cleanup-jellyfin-subtitles'), stopDiscordPresenceService: () => calls.push('stop-discord-presence'), }); cleanup(); - assert.equal(calls.length, 35); + assert.equal(calls.length, 36); assert.equal(calls[0], 'destroy-tray'); assert.equal(calls[calls.length - 1], 'stop-discord-presence'); assert.ok(calls.includes('cleanup-jellyfin-subtitles')); @@ -60,6 +61,7 @@ test('on will quit cleanup handler runs all cleanup steps', () => { assert.ok(calls.includes('clear-linux-mpv-fullscreen-overlay-refresh-timeouts')); assert.ok(calls.includes('cleanup-youtube-subtitles')); assert.ok(calls.includes('cleanup-youtube-media')); + assert.ok(calls.includes('cleanup-remote-media-windows')); assert.ok(calls.indexOf('flush-mpv-log') < calls.indexOf('destroy-socket')); }); @@ -102,6 +104,7 @@ test('on will quit cleanup handler cleans jellyfin subtitle cache when stopping cleanupInternalSubtitleTrackCache: () => calls.push('cleanup-internal-subtitles'), cleanupYoutubeSubtitleTempDirs: () => calls.push('cleanup-youtube-subtitles'), cleanupYoutubeMediaCache: () => calls.push('cleanup-youtube-media'), + cleanupRemoteMediaWindows: () => calls.push('cleanup-remote-media-windows'), cleanupJellyfinSubtitleCache: () => calls.push('cleanup-jellyfin-subtitles'), stopDiscordPresenceService: () => calls.push('stop-discord-presence'), }); diff --git a/src/main/runtime/app-lifecycle-actions.ts b/src/main/runtime/app-lifecycle-actions.ts index c0b77a0a..5a6b20f9 100644 --- a/src/main/runtime/app-lifecycle-actions.ts +++ b/src/main/runtime/app-lifecycle-actions.ts @@ -32,6 +32,7 @@ export function createOnWillQuitCleanupHandler(deps: { cleanupInternalSubtitleTrackCache: () => void; cleanupYoutubeSubtitleTempDirs: () => void; cleanupYoutubeMediaCache: () => void; + cleanupRemoteMediaWindows: () => void; cleanupJellyfinSubtitleCache: () => void; stopDiscordPresenceService: () => void; }) { @@ -76,6 +77,7 @@ export function createOnWillQuitCleanupHandler(deps: { } deps.cleanupYoutubeSubtitleTempDirs(); deps.cleanupYoutubeMediaCache(); + deps.cleanupRemoteMediaWindows(); deps.stopDiscordPresenceService(); return Promise.resolve(stopSyncAutoScheduler); }; diff --git a/src/main/runtime/app-lifecycle-main-cleanup.test.ts b/src/main/runtime/app-lifecycle-main-cleanup.test.ts index f12df1e7..ffe211a0 100644 --- a/src/main/runtime/app-lifecycle-main-cleanup.test.ts +++ b/src/main/runtime/app-lifecycle-main-cleanup.test.ts @@ -75,6 +75,7 @@ test('cleanup deps builder returns handlers that guard optional runtime objects' cleanupInternalSubtitleTrackCache: () => calls.push('cleanup-internal-subtitles'), cleanupYoutubeSubtitleTempDirs: () => calls.push('cleanup-youtube-subtitles'), cleanupYoutubeMediaCache: () => calls.push('cleanup-youtube-media'), + cleanupRemoteMediaWindows: () => calls.push('cleanup-remote-media-windows'), cleanupJellyfinSubtitleCache: () => calls.push('cleanup-jellyfin-subtitles'), stopDiscordPresenceService: () => calls.push('stop-discord-presence'), }); @@ -157,6 +158,7 @@ test('cleanup deps builder skips destroyed yomitan window', () => { cleanupInternalSubtitleTrackCache: () => {}, cleanupYoutubeSubtitleTempDirs: () => {}, cleanupYoutubeMediaCache: () => {}, + cleanupRemoteMediaWindows: () => {}, cleanupJellyfinSubtitleCache: () => {}, stopDiscordPresenceService: () => {}, }); @@ -210,6 +212,7 @@ test('cleanup deps builder skips global shortcut cleanup before app ready', () = cleanupInternalSubtitleTrackCache: () => {}, cleanupYoutubeSubtitleTempDirs: () => {}, cleanupYoutubeMediaCache: () => {}, + cleanupRemoteMediaWindows: () => {}, cleanupJellyfinSubtitleCache: () => {}, stopDiscordPresenceService: () => {}, }); diff --git a/src/main/runtime/app-lifecycle-main-cleanup.ts b/src/main/runtime/app-lifecycle-main-cleanup.ts index 249d1618..9a9d4833 100644 --- a/src/main/runtime/app-lifecycle-main-cleanup.ts +++ b/src/main/runtime/app-lifecycle-main-cleanup.ts @@ -61,6 +61,7 @@ export function createBuildOnWillQuitCleanupDepsHandler(deps: { cleanupInternalSubtitleTrackCache: () => void; cleanupYoutubeSubtitleTempDirs: () => void; cleanupYoutubeMediaCache: () => void; + cleanupRemoteMediaWindows: () => void; cleanupJellyfinSubtitleCache: () => void; stopDiscordPresenceService: () => void; }) { @@ -148,6 +149,7 @@ export function createBuildOnWillQuitCleanupDepsHandler(deps: { cleanupInternalSubtitleTrackCache: () => deps.cleanupInternalSubtitleTrackCache(), cleanupYoutubeSubtitleTempDirs: () => deps.cleanupYoutubeSubtitleTempDirs(), cleanupYoutubeMediaCache: () => deps.cleanupYoutubeMediaCache(), + cleanupRemoteMediaWindows: () => deps.cleanupRemoteMediaWindows(), cleanupJellyfinSubtitleCache: () => deps.cleanupJellyfinSubtitleCache(), stopDiscordPresenceService: () => deps.stopDiscordPresenceService(), }); diff --git a/src/main/runtime/composers/startup-lifecycle-composer.test.ts b/src/main/runtime/composers/startup-lifecycle-composer.test.ts index fbe18443..6c31e386 100644 --- a/src/main/runtime/composers/startup-lifecycle-composer.test.ts +++ b/src/main/runtime/composers/startup-lifecycle-composer.test.ts @@ -52,6 +52,7 @@ test('composeStartupLifecycleHandlers returns callable startup lifecycle handler cleanupInternalSubtitleTrackCache: () => {}, cleanupYoutubeSubtitleTempDirs: () => {}, cleanupYoutubeMediaCache: () => {}, + cleanupRemoteMediaWindows: () => {}, cleanupJellyfinSubtitleCache: () => {}, stopDiscordPresenceService: () => {}, }, diff --git a/src/main/runtime/media-timing-review.test.ts b/src/main/runtime/media-timing-review.test.ts index 5594328a..d08c0172 100644 --- a/src/main/runtime/media-timing-review.test.ts +++ b/src/main/runtime/media-timing-review.test.ts @@ -1,6 +1,15 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; import type { MediaTimingReviewOpenPayload } from '../../types/anki'; +import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform'; +import type { + RemoteMediaWindow, + RemoteMediaWindowRange, + RemoteMediaWindowSource, +} from '../../core/services/remote-media-window-cache'; +import type { MediaTimingPreviewSession } from '../../core/services/media-timing-preview'; + +type MediaTimingPreviewSessionLike = Pick; import { buildMediaTimingReviewPayload, collectMediaTimingContextLines, @@ -177,12 +186,7 @@ test('media timing review pauses playback, resolves exact timing, and restores p }); test('media timing review analyzes the visible range on the selected audio stream', async () => { - const waveformCalls: Array<{ - mediaPath: string; - startTime: number; - endTime: number; - audioStreamIndex?: number; - }> = []; + const waveformCalls: SpeechWaveformOptions[] = []; let runtime: ReturnType; runtime = createMediaTimingReviewRuntime({ getMpvClient: () => ({ @@ -239,6 +243,300 @@ test('media timing review analyzes the visible range on the selected audio strea ]); }); +const REMOTE_STREAM_URL = 'https://jellyfin.example/Videos/abc/stream?static=true'; + +function createWindowStub(options: { fail?: boolean } = {}) { + const calls: Array<{ source: RemoteMediaWindowSource; range: RemoteMediaWindowRange }> = []; + const acquireMediaWindow = async ( + source: RemoteMediaWindowSource, + range: RemoteMediaWindowRange, + ): Promise => { + calls.push({ source, range }); + if (options.fail) throw new Error('offline'); + const windowPath = `/tmp/window-${range.startTime}-${range.endTime}.mkv`; + return { + path: windowPath, + startTime: range.startTime, + endTime: range.endTime, + sourcePath: source.path, + audioStreamIndex: source.audioStreamIndex ?? null, + media: { + path: windowPath, + source: 'remote-window', + singleResolvedStream: true, + absoluteTimestamps: true, + }, + }; + }; + return { calls, acquireMediaWindow }; +} + +function createRemoteReviewRuntime(options: { + windowStub: ReturnType; + waveformCalls: SpeechWaveformOptions[]; + previewStarts: Array[0]>; + previewPlays: Array<[string, number, number]>; + disposed: string[]; + openModal: ( + runtime: ReturnType, + payload: MediaTimingReviewOpenPayload, + ) => Promise; +}) { + let runtime!: ReturnType; + runtime = createMediaTimingReviewRuntime({ + getMpvClient: () => ({ + connected: true, + currentVideoPath: REMOTE_STREAM_URL, + currentAudioStreamIndex: 2, + requestProperty: async (name) => + ({ pause: true, duration: 100, aid: 3, volume: 60 })[ + name as 'pause' | 'duration' | 'aid' | 'volume' + ] ?? null, + send: () => undefined, + }), + getCurrentMediaPath: () => REMOTE_STREAM_URL, + getMpvExecutablePath: () => 'mpv', + resolveMediaSource: async () => ({ + path: REMOTE_STREAM_URL, + inputOptions: { reconnect: true }, + }), + acquireMediaWindow: options.windowStub.acquireMediaWindow, + generateWaveform: async (waveformOptions) => { + options.waveformCalls.push(waveformOptions); + return [0.1, 0.8, 0.2]; + }, + createPreviewSession: () => { + let mediaPath = ''; + return { + start: async (startOptions) => { + mediaPath = startOptions.mediaPath; + options.previewStarts.push(startOptions); + }, + play: async (startTime, endTime) => { + options.previewPlays.push([mediaPath, startTime, endTime]); + }, + stop: async () => undefined, + dispose: () => { + options.disposed.push(mediaPath); + }, + }; + }, + openModal: async (payload) => { + await options.openModal(runtime, payload); + return true; + }, + showStatus: () => undefined, + }); + return runtime; +} + +test('media timing review downloads one window of a remote stream for the waveform and preview', async () => { + const windowStub = createWindowStub(); + const waveformCalls: SpeechWaveformOptions[] = []; + const previewStarts: Array[0]> = []; + const previewPlays: Array<[string, number, number]> = []; + const disposed: string[] = []; + const runtime = createRemoteReviewRuntime({ + windowStub, + waveformCalls, + previewStarts, + previewPlays, + disposed, + openModal: async (active, payload) => { + const waveform = await active.getWaveform({ + reviewId: payload.reviewId, + startTime: payload.timelineStartTime, + endTime: payload.timelineEndTime, + }); + assert.deepEqual(waveform, { ok: true, peaks: [0.1, 0.8, 0.2] }); + assert.deepEqual( + await active.previewRange({ reviewId: payload.reviewId, startTime: 9.5, endTime: 12.5 }), + { ok: true }, + ); + active.resolveReview({ + reviewId: payload.reviewId, + decision: { action: 'confirm', startTime: 9.5, endTime: 12.5 }, + }); + }, + }); + + const decision = await runtime.requestReview({ + kind: 'word', + text: '字幕', + startTime: 10, + endTime: 12, + audioPadding: 0.5, + maxMediaDuration: 30, + }); + + assert.deepEqual(decision, { action: 'confirm', startTime: 9.5, endTime: 12.5 }); + assert.deepEqual(windowStub.calls, [ + { + source: { path: REMOTE_STREAM_URL, inputOptions: { reconnect: true }, audioStreamIndex: 2 }, + range: { startTime: 7.5, endTime: 14.5 }, + }, + ]); + assert.deepEqual(waveformCalls, [ + { + mediaPath: { + path: '/tmp/window-7.5-14.5.mkv', + source: 'remote-window', + singleResolvedStream: true, + absoluteTimestamps: true, + }, + startTime: 7.5, + endTime: 14.5, + }, + ]); + assert.deepEqual(previewStarts, [ + { + mediaPath: '/tmp/window-7.5-14.5.mkv', + executablePath: 'mpv', + volume: 60, + absoluteTimestamps: true, + }, + ]); + assert.deepEqual(previewPlays, [['/tmp/window-7.5-14.5.mkv', 9.5, 12.5]]); + assert.deepEqual(disposed, ['/tmp/window-7.5-14.5.mkv']); +}); + +test('media timing review restarts the preview on a wider window when the timeline grows', async () => { + const windowStub = createWindowStub(); + const waveformCalls: SpeechWaveformOptions[] = []; + const previewStarts: Array[0]> = []; + const previewPlays: Array<[string, number, number]> = []; + const disposed: string[] = []; + const runtime = createRemoteReviewRuntime({ + windowStub, + waveformCalls, + previewStarts, + previewPlays, + disposed, + openModal: async (active, payload) => { + await active.previewRange({ reviewId: payload.reviewId, startTime: 9.5, endTime: 12.5 }); + // The user revealed two more seconds before the clip. + await active.getWaveform({ reviewId: payload.reviewId, startTime: 5.5, endTime: 14.5 }); + await active.previewRange({ reviewId: payload.reviewId, startTime: 6, endTime: 12.5 }); + active.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } }); + }, + }); + + await runtime.requestReview({ + kind: 'sentence', + text: '字幕', + startTime: 10, + endTime: 12, + audioPadding: 0.5, + maxMediaDuration: 30, + }); + + assert.deepEqual( + windowStub.calls.map((call) => call.range), + [ + { startTime: 7.5, endTime: 14.5 }, + { startTime: 5.5, endTime: 14.5 }, + ], + ); + assert.deepEqual( + previewStarts.map((start) => start.mediaPath), + ['/tmp/window-7.5-14.5.mkv', '/tmp/window-5.5-14.5.mkv'], + ); + assert.deepEqual(previewPlays, [ + ['/tmp/window-7.5-14.5.mkv', 9.5, 12.5], + ['/tmp/window-5.5-14.5.mkv', 6, 12.5], + ]); + assert.deepEqual(disposed, ['/tmp/window-7.5-14.5.mkv', '/tmp/window-5.5-14.5.mkv']); + assert.equal(waveformCalls[0]?.startTime, 5.5); +}); + +test('media timing review falls back to the remote stream after one failed window download', async () => { + const windowStub = createWindowStub({ fail: true }); + const waveformCalls: SpeechWaveformOptions[] = []; + const previewStarts: Array[0]> = []; + const previewPlays: Array<[string, number, number]> = []; + const disposed: string[] = []; + const runtime = createRemoteReviewRuntime({ + windowStub, + waveformCalls, + previewStarts, + previewPlays, + disposed, + openModal: async (active, payload) => { + await active.getWaveform({ + reviewId: payload.reviewId, + startTime: payload.timelineStartTime, + endTime: payload.timelineEndTime, + }); + await active.previewRange({ reviewId: payload.reviewId, startTime: 9.5, endTime: 12.5 }); + active.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } }); + }, + }); + + await runtime.requestReview({ + kind: 'word', + text: '字幕', + startTime: 10, + endTime: 12, + audioPadding: 0.5, + maxMediaDuration: 30, + }); + + assert.equal(windowStub.calls.length, 1); + assert.deepEqual(waveformCalls, [ + { + mediaPath: { path: REMOTE_STREAM_URL, inputOptions: { reconnect: true } }, + startTime: 7.5, + endTime: 14.5, + audioStreamIndex: 2, + }, + ]); + assert.deepEqual(previewStarts, [ + { mediaPath: REMOTE_STREAM_URL, executablePath: 'mpv', volume: 60, audioTrackId: 3 }, + ]); + assert.deepEqual(previewPlays, [[REMOTE_STREAM_URL, 9.5, 12.5]]); +}); + +test('media timing review never downloads windows for local media', async () => { + const windowStub = createWindowStub(); + let runtime!: ReturnType; + runtime = createMediaTimingReviewRuntime({ + getMpvClient: () => ({ + connected: true, + currentVideoPath: '/video/show.mkv', + requestProperty: async (name) => (name === 'duration' ? 100 : name === 'pause' ? true : null), + send: () => undefined, + }), + getCurrentMediaPath: () => '/video/show.mkv', + getMpvExecutablePath: () => 'mpv', + resolveMediaSource: async () => ({ path: '/video/show.mkv' }), + acquireMediaWindow: windowStub.acquireMediaWindow, + generateWaveform: async () => [0.1, 0.8, 0.2], + createPreviewSession: () => ({ + start: async () => undefined, + play: async () => undefined, + stop: async () => undefined, + dispose: () => undefined, + }), + openModal: async (payload) => { + await runtime.getWaveform({ reviewId: payload.reviewId, startTime: 7.5, endTime: 14.5 }); + runtime.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } }); + return true; + }, + showStatus: () => undefined, + }); + + await runtime.requestReview({ + kind: 'sentence', + text: '字幕', + startTime: 10, + endTime: 12, + audioPadding: 0.5, + maxMediaDuration: 30, + }); + + assert.equal(windowStub.calls.length, 0); +}); + test('media timing review rejects stale and out-of-range actions before allowing discard', async () => { const { runtime, payload, pendingDecision, previewCalls } = await startActiveMediaTimingReview({ maxMediaDuration: 3, @@ -384,11 +682,16 @@ test('media timing review restores playback when setup fails after pausing', asy send: ({ command }) => commands.push(command), }), getCurrentMediaPath: () => '/video/show.mkv', - getMpvExecutablePath: () => 'mpv', - generateWaveform: async () => [], - createPreviewSession: () => { + getMpvExecutablePath: () => { throw new Error('preview setup failed'); }, + generateWaveform: async () => [], + createPreviewSession: () => ({ + start: async () => undefined, + play: async () => undefined, + stop: async () => undefined, + dispose: () => undefined, + }), openModal: async () => true, showStatus: () => undefined, }); diff --git a/src/main/runtime/media-timing-review.ts b/src/main/runtime/media-timing-review.ts index a3eb96c6..3d052623 100644 --- a/src/main/runtime/media-timing-review.ts +++ b/src/main/runtime/media-timing-review.ts @@ -11,6 +11,13 @@ import type { MediaTimingReviewWaveformResult, } from '../../types/anki'; import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform'; +import { + isRemoteMediaWindowSourcePath, + type RemoteMediaWindow, + type RemoteMediaWindowRange, + type RemoteMediaWindowSource, +} from '../../core/services/remote-media-window-cache'; +import type { MediaInput, MediaInputOptions } from '../../media-input'; const INITIAL_TIMELINE_MARGIN_SECONDS = 2; const REVIEW_DECISION_TIMEOUT_MS = 5 * 60_000; @@ -31,19 +38,36 @@ interface PreviewSession { executablePath?: string; audioTrackId?: number; volume?: number; + absoluteTimestamps?: boolean; }): Promise; play(startTime: number, endTime: number): Promise; stop(): Promise; dispose(): void; } +interface ReviewMediaSource { + path: string; + inputOptions?: MediaInputOptions; + singleResolvedStream?: boolean; +} + interface ActiveReview { payload: MediaTimingReviewOpenPayload; + /** What the hidden mpv preview plays when no cached window is available. */ mediaPath: string; + /** What the waveform reads when no cached window is available. */ + waveformMedia: MediaInput; audioStreamIndex?: number; + /** Remote source to download windows of; null for local media or without a cache. */ + windowSource: RemoteMediaWindowSource | null; + /** Latest window returned for this review; reused while it still covers the request. */ + window: RemoteMediaWindow | null; + windowRequest: (RemoteMediaWindowRange & { promise: Promise }) | null; + windowFailed: boolean; + previewOptions: { executablePath?: string; audioTrackId?: number; volume?: number }; + preview: { path: string; session: Promise } | null; mpvClient: ReviewMpvClient; restorePlayback: boolean; - preview: Promise; resolve: (decision: MediaTimingReviewDecision) => void; } @@ -53,6 +77,13 @@ export interface MediaTimingReviewRuntimeDeps { getMpvExecutablePath: () => string; createPreviewSession: () => PreviewSession; generateWaveform: (options: SpeechWaveformOptions) => Promise; + /** Resolves the FFmpeg-readable stream URL and headers behind the current media path. */ + resolveMediaSource?: () => Promise; + /** Downloads (or reuses) a local window of a remote source covering the range. */ + acquireMediaWindow?: ( + source: RemoteMediaWindowSource, + range: RemoteMediaWindowRange, + ) => Promise; getSubtitleContextLines?: (range: { startTime: number; endTime: number }) => { previous: MediaTimingReviewContextLine[]; next: MediaTimingReviewContextLine[]; @@ -199,6 +230,84 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep } } + function ensureWindow( + review: ActiveReview, + range: RemoteMediaWindowRange, + ): Promise { + const { windowSource } = review; + if (!windowSource || review.windowFailed || !deps.acquireMediaWindow) { + return Promise.resolve(null); + } + const coversRange = (candidate: RemoteMediaWindowRange): boolean => + candidate.startTime <= range.startTime && candidate.endTime >= range.endTime; + if (review.window && coversRange(review.window)) return Promise.resolve(review.window); + const inFlight = review.windowRequest; + if (inFlight && coversRange(inFlight)) return inFlight.promise; + + const request = { + startTime: range.startTime, + endTime: range.endTime, + promise: Promise.resolve(null), + }; + request.promise = deps + .acquireMediaWindow(windowSource, { startTime: range.startTime, endTime: range.endTime }) + .then((window) => { + review.window = window; + return window; + }) + .catch(() => { + // Fall back to the remote source for the rest of this review instead of retrying. + review.windowFailed = true; + return null; + }) + .finally(() => { + if (review.windowRequest === request) review.windowRequest = null; + }); + review.windowRequest = request; + return request.promise; + } + + /** + * Returns the preview player for the range, restarting it when the range needs a + * different file (the first cached window, or a wider one after the timeline grew). + */ + async function previewFor( + review: ActiveReview, + range: RemoteMediaWindowRange, + ): Promise { + const window = await ensureWindow(review, range); + if (active !== review) { + // The review ended during the download; do not start a player nobody will dispose. + throw new Error('This timing review is no longer active.'); + } + const mediaPath = window?.path ?? review.mediaPath; + if (review.preview?.path === mediaPath) return review.preview.session; + + const previous = review.preview; + const session = deps.createPreviewSession(); + const { audioTrackId, ...previewOptions } = review.previewOptions; + const started = session + .start({ + mediaPath, + ...previewOptions, + // A cached window keeps one audio stream, so mpv's track id from the source no longer applies. + ...(window + ? { absoluteTimestamps: true } + : audioTrackId !== undefined + ? { audioTrackId } + : {}), + }) + .then(() => session) + .catch((error) => { + session.dispose(); + throw error; + }); + review.preview = { path: mediaPath, session: started }; + void started.catch(() => {}); + if (previous) void previous.session.then((old) => old.dispose()).catch(() => {}); + return started; + } + async function runReview(request: MediaTimingReviewRequest): Promise { const mpvClient = deps.getMpvClient(); const mediaPath = @@ -208,11 +317,12 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep return { action: 'use-original' }; } - const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw] = await Promise.all([ + const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw, resolvedSource] = await Promise.all([ mpvClient.requestProperty?.('pause').catch(() => null) ?? null, mpvClient.requestProperty?.('duration').catch(() => null) ?? null, mpvClient.requestProperty?.('aid').catch(() => null) ?? null, mpvClient.requestProperty?.('volume').catch(() => null) ?? null, + deps.resolveMediaSource?.().catch(() => null) ?? null, ]); const pauseState = booleanProperty(pauseRaw); mpvClient.send({ command: ['set_property', 'pause', 'yes'] }); @@ -232,38 +342,51 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep mediaDuration: finiteNumber(durationRaw) ?? undefined, ...(contextLines ? { contextLines } : {}), }); - const previewSession = deps.createPreviewSession(); - const preview = previewSession - .start({ - mediaPath, - executablePath: deps.getMpvExecutablePath(), - audioTrackId: finiteNumber(audioTrackRaw) ?? undefined, - volume: finiteNumber(volumeRaw) ?? undefined, - }) - .then(() => previewSession) - .catch((error) => { - previewSession.dispose(); - throw error; - }); - void preview.catch(() => {}); + const sourcePath = resolvedSource?.path.trim() || mediaPath; + const inputOptions = resolvedSource?.inputOptions; + const audioStreamIndex = + resolvedSource?.singleResolvedStream || mpvClient.currentAudioStreamIndex == null + ? undefined + : mpvClient.currentAudioStreamIndex; + const windowSource: RemoteMediaWindowSource | null = + deps.acquireMediaWindow && isRemoteMediaWindowSourcePath(sourcePath) + ? { + path: sourcePath, + ...(inputOptions ? { inputOptions } : {}), + audioStreamIndex: audioStreamIndex ?? null, + } + : null; let resolveDecision!: (decision: MediaTimingReviewDecision) => void; const decisionPromise = new Promise((resolve) => { resolveDecision = resolve; }); - active = { + const review: ActiveReview = { payload, mediaPath, - ...(mpvClient.currentAudioStreamIndex !== null && - mpvClient.currentAudioStreamIndex !== undefined - ? { audioStreamIndex: mpvClient.currentAudioStreamIndex } - : {}), + waveformMedia: inputOptions ? { path: sourcePath, inputOptions } : sourcePath, + ...(audioStreamIndex !== undefined ? { audioStreamIndex } : {}), + windowSource, + window: null, + windowRequest: null, + windowFailed: false, + previewOptions: { + executablePath: deps.getMpvExecutablePath(), + audioTrackId: finiteNumber(audioTrackRaw) ?? undefined, + volume: finiteNumber(volumeRaw) ?? undefined, + }, + preview: null, mpvClient, restorePlayback: pendingPauseRestore === mpvClient, - preview, resolve: resolveDecision, }; + active = review; pendingPauseRestore = null; + // Download the visible timeline once now; the waveform and preview both wait on it. + void previewFor(review, { + startTime: payload.timelineStartTime, + endTime: payload.timelineEndTime, + }).catch(() => {}); const opened = await deps.openModal(payload).catch(() => false); if (!opened) { @@ -317,7 +440,10 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep return { ok: false, message: 'The selected preview range is invalid.' }; } try { - const previewSession = await current.preview; + const previewSession = await previewFor(current, request); + if (active !== current) { + return { ok: false, message: 'This timing review is no longer active.' }; + } await previewSession.play(request.startTime, request.endTime); return { ok: true }; } catch (error) { @@ -347,11 +473,15 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep } try { + const window = await ensureWindow(current, request); + if (active !== current) { + return { ok: false, message: 'This timing review is no longer active.' }; + } const peaks = await deps.generateWaveform({ - mediaPath: current.mediaPath, + mediaPath: window?.media ?? current.waveformMedia, startTime: request.startTime, endTime: request.endTime, - ...(current.audioStreamIndex !== undefined + ...(!window && current.audioStreamIndex !== undefined ? { audioStreamIndex: current.audioStreamIndex } : {}), }); @@ -370,8 +500,8 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep return { ok: false, message: 'This timing review is no longer active.' }; } try { - const previewSession = await current.preview; - await previewSession.stop(); + const previewSession = current.preview ? await current.preview.session : null; + await previewSession?.stop(); return { ok: true }; } catch (error) { return { @@ -403,7 +533,7 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep const current = active; active = null; if (!current) return; - void current.preview.then((session) => session.dispose()).catch(() => {}); + void current.preview?.session.then((session) => session.dispose()).catch(() => {}); if (current.restorePlayback && current.mpvClient.connected) { current.mpvClient.send({ command: ['set_property', 'pause', 'no'] }); } diff --git a/src/media-generator.test.ts b/src/media-generator.test.ts index 68a3fa30..1e48fb38 100644 --- a/src/media-generator.test.ts +++ b/src/media-generator.test.ts @@ -10,6 +10,9 @@ import { MediaGenerator, type MediaGeneratorOptions, } from './media-generator'; +import { RemoteMediaWindowCache } from './core/services/remote-media-window-cache'; + +const REMOTE_STREAM_URL = 'https://jellyfin.example/Videos/abc/stream?static=true'; async function withStubbedFfmpeg( run: (generator: MediaGenerator, argsPath: string) => Promise, @@ -21,6 +24,7 @@ async function withStubbedFfmpeg( const root = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-media-generator-test-')); const binDir = path.join(root, 'bin'); const tempDir = path.join(root, 'media'); + const windowsDir = path.join(root, 'windows'); const argsPath = path.join(root, 'ffmpeg-args.txt'); fs.mkdirSync(binDir, { recursive: true }); const ffmpegStubPath = path.join(binDir, 'ffmpeg-stub.cjs'); @@ -34,7 +38,7 @@ async function withStubbedFfmpeg( " console.log(' V..... libaom-av1');", ' process.exit(0);', '}', - "fs.writeFileSync(process.env.SUBMINER_TEST_FFMPEG_ARGS, JSON.stringify(args), 'utf8');", + "fs.appendFileSync(process.env.SUBMINER_TEST_FFMPEG_ARGS, JSON.stringify(args) + '\\n', 'utf8');", 'const outputPath = args.at(-1);', "if (process.env.SUBMINER_TEST_FFMPEG_SKIP_OUTPUT !== '1') {", " fs.writeFileSync(outputPath, 'avif', 'utf8');", @@ -61,12 +65,18 @@ async function withStubbedFfmpeg( } else { delete process.env.SUBMINER_TEST_FFMPEG_SKIP_OUTPUT; } - const generator = new MediaGenerator(tempDir, options); + // Each test gets its own window cache so remote inputs never leak windows between tests. + const remoteMediaWindows = new RemoteMediaWindowCache({ tempDir: windowsDir, idleTtlMs: 0 }); + const generator = new MediaGenerator(tempDir, { + remoteMediaWindows, + ...options, + }); try { await run(generator, argsPath); } finally { generator.cleanup(); + remoteMediaWindows.cleanup(); process.env.PATH = originalPath; if (originalArgsPath === undefined) { delete process.env.SUBMINER_TEST_FFMPEG_ARGS; @@ -82,8 +92,17 @@ async function withStubbedFfmpeg( } } +function readAllFfmpegArgs(argsPath: string): string[][] { + return fs + .readFileSync(argsPath, 'utf8') + .split('\n') + .filter((line) => line.trim().length > 0) + .map((line) => JSON.parse(line) as string[]); +} + +/** Arguments of the most recent ffmpeg invocation. */ function readFfmpegArgs(argsPath: string): string[] { - return JSON.parse(fs.readFileSync(argsPath, 'utf8')) as string[]; + return readAllFfmpegArgs(argsPath).at(-1) ?? []; } test('buildAnimatedImageVideoFilter holds lead-in until the next frame after the audio boundary', () => { @@ -272,41 +291,131 @@ test('generateAudio recreates missing temp directory before invoking ffmpeg', as }); test('generateAudio adds remote input options before the ffmpeg input', async () => { - await withStubbedFfmpeg(async (generator, argsPath) => { - await generator.generateAudio( - { - path: 'https://rr1---sn.example.googlevideo.com/videoplayback?mime=audio%2Fwebm', - inputOptions: { - reconnect: true, - userAgent: 'Mozilla/5.0', - headers: { - Referer: 'https://www.youtube.com/', - Origin: 'https://www.youtube.com', + await withStubbedFfmpeg( + async (generator, argsPath) => { + await generator.generateAudio( + { + path: 'https://rr1---sn.example.googlevideo.com/videoplayback?mime=audio%2Fwebm', + inputOptions: { + reconnect: true, + userAgent: 'Mozilla/5.0', + headers: { + Referer: 'https://www.youtube.com/', + Origin: 'https://www.youtube.com', + }, }, }, - }, + 10, + 12, + ); + + const args = readFfmpegArgs(argsPath); + const inputIndex = args.indexOf('-i'); + assert.ok(inputIndex > 0); + assert.ok(args.indexOf('-reconnect') > -1); + assert.ok(args.indexOf('-reconnect') < inputIndex); + assert.equal(args[args.indexOf('-reconnect') + 1], '1'); + assert.equal(args[args.indexOf('-reconnect_streamed') + 1], '1'); + assert.equal(args[args.indexOf('-reconnect_on_network_error') + 1], '1'); + assert.equal(args[args.indexOf('-reconnect_on_http_error') + 1], '403,5xx'); + assert.equal(args[args.indexOf('-reconnect_delay_max') + 1], '5'); + assert.equal(args[args.indexOf('-user_agent') + 1], 'Mozilla/5.0'); + assert.equal( + args[args.indexOf('-headers') + 1], + 'Referer: https://www.youtube.com/\r\nOrigin: https://www.youtube.com\r\n', + ); + }, + { remoteMediaWindows: null }, + ); +}); + +test('generateAudio downloads a remote window once and extracts from it with absolute seeks', async () => { + await withStubbedFfmpeg(async (generator, argsPath) => { + await generator.generateAudio( + { path: REMOTE_STREAM_URL, inputOptions: { reconnect: true } }, 10, 12, + 0.5, + 2, ); - const args = readFfmpegArgs(argsPath); - const inputIndex = args.indexOf('-i'); - assert.ok(inputIndex > 0); - assert.ok(args.indexOf('-reconnect') > -1); - assert.ok(args.indexOf('-reconnect') < inputIndex); - assert.equal(args[args.indexOf('-reconnect') + 1], '1'); - assert.equal(args[args.indexOf('-reconnect_streamed') + 1], '1'); - assert.equal(args[args.indexOf('-reconnect_on_network_error') + 1], '1'); - assert.equal(args[args.indexOf('-reconnect_on_http_error') + 1], '403,5xx'); - assert.equal(args[args.indexOf('-reconnect_delay_max') + 1], '5'); - assert.equal(args[args.indexOf('-user_agent') + 1], 'Mozilla/5.0'); - assert.equal( - args[args.indexOf('-headers') + 1], - 'Referer: https://www.youtube.com/\r\nOrigin: https://www.youtube.com\r\n', - ); + const calls = readAllFfmpegArgs(argsPath); + assert.equal(calls.length, 2); + const [fetchArgs, audioArgs] = calls as [string[], string[]]; + assert.equal(fetchArgs[fetchArgs.indexOf('-i') + 1], REMOTE_STREAM_URL); + assert.ok(fetchArgs.indexOf('-reconnect') < fetchArgs.indexOf('-i')); + assert.equal(fetchArgs[fetchArgs.indexOf('-ss') + 1], '9.25'); + assert.equal(fetchArgs[fetchArgs.lastIndexOf('-map') + 1], '0:2'); + assert.ok(fetchArgs.includes('-copyts')); + + const windowPath = audioArgs[audioArgs.indexOf('-i') + 1]; + assert.ok(windowPath?.endsWith('.mkv')); + assert.notEqual(windowPath, REMOTE_STREAM_URL); + assert.equal(audioArgs[audioArgs.indexOf('-ss') + 1], '9.5'); + assert.equal(audioArgs[audioArgs.indexOf('-seek_timestamp') + 1], '1'); + assert.ok(audioArgs.indexOf('-seek_timestamp') < audioArgs.indexOf('-i')); + assert.equal(audioArgs.includes('-reconnect'), false); + assert.equal(audioArgs.includes('-map'), false); + assert.equal(audioArgs.includes('-probesize'), false); + assert.ok(audioArgs.includes('loudnorm=I=-23:TP=-2:LRA=11')); }); }); +test('generateScreenshot reuses a downloaded window but never downloads one itself', async () => { + await withStubbedFfmpeg(async (generator, argsPath) => { + await generator.generateScreenshot(REMOTE_STREAM_URL, 11, { format: 'jpg' }); + let calls = readAllFfmpegArgs(argsPath); + assert.equal(calls.length, 1); + assert.equal(calls[0]![calls[0]!.indexOf('-i') + 1], REMOTE_STREAM_URL); + + await generator.generateAudio(REMOTE_STREAM_URL, 10, 12); + await generator.generateScreenshot(REMOTE_STREAM_URL, 11, { format: 'jpg' }); + await generator.generateScreenshot(REMOTE_STREAM_URL, 40, { format: 'jpg' }); + + calls = readAllFfmpegArgs(argsPath); + assert.equal(calls.length, 5); + const insideWindow = calls[3]!; + assert.ok(insideWindow[insideWindow.indexOf('-i') + 1]?.endsWith('.mkv')); + assert.equal(insideWindow[insideWindow.indexOf('-seek_timestamp') + 1], '1'); + const outsideWindow = calls[4]!; + assert.equal(outsideWindow[outsideWindow.indexOf('-i') + 1], REMOTE_STREAM_URL); + }); +}); + +test('generateAnimatedImage downloads the clip window before encoding', async () => { + await withStubbedFfmpeg(async (generator, argsPath) => { + await generator.generateAnimatedImage(REMOTE_STREAM_URL, 10, 12, 0, { fps: 10 }); + + const calls = readAllFfmpegArgs(argsPath).filter( + (args) => args[0] !== '-hide_banner' || args[1] !== '-encoders', + ); + assert.equal(calls.length, 2); + assert.equal(calls[0]![calls[0]!.indexOf('-i') + 1], REMOTE_STREAM_URL); + assert.ok(calls[1]![calls[1]!.indexOf('-i') + 1]?.endsWith('.mkv')); + assert.equal(calls[1]![calls[1]!.indexOf('-seek_timestamp') + 1], '1'); + }); +}); + +test('generateAudio reads the remote source directly when the window download fails', async () => { + await withStubbedFfmpeg( + async (generator, argsPath) => { + await generator.generateAudio(REMOTE_STREAM_URL, 10, 12); + + const args = readFfmpegArgs(argsPath); + assert.equal(args[args.indexOf('-i') + 1], REMOTE_STREAM_URL); + assert.equal(args.includes('-seek_timestamp'), false); + }, + { + remoteMediaWindows: new RemoteMediaWindowCache({ + execFile: (_file, _args, _options, callback) => + queueMicrotask(() => callback(Object.assign(new Error('offline'), { code: 1 }))), + idleTtlMs: 0, + logDebug: () => undefined, + }), + }, + ); +}); + test('generateAudio skips stale audio stream maps for single resolved streams', async () => { await withStubbedFfmpeg(async (generator, argsPath) => { await generator.generateAudio( @@ -320,8 +429,9 @@ test('generateAudio skips stale audio stream maps for single resolved streams', 22, ); - const args = readFfmpegArgs(argsPath); - assert.equal(args.includes('-map'), false); + const [fetchArgs, audioArgs] = readAllFfmpegArgs(argsPath) as [string[], string[]]; + assert.equal(fetchArgs[fetchArgs.lastIndexOf('-map') + 1], '0:a'); + assert.equal(audioArgs.includes('-map'), false); }); }); diff --git a/src/media-generator.ts b/src/media-generator.ts index 037ae50f..0db61d26 100644 --- a/src/media-generator.ts +++ b/src/media-generator.ts @@ -22,6 +22,12 @@ import * as path from 'path'; import * as os from 'os'; import { createLogger } from './logger'; import { normalizeMediaInput, type MediaInput } from './media-input'; +import { + getSharedRemoteMediaWindowCache, + isRemoteMediaWindowSourcePath, + type RemoteMediaWindowCache, + type RemoteMediaWindowRange, +} from './core/services/remote-media-window-cache'; const log = createLogger('media'); const AUDIO_NORMALIZATION_FILTER = 'loudnorm=I=-23:TP=-2:LRA=11'; @@ -86,6 +92,11 @@ export interface MediaGeneratorOptions { logDebug?: (message: string) => void; now?: () => number; execFile?: MediaGeneratorExecFile; + /** + * Local window cache for http(s) sources. Defaults to the process-wide cache shared + * with the timing review; pass `null` to always read remote sources directly. + */ + remoteMediaWindows?: RemoteMediaWindowCache | null; } function sanitizeDebugToken(value: string, fallback: string): string { @@ -232,6 +243,54 @@ export class MediaGenerator { }, delayMs); } + /** + * Swaps an http(s) input for the locally cached window that covers `range`, so the + * clip is downloaded once instead of per FFmpeg run. `acquire` downloads on a miss; + * `lookup` only reuses a window that another step already fetched. Any failure falls + * back to reading the remote source directly. + */ + private async resolveRemoteWindowInput( + input: MediaInput, + range: RemoteMediaWindowRange, + audioStreamIndex: number | null | undefined, + mode: 'acquire' | 'lookup', + ): Promise { + const cache = + this.options.remoteMediaWindows === undefined + ? getSharedRemoteMediaWindowCache() + : this.options.remoteMediaWindows; + const sourcePath = typeof input === 'string' ? input : input.path; + if (!cache || !isRemoteMediaWindowSourcePath(sourcePath)) { + return input; + } + const source = { + path: sourcePath, + ...(typeof input === 'object' && input.inputOptions + ? { inputOptions: input.inputOptions } + : {}), + audioStreamIndex: + typeof input === 'object' && input.singleResolvedStream ? null : (audioStreamIndex ?? null), + }; + const description = describeMediaInputForDebugLog(input); + try { + const window = + mode === 'acquire' ? await cache.acquire(source, range) : await cache.lookup(source, range); + if (!window) { + this.logMediaDebug(`window miss ${description} mode=${mode}`); + return input; + } + this.logMediaDebug( + `window hit ${description} mode=${mode} start=${window.startTime} end=${window.endTime}`, + ); + return window.media; + } catch (error) { + this.logMediaDebug( + `window failed ${description} mode=${mode} reason=${sanitizeDebugToken((error as Error).message, 'error')}`, + ); + return input; + } + } + private ffmpegError(label: string, error: ExecFileException): Error { if (error.code === 'ENOENT') { return new Error('FFmpeg not found. Install FFmpeg to enable media generation.'); @@ -281,7 +340,13 @@ export class MediaGenerator { const safePadding = Number.isFinite(padding) ? Math.max(0, padding) : 0; const start = Math.max(0, startTime - safePadding); const duration = endTime - start + safePadding; - const mediaInput = normalizeMediaInput(videoPath); + const sourceInput = await this.resolveRemoteWindowInput( + videoPath, + { startTime: start, endTime: start + duration }, + audioStreamIndex, + 'acquire', + ); + const mediaInput = normalizeMediaInput(sourceInput); const inputDescription = describeMediaInputForDebugLog(videoPath); const hasSelectedAudioStream = !mediaInput.singleResolvedStream && @@ -385,7 +450,15 @@ export class MediaGenerator { png: 'png', webp: 'webp', }; - const mediaInput = normalizeMediaInput(videoPath); + // A single frame is cheap to fetch remotely, so only reuse a window another step downloaded. + const mediaInput = normalizeMediaInput( + await this.resolveRemoteWindowInput( + videoPath, + { startTime: timestamp, endTime: timestamp }, + null, + 'lookup', + ), + ); const inputDescription = describeMediaInputForDebugLog(videoPath); const args: string[] = [ @@ -533,9 +606,17 @@ export class MediaGenerator { ); } + const mediaInput = normalizeMediaInput( + await this.resolveRemoteWindowInput( + videoPath, + { startTime: start, endTime: start + duration }, + null, + 'acquire', + ), + ); + return new Promise((resolve, reject) => { const outputPath = this.createTempOutputPath('animation', 'avif'); - const mediaInput = normalizeMediaInput(videoPath); const startedAt = this.nowMs(); const encoderArgs: string[] = ['-c:v', av1Encoder]; diff --git a/src/media-input.ts b/src/media-input.ts index 52349e19..c225354e 100644 --- a/src/media-input.ts +++ b/src/media-input.ts @@ -11,6 +11,12 @@ export type MediaInput = source?: string; inputOptions?: MediaInputOptions; singleResolvedStream?: boolean; + /** + * The file keeps the original media timestamps instead of starting at zero (a + * stream-copied window of a longer source). Seek with `-ss` against those + * absolute timestamps rather than relative to the file's own start time. + */ + absoluteTimestamps?: boolean; }; export type NormalizedMediaInput = { @@ -89,6 +95,10 @@ export function normalizeMediaInput(input: MediaInput): NormalizedMediaInput { inputArgs.push('-headers', headers); } + if (input.absoluteTimestamps) { + inputArgs.push('-seek_timestamp', '1'); + } + return { path: input.path, inputArgs,