diff --git a/changes/media-review-frame-picker-docs.md b/changes/media-review-frame-picker-docs.md new file mode 100644 index 00000000..da48ad59 --- /dev/null +++ b/changes/media-review-frame-picker-docs.md @@ -0,0 +1,4 @@ +type: docs +area: mining + +- Document independent screenshot selection in media timing review. diff --git a/changes/media-review-frame-picker.md b/changes/media-review-frame-picker.md new file mode 100644 index 00000000..f919dbde --- /dev/null +++ b/changes/media-review-frame-picker.md @@ -0,0 +1,4 @@ +type: added +area: mining + +- Choose a still screenshot independently of the audio in media timing review, with a live preview, a separate time slider, and decoded-frame stepping. Local video and seekable remote streams such as Jellyfin are supported. diff --git a/docs-site/anki-integration.md b/docs-site/anki-integration.md index 3a0814c3..235dd2d6 100644 --- a/docs-site/anki-integration.md +++ b/docs-site/anki-integration.md @@ -189,6 +189,10 @@ The review opens on the subtitle range plus your configured audio padding. Subti **Adjusting the clip.** Drag either edge to trim, drag the middle to slide the whole clip without changing its length, or click anywhere on the waveform to snap the nearer edge there. A focused edge also moves with the arrow keys: 100 ms per press, or 500 ms with Shift. The 100 ms buttons do the same thing. Earlier and Later each reveal two more seconds of timeline without moving the selection. +**Choosing the screenshot.** With still images enabled, drag the screenshot slider or use the Frame buttons to choose a video frame independently of the audio. Earlier and Later reveal more time for both sliders. The image follows the audio midpoint until you pick a frame, then stays fixed while you trim audio. Reset below the image restores automatic selection; the audio Reset affects only the audio. The screenshot slider also supports arrow keys, Home, and End. + +The picker supports local video and seekable remote streams, including Jellyfin, without seeking the main player. A selected frame must load before you can confirm; if it fails, choose another frame or Reset. The picker is hidden when image generation is disabled or animated AVIF is selected. Animated images continue to use the reviewed audio range. + **Keys.** Space previews the selection with a playhead sweeping the clip. The preview ends when the hidden player has actually played the last sample, so Bluetooth output latency does not clip the tail. Enter confirms and Escape cancels. **The waveform.** SubMiner reads a center channel when one carries dialogue and falls back to a mono mix otherwise, keeps only the 250 to 3500 Hz speech band, and draws each slice's loudness against the clip's own noise floor. Steady background music flattens out and dialogue stands up, which makes it much easier to tell adjacent lines apart. The mined subtitle appears as a tinted band with labeled line-start and line-end rails. If waveform analysis fails, the timing controls still work. diff --git a/src/anki-integration.ts b/src/anki-integration.ts index a1c47c0b..b2efb88c 100644 --- a/src/anki-integration.ts +++ b/src/anki-integration.ts @@ -1118,9 +1118,11 @@ export class AnkiIntegration { return null; } const mediaRange = this.getSubtitleMediaRange(context); - const timestamp = context - ? mediaRange.startTime + (mediaRange.endTime - mediaRange.startTime) / 2 - : this.mpvClient.currentTimePos || 0; + const timestamp = + context?.screenshotTime ?? + (context + ? mediaRange.startTime + (mediaRange.endTime - mediaRange.startTime) / 2 + : this.mpvClient.currentTimePos || 0); if (this.config.media?.imageType === 'avif') { return this.mediaGenerator.generateAnimatedImage( @@ -1797,6 +1799,8 @@ export class AnkiIntegration { ...request, audioPadding: Math.max(0, this.config.media.audioPadding ?? 0), maxMediaDuration: Math.max(0, this.config.media.maxMediaDuration ?? 30), + screenshotEnabled: + this.config.media.generateImage !== false && this.config.media.imageType !== 'avif', }); } diff --git a/src/anki-integration/card-creation.test.ts b/src/anki-integration/card-creation.test.ts index b1a93438..881ebe5f 100644 --- a/src/anki-integration/card-creation.test.ts +++ b/src/anki-integration/card-creation.test.ts @@ -290,6 +290,9 @@ test('CardCreationService keeps updating after recordCardsMinedCallback throws', }); test('CardCreationService uses stream-open-filename for remote media generation', async () => { + let reviewing = false; + const audioRanges: number[][] = []; + const imageTimes: number[] = []; const audioPaths: string[] = []; const imagePaths: string[] = []; const recordMediaPath = (mediaInput: MediaInput): string => @@ -319,6 +322,10 @@ test('CardCreationService uses stream-open-filename for remote media generation' behavior: {}, ai: false, }) as AnkiConnectConfig, + reviewMediaTiming: async () => + reviewing + ? { action: 'confirm', startTime: 0.2, endTime: 0.8, screenshotTime: 3.125 } + : { action: 'use-original' }, getAiConfig: () => ({}), getTimingTracker: () => ({}) as never, getMpvClient: () => @@ -349,16 +356,18 @@ test('CardCreationService uses stream-open-filename for remote media generation' ], updateNoteFields: async () => undefined, storeMediaFile: async () => undefined, - findNotes: async () => [], + findNotes: async () => [42], retrieveMediaFile: async () => '', deleteNotes: async () => undefined, }, mediaGenerator: { - generateAudio: async (path) => { + generateAudio: async (path, start, end, padding) => { + audioRanges.push([start, end, padding ?? -1]); audioPaths.push(recordMediaPath(path)); return Buffer.from('audio'); }, - generateScreenshot: async (path) => { + generateScreenshot: async (path, timestamp) => { + imageTimes.push(timestamp); imagePaths.push(recordMediaPath(path)); return Buffer.from('image'); }, @@ -406,6 +415,14 @@ test('CardCreationService uses stream-open-filename for remote media generation' assert.equal(created, true); assert.deepEqual(audioPaths, [audioUrl]); assert.deepEqual(imagePaths, [videoUrl]); + reviewing = true; + assert.equal(await service.createSentenceCard('テスト', 0, 1), true); + assert.deepEqual(audioRanges.at(-1), [0.2, 0.8, 0]); + assert.equal(imageTimes.at(-1), 3.125); + await service.markLastCardAsAudioCard(); + assert.equal(imageTimes.length, 3); + assert.deepEqual(audioRanges.at(-1), [0.2, 0.8, 0]); + assert.equal(imageTimes.at(-1), 3.125); }); test('CardCreationService does not use mpv stream indexes for ready cached YouTube media', async () => { diff --git a/src/anki-integration/card-creation.ts b/src/anki-integration/card-creation.ts index e15d1fcc..89913576 100644 --- a/src/anki-integration/card-creation.ts +++ b/src/anki-integration/card-creation.ts @@ -536,6 +536,7 @@ export class CardCreationService { endTime, animatedLeadInSeconds, exactReviewedRange, + timingDecision.action === 'confirm' ? timingDecision.screenshotTime : undefined, ); const imageField = this.deps.getConfig().fields?.image; @@ -796,6 +797,9 @@ export class CardCreationService { generateImage, volumeScale, ...(exactReviewedRange ? { mediaPaddingSeconds: 0 } : {}), + ...(timingDecision.action === 'confirm' && timingDecision.screenshotTime !== undefined + ? { screenshotTime: timingDecision.screenshotTime } + : {}), }); await this.deps.showNotification(noteId, label, 'media queued'); return true; @@ -840,6 +844,7 @@ export class CardCreationService { endTime, 0, exactReviewedRange, + timingDecision.action === 'confirm' ? timingDecision.screenshotTime : undefined, ); const imageField = config.fields?.image; @@ -922,15 +927,16 @@ export class CardCreationService { endTime: number, animatedLeadInSeconds = 0, exactReviewedRange = false, + screenshotTime?: number, ): Promise { const mpvClient = this.deps.getMpvClient(); if (!mpvClient) { return null; } - const timestamp = exactReviewedRange - ? startTime + (endTime - startTime) / 2 - : mpvClient.currentTimePos || 0; + const timestamp = + screenshotTime ?? + (exactReviewedRange ? startTime + (endTime - startTime) / 2 : mpvClient.currentTimePos || 0); if (this.deps.getConfig().media?.imageType === 'avif') { let imageStart = startTime; diff --git a/src/anki-integration/note-update-workflow.test.ts b/src/anki-integration/note-update-workflow.test.ts index 06694d2c..99019886 100644 --- a/src/anki-integration/note-update-workflow.test.ts +++ b/src/anki-integration/note-update-workflow.test.ts @@ -729,6 +729,7 @@ test('NoteUpdateWorkflow uses the combined review sentence for the card and medi startTime: 2, endTime: 7, text: 'previous-line current-line next-line', + screenshotTime: 8, }); harness.deps.generateAudio = async (context) => { audioContexts.push(context); @@ -745,6 +746,7 @@ test('NoteUpdateWorkflow uses the combined review sentence for the card and medi assert.equal(audioContexts[0]?.startTime, 2); assert.equal(audioContexts[0]?.endTime, 7); assert.equal(audioContexts[0]?.mediaPaddingSeconds, 0); + assert.equal(audioContexts[0]?.screenshotTime, 8); }); test('NoteUpdateWorkflow keeps cache unchanged and reports when deletion fails', async () => { diff --git a/src/anki-integration/note-update-workflow.ts b/src/anki-integration/note-update-workflow.ts index 4cc5e7ee..7d1468b1 100644 --- a/src/anki-integration/note-update-workflow.ts +++ b/src/anki-integration/note-update-workflow.ts @@ -253,6 +253,9 @@ export class NoteUpdateWorkflow { startTime: timingDecision.startTime, endTime: timingDecision.endTime, mediaPaddingSeconds: 0, + ...(timingDecision.screenshotTime !== undefined + ? { screenshotTime: timingDecision.screenshotTime } + : {}), }; } else if (timingDecision.action === 'skip-media') { skipMedia = true; diff --git a/src/anki-integration/pending-youtube-media-queue.test.ts b/src/anki-integration/pending-youtube-media-queue.test.ts index 85db1e5d..4c347063 100644 --- a/src/anki-integration/pending-youtube-media-queue.test.ts +++ b/src/anki-integration/pending-youtube-media-queue.test.ts @@ -51,6 +51,41 @@ function createDeps( return deps; } +test('queued media keeps a chosen screenshot separate from the reviewed audio range', async () => { + const screenshots: number[] = []; + const audioRanges: number[][] = []; + const deps = createDeps(); + deps.client.notesInfo = async () => [{ noteId: 42, fields: { Picture: { value: '' } } }]; + deps.mediaGenerator.generateScreenshot = async (_media, time) => { + screenshots.push(time); + return Buffer.from('image'); + }; + deps.mediaGenerator.generateAudio = async (_media, start, end, padding) => { + audioRanges.push([start, end, padding ?? -1]); + return Buffer.from('audio'); + }; + const queue = new PendingYoutubeMediaQueue(deps); + assert.equal( + await queue.queueFromNote({ + noteId: 42, + noteInfo: { noteId: 42, fields: {} }, + label: 'test', + context: { + source: 'overlay', + text: '字幕', + startTime: 1, + endTime: 2, + mediaPaddingSeconds: 0, + screenshotTime: 3.125, + }, + }), + true, + ); + await queue.handleReady('https://youtu.be/abc123', '/cache/video.mkv'); + assert.deepEqual(screenshots, [3.125]); + assert.deepEqual(audioRanges, [[1, 2, 0]]); +}); + test('PendingYoutubeMediaQueue treats cache lookup failures as an immediate generation fallback', async () => { const deps = createDeps({ getCachedMediaPath: async () => { diff --git a/src/anki-integration/pending-youtube-media-queue.ts b/src/anki-integration/pending-youtube-media-queue.ts index 1a3e659f..59802c6d 100644 --- a/src/anki-integration/pending-youtube-media-queue.ts +++ b/src/anki-integration/pending-youtube-media-queue.ts @@ -147,6 +147,9 @@ export class PendingYoutubeMediaQueue { generateAudio: shouldGenerateAudio(config), generateImage: shouldGenerateImage(config), volumeScale, + ...(job.context?.screenshotTime !== undefined + ? { screenshotTime: job.context.screenshotTime } + : {}), ...(job.context?.mediaPaddingSeconds !== undefined ? { mediaPaddingSeconds: job.context.mediaPaddingSeconds } : {}), @@ -320,6 +323,7 @@ export class PendingYoutubeMediaQueue { job.endTime, animatedLeadInSeconds, job.mediaPaddingSeconds, + job.screenshotTime, ); if (imageBuffer) { await this.deps.client.storeMediaFile(imageFilename, imageBuffer); @@ -381,6 +385,7 @@ export class PendingYoutubeMediaQueue { endTime: number, animatedLeadInSeconds = 0, mediaPaddingSeconds?: number, + screenshotTime?: number, ): Promise { const config = this.deps.getConfig(); if (config.media?.imageType === 'avif') { @@ -399,7 +404,7 @@ export class PendingYoutubeMediaQueue { ); } - const timestamp = startTime + (endTime - startTime) / 2; + const timestamp = screenshotTime ?? startTime + (endTime - startTime) / 2; return this.deps.mediaGenerator.generateScreenshot(videoPath, timestamp, { format: config.media?.imageFormat as 'jpg' | 'png' | 'webp', quality: config.media?.imageQuality, diff --git a/src/anki-integration/pending-youtube-media.ts b/src/anki-integration/pending-youtube-media.ts index 4e0abfb0..dec424bf 100644 --- a/src/anki-integration/pending-youtube-media.ts +++ b/src/anki-integration/pending-youtube-media.ts @@ -11,6 +11,7 @@ export interface PendingYoutubeMediaUpdate { generateImage: boolean; volumeScale?: number; mediaPaddingSeconds?: number; + screenshotTime?: number; } function trimToNonEmptyString(value: unknown): string | null { diff --git a/src/core/services/ipc.test.ts b/src/core/services/ipc.test.ts index 8c2ddeeb..91f16769 100644 --- a/src/core/services/ipc.test.ts +++ b/src/core/services/ipc.test.ts @@ -671,7 +671,34 @@ test('registerIpcHandlers accepts the keep-without-media timing decision', async assert.deepEqual(requests, [{ reviewId: 'review-1', decision: { action: 'skip-media' } }]); }); -test('registerIpcHandlers validates and forwards combined timing review text', async () => { +test('frame IPC validates timestamps and directions', async () => { + const { registrar, handlers } = createFakeIpcRegistrar(); + const requests: unknown[] = []; + registerIpcHandlers( + createRegisterIpcDeps({ + getMediaTimingReviewFrame: async (request) => { + requests.push(request); + return { ok: true }; + }, + }), + registrar, + ); + const frame = handlers.handle.get(IPC_CHANNELS.request.mediaTimingReviewFrame)!; + const valid = { reviewId: 'r', timestamp: 13, direction: 1 }; + assert.deepEqual(await frame({}, valid), { ok: true }); + for (const invalid of [ + null, + {}, + { ...valid, timestamp: NaN }, + { ...valid, timestamp: '13' }, + { ...valid, direction: 2 }, + ]) { + assert.equal(((await frame({}, invalid)) as { ok: boolean }).ok, false); + } + assert.deepEqual(requests, [valid]); +}); + +test('registerIpcHandlers validates and forwards timing review text and screenshot selection', async () => { const { registrar, handlers } = createFakeIpcRegistrar(); const requests: unknown[] = []; registerIpcHandlers( @@ -696,6 +723,7 @@ test('registerIpcHandlers validates and forwards combined timing review text', a startTime: 10, endTime: 12, text: '前の行 対象の行', + screenshotTime: 13, }, }, ), @@ -709,20 +737,23 @@ test('registerIpcHandlers validates and forwards combined timing review text', a startTime: 10, endTime: 12, text: '前の行 対象の行', + screenshotTime: 13, }, }, ]); - assert.deepEqual( - await handler!( - {}, - { - reviewId: 'review-1', - decision: { action: 'confirm', startTime: 10, endTime: 12, text: ' ' }, - }, - ), - { ok: false, message: 'Timing review is unavailable.' }, - ); + for (const invalid of [{ text: ' ' }, { screenshotTime: Infinity }]) { + assert.deepEqual( + await handler!( + {}, + { + reviewId: 'review-1', + decision: { action: 'confirm', startTime: 10, endTime: 12, ...invalid }, + }, + ), + { ok: false, message: 'Timing review is unavailable.' }, + ); + } assert.equal(requests.length, 1); }); diff --git a/src/core/services/ipc.ts b/src/core/services/ipc.ts index 2a26dac1..ba77264f 100644 --- a/src/core/services/ipc.ts +++ b/src/core/services/ipc.ts @@ -24,6 +24,8 @@ import type { MediaTimingReviewActionResult, MediaTimingReviewPreviewRequest, MediaTimingReviewResolveRequest, + MediaTimingReviewFrameRequest, + MediaTimingReviewFrameResult, MediaTimingReviewWaveformRequest, MediaTimingReviewWaveformResult, } from '../../types/anki'; @@ -111,6 +113,9 @@ export interface IpcServiceDeps { previewMediaTimingReview?: ( request: MediaTimingReviewPreviewRequest, ) => Promise; + getMediaTimingReviewFrame?: ( + request: MediaTimingReviewFrameRequest, + ) => Promise; getMediaTimingReviewWaveform?: ( request: MediaTimingReviewWaveformRequest, ) => Promise; @@ -269,6 +274,26 @@ function parseMediaTimingReviewWaveformRequest( return parseMediaTimingReviewPreviewRequest(payload); } +function parseMediaTimingReviewFrameRequest( + payload: unknown, +): MediaTimingReviewFrameRequest | null { + if (!payload || typeof payload !== 'object') return null; + const record = payload as Record; + if ( + typeof record.reviewId !== 'string' || + !record.reviewId || + typeof record.timestamp !== 'number' || + !Number.isFinite(record.timestamp) || + (record.direction !== undefined && record.direction !== -1 && record.direction !== 1) + ) + return null; + return { + reviewId: record.reviewId, + timestamp: record.timestamp, + ...(record.direction !== undefined ? { direction: record.direction as -1 | 1 } : {}), + }; +} + function parseMediaTimingReviewResolveRequest( payload: unknown, ): MediaTimingReviewResolveRequest | null { @@ -291,6 +316,9 @@ function parseMediaTimingReviewResolveRequest( Number.isFinite(decisionRecord.startTime) && typeof decisionRecord.endTime === 'number' && Number.isFinite(decisionRecord.endTime) && + (decisionRecord.screenshotTime === undefined || + (typeof decisionRecord.screenshotTime === 'number' && + Number.isFinite(decisionRecord.screenshotTime))) && (decisionRecord.text === undefined || (typeof decisionRecord.text === 'string' && decisionRecord.text.trim().length > 0)) ) { @@ -300,6 +328,9 @@ function parseMediaTimingReviewResolveRequest( action: 'confirm', startTime: decisionRecord.startTime, endTime: decisionRecord.endTime, + ...(decisionRecord.screenshotTime === undefined + ? {} + : { screenshotTime: decisionRecord.screenshotTime as number }), ...(decisionRecord.text === undefined ? {} : { text: decisionRecord.text }), }, }; @@ -365,6 +396,7 @@ export interface IpcDepsRuntimeOptions { request: YoutubePickerResolveRequest, ) => Promise; previewMediaTimingReview?: IpcServiceDeps['previewMediaTimingReview']; + getMediaTimingReviewFrame?: IpcServiceDeps['getMediaTimingReviewFrame']; getMediaTimingReviewWaveform?: IpcServiceDeps['getMediaTimingReviewWaveform']; stopMediaTimingReviewPreview?: IpcServiceDeps['stopMediaTimingReviewPreview']; resolveMediaTimingReview?: IpcServiceDeps['resolveMediaTimingReview']; @@ -463,6 +495,7 @@ export function createIpcDepsRuntime(options: IpcDepsRuntimeOptions): IpcService runSubsyncManual: options.runSubsyncManual, onYoutubePickerResolve: options.onYoutubePickerResolve, previewMediaTimingReview: options.previewMediaTimingReview, + getMediaTimingReviewFrame: options.getMediaTimingReviewFrame, getMediaTimingReviewWaveform: options.getMediaTimingReviewWaveform, stopMediaTimingReviewPreview: options.stopMediaTimingReviewPreview, resolveMediaTimingReview: options.resolveMediaTimingReview, @@ -613,6 +646,16 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar return await deps.getMediaTimingReviewWaveform(request); }, ); + ipc.handle( + IPC_CHANNELS.request.mediaTimingReviewFrame, + async (_event: unknown, payload: unknown) => { + const request = parseMediaTimingReviewFrameRequest(payload); + if (!request || !deps.getMediaTimingReviewFrame) { + return { ok: false, message: 'Screenshot preview is unavailable.' }; + } + return await deps.getMediaTimingReviewFrame(request); + }, + ); ipc.handle( IPC_CHANNELS.request.mediaTimingReviewStopPreview, async (_event: unknown, reviewId: unknown) => { diff --git a/src/core/services/media-timing-frame.test.ts b/src/core/services/media-timing-frame.test.ts new file mode 100644 index 00000000..8818ec10 --- /dev/null +++ b/src/core/services/media-timing-frame.test.ts @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createMediaTimingFrameExtractor, selectMediaTimingFrame } from './media-timing-frame'; + +test('frame stepping follows decoded timestamps with variable frame durations', () => { + const times = [10, 10.041667, 10.125, 10.166667]; + assert.equal(selectMediaTimingFrame(times, 10.05), 10.125); + assert.equal(selectMediaTimingFrame(times, 10.125 - 0.000001, 1), 10.166667); + assert.equal(selectMediaTimingFrame(times, 10.125 - 0.000001, -1), 10.041667); + assert.equal(selectMediaTimingFrame(times, 10, -1), undefined); + assert.equal(selectMediaTimingFrame(times, 10.166667, 1), undefined); + assert.equal(selectMediaTimingFrame([], 10), undefined); +}); + +function fixture(startTime = 0) { + const calls: Array<{ file: string; args: string[] }> = []; + const extractor = createMediaTimingFrameExtractor(async (file, args) => { + calls.push({ file, args }); + if (file === 'ffmpeg') return Buffer.from('image'); + if (args.includes('format=start_time')) + return Buffer.from(JSON.stringify({ format: { start_time: String(startTime) } })); + return Buffer.from( + JSON.stringify({ + frames: [10, 10.04, 10.12, 10.16].map((time) => ({ + best_effort_timestamp_time: String(time + startTime), + })), + }), + ); + }); + return { calls, extractor }; +} + +test('frame extraction normalizes nonzero source start times and reuses the frame index', async () => { + const { calls, extractor } = fixture(5); + const first = await extractor.generate({ media: '/movie.mkv', timestamp: 10.05 }); + assert.ok(Math.abs(first.timestamp - 10.119999) < 0.000001); + assert.equal(first.dataUrl, 'data:image/jpeg;base64,aW1hZ2U='); + await extractor.generate({ media: '/movie.mkv', timestamp: first.timestamp, direction: 1 }); + assert.equal(calls.filter((call) => call.file === 'ffprobe').length, 2); + assert.ok(calls[1]!.args.includes('13.05%17.05')); + extractor.clear(); + await extractor.generate({ media: '/movie.mkv', timestamp: 10.05 }); + assert.equal(calls.filter((call) => call.file === 'ffprobe').length, 4); +}); + +test('cached windows keep source timestamps for ffmpeg and use absolute ffprobe intervals', async () => { + const { calls, extractor } = fixture(); + await extractor.generate({ + media: { path: '/window.mkv', absoluteTimestamps: true }, + timestamp: 10.05, + }); + assert.ok(calls[1]!.args.includes('8.05%12.05')); + assert.equal(calls[1]!.args.includes('-seek_timestamp'), false); + assert.ok(calls[2]!.args.includes('-seek_timestamp')); +}); + +test('remote frame reads carry source headers and do not silently reuse a different input', async () => { + const { calls, extractor } = fixture(); + const media = { + path: 'https://example.test/video', + inputOptions: { headers: { 'X-Emby-Token': 'test-token' } }, + }; + await extractor.generate({ media, timestamp: 10.05 }); + assert.ok(calls.every((call) => call.args.includes('X-Emby-Token: test-token\r\n'))); + await extractor.generate({ + media: { ...media, path: 'https://example.test/other' }, + timestamp: 10.05, + }); + assert.equal(calls.filter((call) => call.file === 'ffprobe').length, 4); +}); diff --git a/src/core/services/media-timing-frame.ts b/src/core/services/media-timing-frame.ts new file mode 100644 index 00000000..9c9e0a11 --- /dev/null +++ b/src/core/services/media-timing-frame.ts @@ -0,0 +1,159 @@ +import { execFile } from 'node:child_process'; +import { normalizeMediaInput, type MediaInput } from '../../media-input'; + +export interface MediaTimingFrameOptions { + media: MediaInput; + timestamp: number; + direction?: -1 | 1; +} + +const FRAME_EPSILON = 0.00001; +const PROBE_RADIUS_SECONDS = 2; + +function run(file: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + execFile( + file, + args, + { encoding: 'buffer', timeout: 30_000, maxBuffer: 8 * 1024 * 1024, windowsHide: true }, + (error, stdout) => { + // Child-process errors contain the input URL, which may contain authentication tokens. + if (error) + reject( + new Error('Screenshot preview is unavailable. Check FFmpeg and the video source.'), + ); + else resolve(stdout); + }, + ); + }); +} + +/** Uses decoded timestamps, rather than an assumed FPS, including for variable-rate video. */ +export function selectMediaTimingFrame( + times: readonly number[], + timestamp: number, + direction?: -1 | 1, +): number | undefined { + if (direction === -1) + return [...times].reverse().find((time) => time < timestamp - FRAME_EPSILON); + if (direction === 1) return times.find((time) => time > timestamp + FRAME_EPSILON); + return times.find((time) => time >= timestamp - FRAME_EPSILON) ?? times.at(-1); +} + +export function createMediaTimingFrameExtractor(execute: typeof run = run) { + let cached: { key: string; start: number; end: number; times: number[] } | null = null; + let source: { key: string; offset: number } | null = null; + let generation = 0; + + async function generate( + options: MediaTimingFrameOptions, + ): Promise<{ dataUrl: string; timestamp: number }> { + const input = normalizeMediaInput(options.media); + const key = JSON.stringify(options.media); + const currentGeneration = generation; + const absolute = typeof options.media !== 'string' && options.media.absoluteTimestamps; + // -seek_timestamp is an ffmpeg option; ffprobe intervals already use stream timestamps. + const probeInputArgs = normalizeMediaInput({ + path: input.path, + ...(typeof options.media !== 'string' ? { inputOptions: options.media.inputOptions } : {}), + }).inputArgs; + let offset = source?.key === key ? source.offset : undefined; + if (offset === undefined) { + const metadata = JSON.parse( + ( + await execute('ffprobe', [ + '-v', + 'error', + ...probeInputArgs, + '-show_entries', + 'format=start_time', + '-of', + 'json', + input.path, + ]) + ).toString(), + ) as { format?: { start_time?: string } }; + const start = Number(metadata.format?.start_time ?? 0); + offset = absolute || !Number.isFinite(start) ? 0 : start; + if (generation === currentGeneration) source = { key, offset }; + } + let times: number[]; + if ( + cached?.key === key && + options.timestamp > cached.start + 0.5 && + options.timestamp < cached.end - 0.5 + ) { + times = cached.times; + } else { + const start = Math.max(0, options.timestamp - PROBE_RADIUS_SECONDS); + const end = options.timestamp + PROBE_RADIUS_SECONDS; + const result = JSON.parse( + ( + await execute('ffprobe', [ + '-v', + 'error', + ...probeInputArgs, + '-read_intervals', + `${start + offset}%${end + offset}`, + '-select_streams', + 'v:0', + '-show_entries', + 'frame=best_effort_timestamp_time', + '-of', + 'json', + input.path, + ]) + ).toString(), + ) as { frames?: { best_effort_timestamp_time?: string }[] }; + times = [ + ...new Set( + (result.frames ?? []) + .map((frame) => Number(frame.best_effort_timestamp_time) - offset) + .filter((time) => Number.isFinite(time) && time >= 0 && time >= start && time <= end), + ), + ].sort((a, b) => a - b); + if (generation === currentGeneration) cached = { key, start, end, times }; + } + const timestamp = selectMediaTimingFrame(times, options.timestamp, options.direction); + if (timestamp === undefined) throw new Error('No adjacent video frame is available here.'); + // Round down by one microsecond so decimal timestamp rounding cannot skip the chosen frame. + const seekTime = Math.max(0, timestamp - 0.000001); + const image = await execute('ffmpeg', [ + '-hide_banner', + '-nostdin', + '-loglevel', + 'error', + '-ss', + String(seekTime), + ...input.inputArgs, + '-i', + input.path, + '-map', + '0:v:0', + '-frames:v', + '1', + '-an', + '-sn', + '-vf', + 'scale=w=640:h=360:force_original_aspect_ratio=decrease', + '-c:v', + 'mjpeg', + '-q:v', + '3', + '-f', + 'image2pipe', + 'pipe:1', + ]); + if (!image.length) throw new Error('No video frame is available here.'); + return { dataUrl: `data:image/jpeg;base64,${image.toString('base64')}`, timestamp: seekTime }; + } + + return { + generate, + clear: () => { + generation += 1; + cached = null; + source = null; + }, + }; +} diff --git a/src/main.ts b/src/main.ts index 0241988e..68ad8620 100644 --- a/src/main.ts +++ b/src/main.ts @@ -475,6 +475,7 @@ import { MediaTimingPreviewSession } from './core/services/media-timing-preview' import { getSharedRemoteMediaWindowCache } from './core/services/remote-media-window-cache'; import { resolveMediaGenerationInput } from './anki-integration/media-source'; import { generateSpeechWaveform } from './core/services/media-timing-waveform'; +import { createMediaTimingFrameExtractor } from './core/services/media-timing-frame'; import { collectMediaTimingContextLines, createMediaTimingReviewRuntime, @@ -2905,6 +2906,7 @@ function createOverlayHostedModalOpenDeps(): { }; } +const mediaTimingFrameExtractor = createMediaTimingFrameExtractor(); const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({ getMpvClient: () => appState.mpvClient, getCurrentMediaPath: () => @@ -2913,6 +2915,14 @@ const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({ configService.getConfig().mpv.executablePath || process.env.SUBMINER_MPV_PATH?.trim() || '', createPreviewSession: () => new MediaTimingPreviewSession(), generateWaveform: (options) => generateSpeechWaveform(options), + generateFrame: (options) => mediaTimingFrameExtractor.generate(options), + clearFrameCache: () => mediaTimingFrameExtractor.clear(), + resolveVideoSource: () => + resolveMediaGenerationInput(appState.mpvClient, 'video', { + getCachedMediaPath: (currentVideoPath, kind) => + getCachedYoutubeMediaPathForCurrentPlayback(currentVideoPath, kind), + remoteCacheMode: shouldRequireYoutubeMediaCacheForCurrentPlayback() ? 'required' : 'optional', + }), resolveMediaSource: async () => { const resolved = await resolveMediaGenerationInput(appState.mpvClient, 'audio', { getCachedMediaPath: (currentVideoPath, kind) => @@ -5586,6 +5596,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({ }, mainDeps: { previewMediaTimingReview: (request) => mediaTimingReviewRuntime.previewRange(request), + getMediaTimingReviewFrame: (request) => mediaTimingReviewRuntime.getFrame(request), getMediaTimingReviewWaveform: (request) => mediaTimingReviewRuntime.getWaveform(request), stopMediaTimingReviewPreview: (reviewId) => mediaTimingReviewRuntime.stopPreview(reviewId), resolveMediaTimingReview: (request) => mediaTimingReviewRuntime.resolveReview(request), diff --git a/src/main/dependencies.ts b/src/main/dependencies.ts index 1756b02c..90e10f21 100644 --- a/src/main/dependencies.ts +++ b/src/main/dependencies.ts @@ -63,6 +63,7 @@ export interface MainIpcRuntimeServiceDepsParams { handleOverlayNotificationAction?: IpcDepsRuntimeOptions['handleOverlayNotificationAction']; onYoutubePickerResolve: IpcDepsRuntimeOptions['onYoutubePickerResolve']; previewMediaTimingReview?: IpcDepsRuntimeOptions['previewMediaTimingReview']; + getMediaTimingReviewFrame?: IpcDepsRuntimeOptions['getMediaTimingReviewFrame']; getMediaTimingReviewWaveform?: IpcDepsRuntimeOptions['getMediaTimingReviewWaveform']; stopMediaTimingReviewPreview?: IpcDepsRuntimeOptions['stopMediaTimingReviewPreview']; resolveMediaTimingReview?: IpcDepsRuntimeOptions['resolveMediaTimingReview']; @@ -263,6 +264,7 @@ export function createMainIpcRuntimeServiceDeps( handleOverlayNotificationAction: params.handleOverlayNotificationAction, onYoutubePickerResolve: params.onYoutubePickerResolve, previewMediaTimingReview: params.previewMediaTimingReview, + getMediaTimingReviewFrame: params.getMediaTimingReviewFrame, getMediaTimingReviewWaveform: params.getMediaTimingReviewWaveform, stopMediaTimingReviewPreview: params.stopMediaTimingReviewPreview, resolveMediaTimingReview: params.resolveMediaTimingReview, diff --git a/src/main/runtime/media-timing-review-frame.test.ts b/src/main/runtime/media-timing-review-frame.test.ts new file mode 100644 index 00000000..9d7a9ddf --- /dev/null +++ b/src/main/runtime/media-timing-review-frame.test.ts @@ -0,0 +1,161 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import type { MediaTimingReviewOpenPayload } from '../../types/anki'; +import type { MediaTimingFrameOptions } from '../../core/services/media-timing-frame'; +import { + createMediaTimingReviewRuntime, + type MediaTimingReviewRuntimeDeps, +} from './media-timing-review'; + +async function start( + options: Partial = {}, + screenshotEnabled = true, +) { + let opened!: (payload: MediaTimingReviewOpenPayload) => void; + const payloadPromise = new Promise((resolve) => { + opened = resolve; + }); + const frames: MediaTimingFrameOptions[] = []; + const runtime = createMediaTimingReviewRuntime({ + getMpvClient: () => ({ + connected: true, + currentVideoPath: '/video.mkv', + send: () => {}, + requestProperty: async (name) => (name === 'duration' ? 100 : true), + }), + getCurrentMediaPath: () => '/video.mkv', + getMpvExecutablePath: () => '', + createPreviewSession: () => ({ + start: async () => {}, + play: async () => {}, + stop: async () => {}, + onPlaybackEnded: () => {}, + dispose: () => {}, + }), + generateWaveform: async () => [0, 1], + resolveVideoSource: async () => ({ path: '/video.mkv' }), + generateFrame: async (options) => { + frames.push(options); + return { dataUrl: 'data:image/jpeg;base64,AA==', timestamp: options.timestamp }; + }, + openModal: async (payload) => { + opened(payload); + return true; + }, + showStatus: () => {}, + ...options, + }); + const decision = runtime.requestReview({ + kind: 'word', + text: '字幕', + startTime: 10, + endTime: 12, + audioPadding: 0, + maxMediaDuration: 30, + screenshotEnabled, + }); + return { runtime, decision, payload: await payloadPromise, frames }; +} + +test('review accepts a screenshot outside the audio range and validates media bounds', async () => { + const { runtime, decision, payload, frames } = await start(); + for (const timestamp of [NaN, Infinity, -1, 100]) { + assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp })).ok, false); + assert.equal( + runtime.resolveReview({ + reviewId: payload.reviewId, + decision: { action: 'confirm', startTime: 10, endTime: 11, screenshotTime: timestamp }, + }).ok, + false, + ); + } + assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 13 })).ok, true); + assert.equal(frames[0]?.timestamp, 13); + assert.equal( + runtime.resolveReview({ + reviewId: payload.reviewId, + decision: { action: 'confirm', startTime: 10, endTime: 11, screenshotTime: 13 }, + }).ok, + true, + ); + assert.deepEqual(await decision, { + action: 'confirm', + startTime: 10, + endTime: 11, + screenshotTime: 13, + }); +}); + +test('screenshot preview reuses the remote audio window and its absolute timestamps', async () => { + let downloads = 0; + const remote = 'https://example.test/video'; + const { runtime, payload, decision, frames } = await start({ + resolveMediaSource: async () => ({ path: remote }), + resolveVideoSource: async () => ({ path: remote }), + acquireMediaWindow: async () => { + downloads += 1; + return { + path: '/window.mkv', + sourcePath: remote, + startTime: 7, + endTime: 15, + audioStreamIndex: null, + media: { path: '/window.mkv', absoluteTimestamps: true }, + }; + }, + }); + await runtime.getWaveform({ reviewId: payload.reviewId, startTime: 8, endTime: 14 }); + assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 })).ok, true); + assert.equal(downloads, 1); + assert.deepEqual(frames[0]?.media, { path: '/window.mkv', absoluteTimestamps: true }); + await runtime.dispose(); + await decision; +}); + +test('split video streams never read an audio-only cache as a video source', async () => { + const video = { + path: 'https://example.test/video', + inputOptions: { headers: { 'X-Test': 'value' } }, + }; + const { runtime, payload, decision, frames } = await start({ + resolveMediaSource: async () => ({ path: 'https://example.test/audio' }), + resolveVideoSource: async () => video, + }); + await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 }); + assert.deepEqual(frames[0]?.media, video); + await runtime.dispose(); + await decision; +}); + +test('disabled screenshots and missing video inputs do not trigger extraction', async () => { + for (const [enabled, options] of [ + [false, {}], + [true, { resolveVideoSource: async () => null }], + ] as const) { + const { runtime, payload, decision, frames } = await start(options, enabled); + assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 })).ok, false); + assert.equal(frames.length, 0); + await runtime.dispose(); + await decision; + } +}); + +test('closing a review invalidates in-flight frame results and clears the frame index', async () => { + let finish!: (value: { dataUrl: string; timestamp: number }) => void; + let clearCount = 0; + const { runtime, payload, decision } = await start({ + generateFrame: () => + new Promise((resolve) => { + finish = resolve; + }), + clearFrameCache: () => { + clearCount += 1; + }, + }); + const frame = runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 }); + await runtime.dispose(); + finish({ dataUrl: 'data:image/jpeg;base64,AA==', timestamp: 11 }); + assert.equal((await frame).stale, true); + await decision; + assert.equal(clearCount, 1); +}); diff --git a/src/main/runtime/media-timing-review.ts b/src/main/runtime/media-timing-review.ts index b78a23cb..c611a0c3 100644 --- a/src/main/runtime/media-timing-review.ts +++ b/src/main/runtime/media-timing-review.ts @@ -7,10 +7,13 @@ import type { MediaTimingReviewPreviewRequest, MediaTimingReviewRequest, MediaTimingReviewResolveRequest, + MediaTimingReviewFrameRequest, + MediaTimingReviewFrameResult, MediaTimingReviewWaveformRequest, MediaTimingReviewWaveformResult, } from '../../types/anki'; import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform'; +import type { MediaTimingFrameOptions } from '../../core/services/media-timing-frame'; import { isRemoteMediaWindowSourcePath, type RemoteMediaWindow, @@ -59,6 +62,8 @@ interface ActiveReview { mediaPath: string; /** What the waveform reads when no cached window is available. */ waveformMedia: MediaInput; + videoSource: ReviewMediaSource | null; + frameInFlight: boolean; audioStreamIndex?: number; /** Remote source to download windows of; null for local media or without a cache. */ windowSource: RemoteMediaWindowSource | null; @@ -81,6 +86,11 @@ export interface MediaTimingReviewRuntimeDeps { generateWaveform: (options: SpeechWaveformOptions) => Promise; /** Resolves the FFmpeg-readable stream URL and headers behind the current media path. */ resolveMediaSource?: () => Promise; + resolveVideoSource?: () => Promise; + generateFrame?: ( + options: MediaTimingFrameOptions, + ) => Promise<{ dataUrl: string; timestamp: number }>; + clearFrameCache?: () => void; /** Downloads (or reuses) a local window of a remote source covering the range. */ acquireMediaWindow?: ( source: RemoteMediaWindowSource, @@ -227,6 +237,9 @@ export function buildMediaTimingReviewPayload( timelineEndTime, ...(duration !== null && duration > 0 ? { mediaDuration: duration } : {}), maxMediaDuration, + ...(request.screenshotEnabled !== undefined + ? { screenshotEnabled: request.screenshotEnabled } + : {}), }; } @@ -335,13 +348,15 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep return { action: 'use-original' }; } - const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw, resolvedSource] = await Promise.all([ - mpvClient.requestProperty?.('pause').catch(() => null) ?? null, - mpvClient.requestProperty?.('duration').catch(() => null) ?? null, - mpvClient.requestProperty?.('aid').catch(() => null) ?? null, - mpvClient.requestProperty?.('volume').catch(() => null) ?? null, - deps.resolveMediaSource?.().catch(() => null) ?? null, - ]); + const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw, resolvedSource, videoSource] = + await Promise.all([ + mpvClient.requestProperty?.('pause').catch(() => null) ?? null, + mpvClient.requestProperty?.('duration').catch(() => null) ?? null, + mpvClient.requestProperty?.('aid').catch(() => null) ?? null, + mpvClient.requestProperty?.('volume').catch(() => null) ?? null, + deps.resolveMediaSource?.().catch(() => null) ?? null, + request.screenshotEnabled ? (deps.resolveVideoSource?.().catch(() => null) ?? null) : null, + ]); const pauseState = booleanProperty(pauseRaw); mpvClient.send({ command: ['set_property', 'pause', 'yes'] }); pendingPauseRestore = pauseState === false ? mpvClient : null; @@ -383,6 +398,8 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep payload, mediaPath, waveformMedia: inputOptions ? { path: sourcePath, inputOptions } : sourcePath, + videoSource, + frameInFlight: false, ...(audioStreamIndex !== undefined ? { audioStreamIndex } : {}), windowSource, window: null, @@ -527,6 +544,58 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep } } + async function getFrame( + request: MediaTimingReviewFrameRequest, + ): Promise { + const current = active; + if (!current || request.reviewId !== current.payload.reviewId) return staleReviewResult(); + if (!current.payload.screenshotEnabled || !deps.generateFrame || !current.videoSource) { + return { ok: false, message: 'Screenshot preview is unavailable for this media.' }; + } + if ( + !Number.isFinite(request.timestamp) || + request.timestamp < 0 || + (current.payload.mediaDuration !== undefined && + request.timestamp >= current.payload.mediaDuration) || + (request.direction !== undefined && request.direction !== -1 && request.direction !== 1) + ) { + return { ok: false, message: 'The screenshot time is invalid.' }; + } + if (current.frameInFlight) + return { ok: false, message: 'A screenshot preview is already loading.' }; + current.frameInFlight = true; + try { + // Reuse the audio window only when it contains this same video source (not split streams). + const range = { + startTime: Math.max(0, Math.min(current.payload.timelineStartTime, request.timestamp - 2)), + endTime: Math.min( + current.payload.mediaDuration ?? Infinity, + Math.max(current.payload.timelineEndTime, request.timestamp + 2), + ), + }; + const window = + current.windowSource?.path === current.videoSource.path + ? await ensureWindow(current, range) + : null; + if (active !== current) return staleReviewResult(); + const frame = await deps.generateFrame({ + media: window?.media ?? current.videoSource, + timestamp: request.timestamp, + ...(request.direction !== undefined ? { direction: request.direction } : {}), + }); + if (active !== current) return staleReviewResult(); + return { ok: true, ...frame }; + } catch { + if (active !== current) return staleReviewResult(); + return { + ok: false, + message: 'Screenshot preview unavailable. Try another time or reset to the midpoint.', + }; + } finally { + current.frameInFlight = false; + } + } + async function stopPreview(reviewId: string): Promise { const current = active; if (!current || reviewId !== current.payload.reviewId) { @@ -550,13 +619,23 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep return staleReviewResult(); } if (request.decision.action === 'confirm') { - const { startTime, endTime, text } = request.decision; + const { startTime, endTime, text, screenshotTime } = request.decision; if (!isValidMediaTimingRange(current.payload, startTime, endTime)) { return { ok: false, message: 'The selected timing range is invalid.' }; } if (text !== undefined && (typeof text !== 'string' || text.trim().length === 0)) { return { ok: false, message: 'The combined sentence text is invalid.' }; } + if ( + screenshotTime !== undefined && + (!current.payload.screenshotEnabled || + !Number.isFinite(screenshotTime) || + screenshotTime < 0 || + (current.payload.mediaDuration !== undefined && + screenshotTime >= current.payload.mediaDuration)) + ) { + return { ok: false, message: 'The screenshot time is invalid.' }; + } } current.resolve(request.decision); return { ok: true }; @@ -566,6 +645,7 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep const current = active; active = null; if (!current) return; + deps.clearFrameCache?.(); void current.preview?.session.then((session) => session.dispose()).catch(() => {}); if (current.restorePlayback && current.mpvClient.connected) { current.mpvClient.send({ command: ['set_property', 'pause', 'no'] }); @@ -582,6 +662,7 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep requestReview, previewRange, getWaveform, + getFrame, stopPreview, resolveReview, dispose, diff --git a/src/preload.ts b/src/preload.ts index bbd05fd0..d9cc260f 100644 --- a/src/preload.ts +++ b/src/preload.ts @@ -73,6 +73,8 @@ import type { MediaTimingReviewOpenPayload, MediaTimingReviewPreviewRequest, MediaTimingReviewResolveRequest, + MediaTimingReviewFrameRequest, + MediaTimingReviewFrameResult, MediaTimingReviewWaveformRequest, } from './types'; import { IPC_CHANNELS } from './shared/ipc/contracts'; @@ -507,6 +509,10 @@ const electronAPI: ElectronAPI = { request: MediaTimingReviewPreviewRequest, ): Promise => ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewPreview, request), + getMediaTimingReviewFrame: ( + request: MediaTimingReviewFrameRequest, + ): Promise => + ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewFrame, request), getMediaTimingReviewWaveform: (request: MediaTimingReviewWaveformRequest) => ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewWaveform, request), stopMediaTimingReviewPreview: (reviewId: string): Promise => diff --git a/src/renderer/index.html b/src/renderer/index.html index e1377395..52f75c77 100644 --- a/src/renderer/index.html +++ b/src/renderer/index.html @@ -297,6 +297,57 @@
+ +
Starts diff --git a/src/renderer/modals/media-timing-frame-picker.test.ts b/src/renderer/modals/media-timing-frame-picker.test.ts new file mode 100644 index 00000000..84c22d46 --- /dev/null +++ b/src/renderer/modals/media-timing-frame-picker.test.ts @@ -0,0 +1,116 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { setTimeout as tick } from 'node:timers/promises'; +import { createMediaTimingFramePicker } from './media-timing-frame-picker'; +import type { MediaTimingReviewFrameRequest, MediaTimingReviewFrameResult } from '../../types/anki'; + +function fixture() { + const requests: Array<{ + request: MediaTimingReviewFrameRequest; + resolve: (result: MediaTimingReviewFrameResult) => void; + }> = []; + let stale = false; + const picker = createMediaTimingFramePicker({ + debounceMs: 0, + load: (request) => new Promise((resolve) => requests.push({ request, resolve })), + onChange: () => {}, + onStale: () => { + stale = true; + picker.close(); + }, + }); + const complete = async (index: number, result: number | MediaTimingReviewFrameResult) => { + requests[index]!.resolve( + typeof result === 'number' + ? { ok: true, timestamp: result, dataUrl: `data:image/jpeg;base64,${result}` } + : result, + ); + await tick(0); + }; + return { picker, requests, complete, isStale: () => stale }; +} + +test('a chosen frame stays fixed until Reset restores midpoint tracking', async () => { + const { picker, requests, complete } = fixture(); + picker.open('r', true, 11); + await tick(5); + await complete(0, 11); + assert.equal(picker.getScreenshotTime(), undefined); + picker.choose(13); + await tick(5); + await complete(1, 13); + picker.updateMidpoint(12); + await tick(5); + assert.equal(picker.getScreenshotTime(), 13); + assert.equal(requests.length, 2); + picker.reset(); + await tick(5); + assert.equal(requests[2]?.request.timestamp, 12); + await complete(2, 12); + assert.equal(picker.getScreenshotTime(), undefined); + picker.updateMidpoint(14); + await tick(5); + assert.equal(requests[3]?.request.timestamp, 14); + await complete(3, 14); + picker.close(); +}); + +test('scrubbing keeps only the latest requested frame', async () => { + const { picker, requests, complete } = fixture(); + picker.open('r', true, 11); + await tick(5); + picker.choose(12); + picker.choose(13); + picker.choose(14); + await tick(5); + assert.equal(requests.length, 1); + assert.equal(picker.getState().blockConfirm, true); + await complete(0, 11); + await tick(5); + assert.equal(picker.getState().timestamp, undefined); + assert.equal(requests[1]?.request.timestamp, 14); + await complete(1, 14); + assert.equal(picker.getScreenshotTime(), 14); + assert.equal(picker.getState().blockConfirm, false); + picker.close(); +}); + +test('failed manual previews block confirmation; Reset allows the default fallback', async () => { + const { picker, complete } = fixture(); + picker.open('r', true, 11); + await tick(5); + await complete(0, { ok: false }); + assert.equal(picker.getState().blockConfirm, false); + picker.choose(12); + await tick(5); + await complete(1, { ok: false }); + assert.equal(picker.getState().blockConfirm, true); + picker.reset(); + assert.equal(picker.getState().blockConfirm, false); + picker.close(); +}); + +test('replacing a review ignores old frames; stale responses close the current review', async () => { + const { picker, requests, complete, isStale } = fixture(); + picker.open('old', true, 11); + await tick(5); + picker.close(); + picker.open('new', true, 22); + await tick(5); + await complete(0, 11); + await tick(5); + assert.equal(picker.getState().timestamp, undefined); + assert.equal(requests[1]?.request.reviewId, 'new'); + await complete(1, { ok: false, stale: true }); + assert.equal(isStale(), true); + assert.equal(picker.getState().enabled, false); +}); + +test('disabled screenshots perform no extraction', async () => { + const { picker, requests } = fixture(); + picker.open('r', false, 11); + picker.updateMidpoint(12); + await tick(5); + assert.equal(requests.length, 0); + picker.close(); +}); diff --git a/src/renderer/modals/media-timing-frame-picker.ts b/src/renderer/modals/media-timing-frame-picker.ts new file mode 100644 index 00000000..41138862 --- /dev/null +++ b/src/renderer/modals/media-timing-frame-picker.ts @@ -0,0 +1,129 @@ +import type { MediaTimingReviewFrameRequest, MediaTimingReviewFrameResult } from '../../types/anki'; + +export interface MediaTimingFramePickerState { + enabled: boolean; + manual: boolean; + loading: boolean; + timestamp?: number; + requestedTime?: number; + dataUrl?: string; + message: string; + blockConfirm: boolean; +} + +/** Coalesces scrubbing into one active extraction and the latest requested frame. */ +export function createMediaTimingFramePicker(options: { + load: (request: MediaTimingReviewFrameRequest) => Promise; + onChange: (state: MediaTimingFramePickerState) => void; + onStale: () => void; + debounceMs?: number; +}) { + let reviewId: string | null = null; + let midpoint = 0; + let sequence = 0; + let inFlight = false; + let pending: (MediaTimingReviewFrameRequest & { sequence: number }) | null = null; + let timer: ReturnType | null = null; + let state: MediaTimingFramePickerState = emptyState(); + + function emptyState(): MediaTimingFramePickerState { + return { enabled: false, manual: false, loading: false, message: '', blockConfirm: false }; + } + + function publish(): void { + options.onChange({ ...state }); + } + + async function drain(): Promise { + if (inFlight || !pending) return; + const request = pending; + pending = null; + inFlight = true; + try { + const result = await options.load(request); + if (reviewId !== request.reviewId || sequence !== request.sequence) return; + if (result.stale) { + options.onStale(); + return; + } + if ( + !result.ok || + !Number.isFinite(result.timestamp) || + !result.dataUrl?.startsWith('data:image/jpeg;base64,') + ) { + throw new Error(result.message ?? 'Screenshot preview is unavailable.'); + } + state.timestamp = result.timestamp; + state.dataUrl = result.dataUrl; + state.blockConfirm = false; + state.message = state.manual + ? 'Selected frame stays fixed when you trim audio.' + : 'Following the audio midpoint.'; + } catch (error) { + if (reviewId !== request.reviewId || sequence !== request.sequence) return; + state.message = error instanceof Error ? error.message : 'Screenshot preview is unavailable.'; + state.blockConfirm = state.manual; + } finally { + inFlight = false; + if (reviewId === request.reviewId && sequence === request.sequence) { + state.loading = false; + publish(); + } + if (timer === null) void drain(); + } + } + + function request(timestamp: number, direction?: -1 | 1): void { + if (!reviewId || !state.enabled) return; + sequence += 1; + pending = { reviewId, timestamp, ...(direction ? { direction } : {}), sequence }; + state.requestedTime = timestamp; + state.loading = true; + state.blockConfirm = state.manual; + state.message = 'Loading screenshot…'; + publish(); + if (timer !== null) clearTimeout(timer); + timer = setTimeout(() => { + timer = null; + void drain(); + }, options.debounceMs ?? 120); + } + + function close(): void { + sequence += 1; + reviewId = null; + pending = null; + if (timer !== null) clearTimeout(timer); + timer = null; + state = emptyState(); + publish(); + } + + return { + open(id: string, enabled: boolean, time: number) { + close(); + reviewId = id; + midpoint = time; + state.enabled = enabled; + publish(); + if (enabled) request(time); + }, + updateMidpoint(time: number) { + if (midpoint === time) return; + midpoint = time; + if (!state.manual) request(time); + }, + choose(time: number, direction?: -1 | 1) { + if (!Number.isFinite(time) || time < 0) return; + state.manual = true; + request(time, direction); + }, + reset() { + state.manual = false; + request(midpoint); + }, + close, + getState: () => ({ ...state }), + getScreenshotTime: () => (state.manual && !state.blockConfirm ? state.timestamp : undefined), + }; +} diff --git a/src/renderer/modals/media-timing-review.ts b/src/renderer/modals/media-timing-review.ts index 5d9ed85d..52a755b6 100644 --- a/src/renderer/modals/media-timing-review.ts +++ b/src/renderer/modals/media-timing-review.ts @@ -1,3 +1,4 @@ +import { createMediaTimingFramePicker } from './media-timing-frame-picker'; import type { MediaTimingReviewContextLine, MediaTimingReviewDecision, @@ -252,6 +253,45 @@ export function createMediaTimingReviewModal( isModalLayer: ctx.platform.isModalLayer, }); const previewRequest = createMediaTimingPreviewRequestGuard(); + const framePicker = createMediaTimingFramePicker({ + load: (request) => window.electronAPI.getMediaTimingReviewFrame(request), + onChange: () => renderFramePicker(), + onStale: () => closeResolvedReview(), + }); + + function renderFramePicker(): void { + const state = framePicker.getState(); + const dom = ctx.dom; + dom.mediaTimingReviewFramePicker.classList.toggle('hidden', !state.enabled); + dom.mediaTimingReviewFramePicker.setAttribute('aria-busy', String(state.loading)); + dom.mediaTimingReviewFrameSlider.min = String(timelineStart); + dom.mediaTimingReviewFrameSlider.max = String(Math.max(timelineStart, timelineEnd - 0.001)); + const shownTime = state.loading ? state.requestedTime : state.timestamp; + if (shownTime !== undefined) dom.mediaTimingReviewFrameSlider.value = String(shownTime); + dom.mediaTimingReviewFrameTime.textContent = + shownTime === undefined ? '—' : formatMediaTimingTimestamp(shownTime); + dom.mediaTimingReviewFrameSlider.setAttribute( + 'aria-valuetext', + dom.mediaTimingReviewFrameTime.textContent, + ); + dom.mediaTimingReviewFrameStatus.textContent = state.message; + dom.mediaTimingReviewFrameImage.classList.toggle('hidden', !state.dataUrl); + if (state.dataUrl) dom.mediaTimingReviewFrameImage.src = state.dataUrl; + else dom.mediaTimingReviewFrameImage.removeAttribute('src'); + dom.mediaTimingReviewFramePrevious.disabled = + resolveInFlight || + state.loading || + state.timestamp === undefined || + state.timestamp <= timelineStart; + dom.mediaTimingReviewFrameNext.disabled = + resolveInFlight || + state.loading || + state.timestamp === undefined || + state.timestamp >= timelineEnd - 0.001; + dom.mediaTimingReviewFrameSlider.disabled = resolveInFlight; + dom.mediaTimingReviewFrameReset.disabled = resolveInFlight || !state.manual; + dom.mediaTimingReviewConfirm.disabled = resolveInFlight || state.blockConfirm; + } function setStatus(message: string, isError = false): void { ctx.dom.mediaTimingReviewStatus.textContent = message; @@ -308,6 +348,8 @@ export function createMediaTimingReviewModal( } function renderSelection(): void { + framePicker.updateMidpoint(selectionStart + (selectionEnd - selectionStart) / 2); + renderFramePicker(); const span = Math.max(MINIMUM_CLIP_SECONDS, timelineEnd - timelineStart); const startPercent = ((selectionStart - timelineStart) / span) * 100; const endPercent = ((selectionEnd - timelineStart) / span) * 100; @@ -690,6 +732,7 @@ export function createMediaTimingReviewModal( window.electronAPI.notifyOverlayModalClosed('media-timing-review'); options.syncSettingsModalSubtitleSuppression(); payload = null; + framePicker.close(); if (!options.modalStateReader.isAnyModalOpen()) { ctx.dom.overlay.classList.remove('interactive'); if (ctx.platform.shouldToggleMouseIgnore) { @@ -701,6 +744,7 @@ export function createMediaTimingReviewModal( async function resolveReview(decision: MediaTimingReviewDecision): Promise { if (!payload || resolveInFlight) return; resolveInFlight = true; + renderFramePicker(); stopPreview(); const controls = ctx.dom.mediaTimingReviewModal.querySelectorAll('button'); controls.forEach((button) => { @@ -733,12 +777,14 @@ export function createMediaTimingReviewModal( } function confirmSelection(): void { - if (!payload) return; + if (!payload || framePicker.getState().blockConfirm) return; + const screenshotTime = framePicker.getScreenshotTime(); const includesAdjacentLines = previousCount > 0 || nextCount > 0; void resolveReview({ action: 'confirm', startTime: selectionStart, endTime: selectionEnd, + ...(screenshotTime !== undefined ? { screenshotTime } : {}), ...(includesAdjacentLines ? { text: currentLineSelection().sentence } : {}), }); } @@ -818,6 +864,11 @@ export function createMediaTimingReviewModal( nextPayload.noteId !== undefined ? 'Delete card' : "Don't create card"; setStatus(''); showEditor(); + framePicker.open( + nextPayload.reviewId, + nextPayload.screenshotEnabled === true, + selectionStart + (selectionEnd - selectionStart) / 2, + ); renderSelection(); renderSentence(); ctx.state.mediaTimingReviewModalOpen = true; @@ -892,6 +943,28 @@ export function createMediaTimingReviewModal( } function wireDomEvents(): void { + ctx.dom.mediaTimingReviewFrameSlider.addEventListener('input', () => { + if (!resolveInFlight) framePicker.choose(Number(ctx.dom.mediaTimingReviewFrameSlider.value)); + }); + const stepFrame = (direction: -1 | 1) => { + const state = framePicker.getState(); + if (!resolveInFlight && !state.loading && state.timestamp !== undefined) + framePicker.choose(state.timestamp, direction); + }; + ctx.dom.mediaTimingReviewFrameSlider.addEventListener('keydown', (event) => { + if (event.key === 'ArrowLeft' || event.key === 'ArrowDown') { + event.preventDefault(); + stepFrame(-1); + } else if (event.key === 'ArrowRight' || event.key === 'ArrowUp') { + event.preventDefault(); + stepFrame(1); + } + }); + ctx.dom.mediaTimingReviewFramePrevious.addEventListener('click', () => stepFrame(-1)); + ctx.dom.mediaTimingReviewFrameNext.addEventListener('click', () => stepFrame(1)); + ctx.dom.mediaTimingReviewFrameReset.addEventListener('click', () => { + if (!resolveInFlight) framePicker.reset(); + }); const track = ctx.dom.mediaTimingReviewSelectionTrack; track.addEventListener('pointerdown', beginDrag); track.addEventListener('pointermove', (event) => { diff --git a/src/renderer/style.css b/src/renderer/style.css index b65e4a72..598aacf9 100644 --- a/src/renderer/style.css +++ b/src/renderer/style.css @@ -1347,7 +1347,7 @@ body:focus-visible, .media-timing-review-content { width: min(720px, calc(100vw - 32px)); - max-height: min(700px, calc(100vh - 32px)); + max-height: min(900px, calc(100vh - 32px)); overflow: auto; gap: 0; padding: 0; @@ -1425,6 +1425,86 @@ body:focus-visible, padding: 14px 20px 18px; } +.media-timing-frame-picker { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + gap: 16px; + align-items: center; + margin: 0 0 16px; + padding: 12px; + border: 1px solid var(--ctp-surface1); + border-radius: 10px; + background: var(--ctp-mantle); +} + +.media-timing-frame-picker.hidden { + display: none; +} +.media-timing-frame-image-shell { + aspect-ratio: 16 / 9; + display: flex; + align-items: center; + justify-content: center; + overflow: hidden; + border-radius: 6px; + background: var(--ctp-crust); +} +.media-timing-frame-image-shell img { + width: 100%; + height: 100%; + object-fit: contain; +} +.media-timing-frame-picker[aria-busy='true'] img { + opacity: 0.55; +} +.media-timing-frame-heading { + display: flex; + justify-content: space-between; + gap: 8px; + font-size: 12px; +} +.media-timing-frame-heading output { + color: var(--ctp-teal); + font-variant-numeric: tabular-nums; +} +.media-timing-frame-controls input { + width: 100%; + margin: 12px 0; + accent-color: var(--ctp-teal); +} +.media-timing-frame-buttons { + display: flex; + flex-wrap: wrap; + gap: 6px; +} +.media-timing-frame-buttons button { + padding: 5px 8px; + border: 1px solid var(--ctp-surface2); + border-radius: 6px; + background: var(--ctp-surface0); + color: var(--ctp-text); + font-size: 11px; + cursor: pointer; +} +.media-timing-frame-buttons button:disabled { + opacity: 0.45; + cursor: default; +} +.media-timing-frame-controls p { + margin: 8px 0 0; + min-height: 2.6em; + color: var(--ctp-subtext0); + font-size: 11px; +} +@media (max-width: 520px) { + .media-timing-frame-picker { + grid-template-columns: 1fr; + } + .media-timing-frame-image-shell { + max-height: 180px; + } +} + .media-timing-review-sentence-header { display: flex; align-items: center; diff --git a/src/renderer/utils/dom.ts b/src/renderer/utils/dom.ts index 3cde304a..fe9216d2 100644 --- a/src/renderer/utils/dom.ts +++ b/src/renderer/utils/dom.ts @@ -46,6 +46,14 @@ export type RendererDom = { youtubePickerStatus: HTMLDivElement; youtubePickerTracks: HTMLUListElement; + mediaTimingReviewFramePicker: HTMLDivElement; + mediaTimingReviewFrameImage: HTMLImageElement; + mediaTimingReviewFrameSlider: HTMLInputElement; + mediaTimingReviewFrameTime: HTMLElement; + mediaTimingReviewFrameStatus: HTMLElement; + mediaTimingReviewFramePrevious: HTMLButtonElement; + mediaTimingReviewFrameNext: HTMLButtonElement; + mediaTimingReviewFrameReset: HTMLButtonElement; mediaTimingReviewModal: HTMLDivElement; mediaTimingReviewKind: HTMLDivElement; mediaTimingReviewText: HTMLElement; @@ -249,6 +257,24 @@ export function resolveRendererDom(): RendererDom { youtubePickerStatus: getRequiredElement('youtubePickerStatus'), youtubePickerTracks: getRequiredElement('youtubePickerTracks'), + mediaTimingReviewFramePicker: getRequiredElement( + 'mediaTimingReviewFramePicker', + ), + mediaTimingReviewFrameImage: getRequiredElement( + 'mediaTimingReviewFrameImage', + ), + mediaTimingReviewFrameSlider: getRequiredElement( + 'mediaTimingReviewFrameSlider', + ), + mediaTimingReviewFrameTime: getRequiredElement('mediaTimingReviewFrameTime'), + mediaTimingReviewFrameStatus: getRequiredElement('mediaTimingReviewFrameStatus'), + mediaTimingReviewFramePrevious: getRequiredElement( + 'mediaTimingReviewFramePrevious', + ), + mediaTimingReviewFrameNext: getRequiredElement('mediaTimingReviewFrameNext'), + mediaTimingReviewFrameReset: getRequiredElement( + 'mediaTimingReviewFrameReset', + ), mediaTimingReviewModal: getRequiredElement('mediaTimingReviewModal'), mediaTimingReviewKind: getRequiredElement('mediaTimingReviewKind'), mediaTimingReviewText: getRequiredElement('mediaTimingReviewText'), diff --git a/src/shared/ipc/contracts.ts b/src/shared/ipc/contracts.ts index 6bfb6644..d6fe7d4c 100644 --- a/src/shared/ipc/contracts.ts +++ b/src/shared/ipc/contracts.ts @@ -138,6 +138,7 @@ export const IPC_CHANNELS = { getChangelogSnapshot: 'changelog:get-snapshot', mediaTimingReviewPreview: 'media-timing-review:preview', mediaTimingReviewWaveform: 'media-timing-review:waveform', + mediaTimingReviewFrame: 'media-timing-review:frame', mediaTimingReviewStopPreview: 'media-timing-review:stop-preview', mediaTimingReviewResolve: 'media-timing-review:resolve', }, diff --git a/src/types/anki.ts b/src/types/anki.ts index 3329e305..2874b1a4 100644 --- a/src/types/anki.ts +++ b/src/types/anki.ts @@ -21,6 +21,8 @@ export interface MediaTimingReviewRequest { noteId?: number; audioPadding: number; maxMediaDuration: number; + /** Still screenshots only; animated images continue to follow the audio range. */ + screenshotEnabled?: boolean; } /** A subtitle line adjacent to the mined one that the review can pull onto the card. */ @@ -32,7 +34,13 @@ export interface MediaTimingReviewContextLine { export type MediaTimingReviewDecision = /** `text` is set when the review combined adjacent lines into the card sentence. */ - | { action: 'confirm'; startTime: number; endTime: number; text?: string } + | { + action: 'confirm'; + startTime: number; + endTime: number; + text?: string; + screenshotTime?: number; + } | { action: 'use-original' } | { action: 'skip-media' } | { action: 'discard' }; @@ -53,6 +61,19 @@ export interface MediaTimingReviewOpenPayload { timelineEndTime: number; mediaDuration?: number; maxMediaDuration: number; + screenshotEnabled?: boolean; +} + +export interface MediaTimingReviewFrameRequest { + reviewId: string; + timestamp: number; + /** Step to the adjacent decoded frame instead of seeking to a time. */ + direction?: -1 | 1; +} + +export interface MediaTimingReviewFrameResult extends MediaTimingReviewActionResult { + dataUrl?: string; + timestamp?: number; } export interface MediaTimingReviewPreviewRequest { diff --git a/src/types/runtime.ts b/src/types/runtime.ts index e3c46bb1..edf1b1a2 100644 --- a/src/types/runtime.ts +++ b/src/types/runtime.ts @@ -8,6 +8,8 @@ import type { MediaTimingReviewOpenPayload, MediaTimingReviewPreviewRequest, MediaTimingReviewResolveRequest, + MediaTimingReviewFrameRequest, + MediaTimingReviewFrameResult, MediaTimingReviewWaveformRequest, MediaTimingReviewWaveformResult, } from './anki'; @@ -550,6 +552,9 @@ export interface ElectronAPI { previewMediaTimingReview: ( request: MediaTimingReviewPreviewRequest, ) => Promise; + getMediaTimingReviewFrame: ( + request: MediaTimingReviewFrameRequest, + ) => Promise; getMediaTimingReviewWaveform: ( request: MediaTimingReviewWaveformRequest, ) => Promise; diff --git a/src/types/subtitle.ts b/src/types/subtitle.ts index 40de3151..64b0226d 100644 --- a/src/types/subtitle.ts +++ b/src/types/subtitle.ts @@ -246,6 +246,8 @@ export interface SubtitleMiningContext { capturedAtMs?: number; /** Explicit generator padding. Confirmed timing-review ranges set this to zero. */ mediaPaddingSeconds?: number; + /** Independent still screenshot selected during media timing review. */ + screenshotTime?: number; } export interface SubtitleHoverTokenPayload {