From e22117fe83300ebf9b9544500d360aded37b1136 Mon Sep 17 00:00:00 2001 From: sudacode Date: Sun, 16 Aug 2026 16:41:14 -0700 Subject: [PATCH] feat(anki): add waveform-guided timing controls - Add speech-weighted waveform analysis and playback playhead - Support dragging, sliding, and keyboard nudging for clip timing --- changes/media-timing-review.md | 2 +- docs-site/anki-integration.md | 2 +- src/core/services/ipc.ts | 23 + .../services/media-timing-waveform.test.ts | 74 +++ src/core/services/media-timing-waveform.ts | 158 +++++++ src/main.ts | 3 + src/main/dependencies.ts | 2 + src/main/runtime/media-timing-review.test.ts | 67 +++ src/main/runtime/media-timing-review.ts | 49 ++ src/preload.ts | 3 + src/renderer/index.html | 86 ++-- .../modals/media-timing-review.test.ts | 32 ++ src/renderer/modals/media-timing-review.ts | 296 +++++++++++- src/renderer/style.css | 420 ++++++++++++++---- src/renderer/utils/dom.ts | 28 +- src/shared/ipc/contracts.ts | 1 + src/types/anki.ts | 10 + src/types/runtime.ts | 5 + 18 files changed, 1120 insertions(+), 141 deletions(-) create mode 100644 src/core/services/media-timing-waveform.test.ts create mode 100644 src/core/services/media-timing-waveform.ts diff --git a/changes/media-timing-review.md b/changes/media-timing-review.md index 9b409efd..3ee86c74 100644 --- a/changes/media-timing-review.md +++ b/changes/media-timing-review.md @@ -1,4 +1,4 @@ type: added area: mining -- Added optional pre-generation timing review for word, sentence, and audio cards with draggable clip bounds, audio preview, exact screenshot and AVIF timing, and explicit cancellation choices. +- Added optional pre-generation timing review for word, sentence, and audio cards with a speech-weighted waveform you can drag to trim or slide the clip, keyboard nudging, audio preview with a sweeping playhead, exact screenshot and AVIF timing, and explicit cancellation choices. diff --git a/docs-site/anki-integration.md b/docs-site/anki-integration.md index 0e9ddc20..97eb7dfc 100644 --- a/docs-site/anki-integration.md +++ b/docs-site/anki-integration.md @@ -179,7 +179,7 @@ Output format: MP3 at 44100 Hz. If the video has multiple audio streams, SubMine The audio is uploaded to Anki's media folder and inserted as `[sound:audio_.mp3]`. -Set `media.reviewTiming` to `true` to pause playback and review each word, sentence, or audio card before its media is generated. The review opens with the subtitle range plus configured audio padding, provides draggable start and end handles, 100 ms adjustments, audio preview, and controls to reveal another five seconds before or after the visible timeline. The confirmed range is exact: SubMiner does not apply audio padding a second time. Static screenshots use its midpoint, and animated AVIF clips use the full confirmed range. +Set `media.reviewTiming` to `true` to pause playback and review each word, sentence, or audio card before its media is generated. The review opens with the subtitle range plus configured audio padding. Drag either edge of the clip to trim it, drag the middle to slide it without changing its length, or press anywhere else on the waveform to snap the nearer edge there. A focused edge also moves with the arrow keys, by 100 ms alone or 500 ms with Shift, and the 100 ms buttons do the same. Space previews the selection with a playhead that sweeps the clip, Enter confirms, and Escape cancels; buttons reveal another five seconds before or after the visible timeline. A speech-weighted waveform and markers for the original subtitle timing help locate sentence boundaries. SubMiner uses a center channel when one carries dialogue, then falls back to a speech-band mono mix. Waveform analysis failure leaves the timing controls available. The confirmed range is exact: SubMiner does not apply audio padding a second time. Static screenshots use its midpoint, and animated AVIF clips use the full confirmed range. Canceling the review lets you keep editing, finish with the original timing, or discard the card. Discard deletes an existing Yomitan or audio card and skips creation for a direct sentence card. Clipboard updates and stats-dashboard mining do not open timing review. Audio preview failure does not block confirmation or card creation. The option is disabled by default and hot-reloads. diff --git a/src/core/services/ipc.ts b/src/core/services/ipc.ts index 64a6899c..0f82051e 100644 --- a/src/core/services/ipc.ts +++ b/src/core/services/ipc.ts @@ -23,6 +23,8 @@ import type { MediaTimingReviewActionResult, MediaTimingReviewPreviewRequest, MediaTimingReviewResolveRequest, + MediaTimingReviewWaveformRequest, + MediaTimingReviewWaveformResult, } from '../../types/anki'; import { IPC_CHANNELS, type OverlayHostedModal } from '../../shared/ipc/contracts'; import { @@ -106,6 +108,9 @@ export interface IpcServiceDeps { previewMediaTimingReview?: ( request: MediaTimingReviewPreviewRequest, ) => Promise; + getMediaTimingReviewWaveform?: ( + request: MediaTimingReviewWaveformRequest, + ) => Promise; stopMediaTimingReviewPreview?: (reviewId: string) => Promise; resolveMediaTimingReview?: ( request: MediaTimingReviewResolveRequest, @@ -255,6 +260,12 @@ function parseMediaTimingReviewPreviewRequest( }; } +function parseMediaTimingReviewWaveformRequest( + payload: unknown, +): MediaTimingReviewWaveformRequest | null { + return parseMediaTimingReviewPreviewRequest(payload); +} + function parseMediaTimingReviewResolveRequest( payload: unknown, ): MediaTimingReviewResolveRequest | null { @@ -343,6 +354,7 @@ export interface IpcDepsRuntimeOptions { request: YoutubePickerResolveRequest, ) => Promise; previewMediaTimingReview?: IpcServiceDeps['previewMediaTimingReview']; + getMediaTimingReviewWaveform?: IpcServiceDeps['getMediaTimingReviewWaveform']; stopMediaTimingReviewPreview?: IpcServiceDeps['stopMediaTimingReviewPreview']; resolveMediaTimingReview?: IpcServiceDeps['resolveMediaTimingReview']; getAnkiConnectStatus: () => boolean; @@ -439,6 +451,7 @@ export function createIpcDepsRuntime(options: IpcDepsRuntimeOptions): IpcService runSubsyncManual: options.runSubsyncManual, onYoutubePickerResolve: options.onYoutubePickerResolve, previewMediaTimingReview: options.previewMediaTimingReview, + getMediaTimingReviewWaveform: options.getMediaTimingReviewWaveform, stopMediaTimingReviewPreview: options.stopMediaTimingReviewPreview, resolveMediaTimingReview: options.resolveMediaTimingReview, getAnkiConnectStatus: options.getAnkiConnectStatus, @@ -572,6 +585,16 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar return await deps.previewMediaTimingReview(request); }, ); + ipc.handle( + IPC_CHANNELS.request.mediaTimingReviewWaveform, + async (_event: unknown, payload: unknown) => { + const request = parseMediaTimingReviewWaveformRequest(payload); + if (!request || !deps.getMediaTimingReviewWaveform) { + return { ok: false, message: 'Timing waveform is unavailable.' }; + } + return await deps.getMediaTimingReviewWaveform(request); + }, + ); ipc.handle( IPC_CHANNELS.request.mediaTimingReviewStopPreview, async (_event: unknown, reviewId: unknown) => { diff --git a/src/core/services/media-timing-waveform.test.ts b/src/core/services/media-timing-waveform.test.ts new file mode 100644 index 00000000..38376ba1 --- /dev/null +++ b/src/core/services/media-timing-waveform.test.ts @@ -0,0 +1,74 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + buildSpeechWaveformArgs, + computeWaveformPeaks, + generateSpeechWaveform, +} from './media-timing-waveform'; + +function pcm(samples: number[]): Buffer { + const result = Buffer.alloc(samples.length * 2); + samples.forEach((sample, index) => result.writeInt16LE(sample, index * 2)); + return result; +} + +test('speech waveform maps the selected FFmpeg stream and visible range', () => { + const args = buildSpeechWaveformArgs( + { + mediaPath: '/video/show.mkv', + startTime: 8, + endTime: 15, + audioStreamIndex: 3, + }, + 'center', + ); + + assert.deepEqual(args.slice(args.indexOf('-ss'), args.indexOf('-t') + 2), [ + '-ss', + '8', + '-i', + '/video/show.mkv', + '-t', + '7', + ]); + assert.deepEqual(args.slice(args.indexOf('-map'), args.indexOf('-map') + 2), ['-map', '0:3']); + assert.match(args[args.indexOf('-af') + 1] ?? '', /c0=FC/); +}); + +test('waveform peaks are normalized without flattening quieter sections', () => { + const peaks = computeWaveformPeaks(pcm([0, 1_000, -2_000, 4_000, -8_000, 16_000]), 3); + + assert.equal(peaks.length, 3); + assert.ok((peaks[0] ?? 0) > 0); + assert.ok((peaks[0] ?? 0) < (peaks[1] ?? 0)); + assert.ok((peaks[1] ?? 0) < (peaks[2] ?? 0)); + assert.equal(peaks[2], 1); +}); + +test('speech waveform uses a mono downmix when the source has no center activity', async () => { + const calls: string[][] = []; + const peaks = await generateSpeechWaveform( + { mediaPath: '/video/show.mkv', startTime: 0, endTime: 2 }, + async (args) => { + calls.push(args); + return calls.length === 1 ? pcm([0, 0, 0, 0]) : pcm([0, 4_000, -8_000, 16_000]); + }, + ); + + assert.equal(calls.length, 2); + assert.match(calls[1]?.[calls[1].indexOf('-af') + 1] ?? '', /channel_layouts=mono/); + assert.equal(Math.max(...peaks), 1); +}); + +test('speech waveform keeps an active center channel without doing a second decode', async () => { + let calls = 0; + await generateSpeechWaveform( + { mediaPath: '/video/show.mkv', startTime: 0, endTime: 2 }, + async () => { + calls += 1; + return pcm([0, 4_000, -8_000, 16_000]); + }, + ); + + assert.equal(calls, 1); +}); diff --git a/src/core/services/media-timing-waveform.ts b/src/core/services/media-timing-waveform.ts new file mode 100644 index 00000000..8fbd1a63 --- /dev/null +++ b/src/core/services/media-timing-waveform.ts @@ -0,0 +1,158 @@ +import { spawn } from 'node:child_process'; + +const WAVEFORM_SAMPLE_RATE = 8_000; +const WAVEFORM_POINT_COUNT = 480; +const WAVEFORM_TIMEOUT_MS = 15_000; +const MAX_WAVEFORM_BYTES = 16 * 1024 * 1024; +const SPEECH_FILTER = 'highpass=f=120,lowpass=f=4000'; +const CENTER_CHANNEL_FILTER = `pan=mono|c0=FC,${SPEECH_FILTER}`; +const DOWNMIX_FILTER = `aformat=channel_layouts=mono,${SPEECH_FILTER}`; + +export interface SpeechWaveformOptions { + mediaPath: string; + startTime: number; + endTime: number; + audioStreamIndex?: number; +} + +type RunFfmpeg = (args: string[]) => Promise; + +export function buildSpeechWaveformArgs( + options: SpeechWaveformOptions, + mode: 'center' | 'downmix', +): string[] { + const duration = options.endTime - options.startTime; + const args = [ + '-hide_banner', + '-nostdin', + '-loglevel', + 'error', + '-ss', + String(options.startTime), + '-i', + options.mediaPath, + '-t', + String(duration), + ]; + if ( + options.audioStreamIndex !== undefined && + Number.isInteger(options.audioStreamIndex) && + options.audioStreamIndex >= 0 + ) { + args.push('-map', `0:${options.audioStreamIndex}`); + } + args.push( + '-vn', + '-sn', + '-dn', + '-af', + mode === 'center' ? CENTER_CHANNEL_FILTER : DOWNMIX_FILTER, + '-ac', + '1', + '-ar', + String(WAVEFORM_SAMPLE_RATE), + '-f', + 's16le', + 'pipe:1', + ); + return args; +} + +function runFfmpeg(args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn('ffmpeg', args, { stdio: ['ignore', 'pipe', 'pipe'] }); + const chunks: Buffer[] = []; + let byteLength = 0; + let stderr = ''; + let settled = false; + const timeout = setTimeout(() => { + if (settled) return; + settled = true; + child.kill('SIGKILL'); + reject(new Error(`FFmpeg waveform analysis timed out after ${WAVEFORM_TIMEOUT_MS}ms`)); + }, WAVEFORM_TIMEOUT_MS); + + const settle = (callback: () => void): void => { + if (settled) return; + settled = true; + clearTimeout(timeout); + callback(); + }; + + child.stdout.on('data', (chunk: Buffer) => { + if (settled) return; + byteLength += chunk.byteLength; + if (byteLength > MAX_WAVEFORM_BYTES) { + settle(() => { + child.kill('SIGKILL'); + reject(new Error('The visible waveform range is too large to analyze.')); + }); + return; + } + chunks.push(chunk); + }); + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk) => { + if (stderr.length < 4_000) stderr += String(chunk); + }); + child.once('error', (error) => settle(() => reject(error))); + child.once('close', (code) => { + settle(() => { + if (code === 0) { + resolve(Buffer.concat(chunks, byteLength)); + return; + } + reject(new Error(stderr.trim() || `FFmpeg exited with status ${code ?? 'unknown'}`)); + }); + }); + }); +} + +export function computeWaveformPeaks(pcm: Buffer, pointCount = WAVEFORM_POINT_COUNT): number[] { + const sampleCount = Math.floor(pcm.byteLength / 2); + if (sampleCount === 0 || pointCount <= 0) return []; + const resolvedPointCount = Math.min(pointCount, sampleCount); + const peaks = Array.from({ length: resolvedPointCount }, () => 0); + + for (let point = 0; point < resolvedPointCount; point += 1) { + const sampleStart = Math.floor((point * sampleCount) / resolvedPointCount); + const sampleEnd = Math.max( + sampleStart + 1, + Math.floor(((point + 1) * sampleCount) / resolvedPointCount), + ); + let peak = 0; + for (let sample = sampleStart; sample < sampleEnd; sample += 1) { + peak = Math.max(peak, Math.abs(pcm.readInt16LE(sample * 2)) / 32_768); + } + peaks[point] = peak; + } + + const sortedPeaks = [...peaks].sort((left, right) => left - right); + const referenceIndex = Math.min(sortedPeaks.length - 1, Math.floor(sortedPeaks.length * 0.95)); + const referencePeak = Math.max(sortedPeaks[referenceIndex] ?? 0, 0.01); + return peaks.map( + (peak) => Math.round(Math.sqrt(Math.min(1, peak / referencePeak)) * 1_000) / 1_000, + ); +} + +function hasAudibleSamples(pcm: Buffer): boolean { + for (let offset = 0; offset + 1 < pcm.byteLength; offset += 2) { + if (Math.abs(pcm.readInt16LE(offset)) >= 164) return true; + } + return false; +} + +export async function generateSpeechWaveform( + options: SpeechWaveformOptions, + execute: RunFfmpeg = runFfmpeg, +): Promise { + try { + const centerPcm = await execute(buildSpeechWaveformArgs(options, 'center')); + if (hasAudibleSamples(centerPcm)) return computeWaveformPeaks(centerPcm); + } catch { + // Sources without a named center channel can reject the center-only filter. + } + + const downmixPcm = await execute(buildSpeechWaveformArgs(options, 'downmix')); + return computeWaveformPeaks(downmixPcm); +} diff --git a/src/main.ts b/src/main.ts index feaf49e0..5edefc18 100644 --- a/src/main.ts +++ b/src/main.ts @@ -463,6 +463,7 @@ import { handleCliCommandRuntimeServiceWithContext } from './main/cli-runtime'; import { createOverlayModalRuntimeService } from './main/overlay-runtime'; import { createOverlayModalInputState } from './main/runtime/overlay-modal-input-state'; import { MediaTimingPreviewSession } from './core/services/media-timing-preview'; +import { generateSpeechWaveform } from './core/services/media-timing-waveform'; import { createMediaTimingReviewRuntime } from './main/runtime/media-timing-review'; import { openMediaTimingReviewModal } from './main/runtime/media-timing-review-open'; import { openYoutubeTrackPicker } from './main/runtime/youtube-picker-open'; @@ -2807,6 +2808,7 @@ const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({ getMpvExecutablePath: () => configService.getConfig().mpv.executablePath || process.env.SUBMINER_MPV_PATH?.trim() || '', createPreviewSession: () => new MediaTimingPreviewSession(), + generateWaveform: (options) => generateSpeechWaveform(options), openModal: (payload) => openMediaTimingReviewModal(createOverlayHostedModalOpenDeps(), payload), showStatus: (message) => overlayNotificationsRuntime.showConfiguredStatusNotification(message, { variant: 'warning' }), @@ -5440,6 +5442,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({ }, mainDeps: { previewMediaTimingReview: (request) => mediaTimingReviewRuntime.previewRange(request), + getMediaTimingReviewWaveform: (request) => mediaTimingReviewRuntime.getWaveform(request), stopMediaTimingReviewPreview: (reviewId) => mediaTimingReviewRuntime.stopPreview(reviewId), resolveMediaTimingReview: (request) => mediaTimingReviewRuntime.resolveReview(request), getMainWindow: () => overlayManager.getMainWindow(), diff --git a/src/main/dependencies.ts b/src/main/dependencies.ts index 183a5025..e6f11adb 100644 --- a/src/main/dependencies.ts +++ b/src/main/dependencies.ts @@ -63,6 +63,7 @@ export interface MainIpcRuntimeServiceDepsParams { handleOverlayNotificationAction?: IpcDepsRuntimeOptions['handleOverlayNotificationAction']; onYoutubePickerResolve: IpcDepsRuntimeOptions['onYoutubePickerResolve']; previewMediaTimingReview?: IpcDepsRuntimeOptions['previewMediaTimingReview']; + getMediaTimingReviewWaveform?: IpcDepsRuntimeOptions['getMediaTimingReviewWaveform']; stopMediaTimingReviewPreview?: IpcDepsRuntimeOptions['stopMediaTimingReviewPreview']; resolveMediaTimingReview?: IpcDepsRuntimeOptions['resolveMediaTimingReview']; openYomitanSettings: IpcDepsRuntimeOptions['openYomitanSettings']; @@ -260,6 +261,7 @@ export function createMainIpcRuntimeServiceDeps( handleOverlayNotificationAction: params.handleOverlayNotificationAction, onYoutubePickerResolve: params.onYoutubePickerResolve, previewMediaTimingReview: params.previewMediaTimingReview, + getMediaTimingReviewWaveform: params.getMediaTimingReviewWaveform, stopMediaTimingReviewPreview: params.stopMediaTimingReviewPreview, resolveMediaTimingReview: params.resolveMediaTimingReview, openYomitanSettings: params.openYomitanSettings, diff --git a/src/main/runtime/media-timing-review.test.ts b/src/main/runtime/media-timing-review.test.ts index 5568b701..60049de5 100644 --- a/src/main/runtime/media-timing-review.test.ts +++ b/src/main/runtime/media-timing-review.test.ts @@ -79,6 +79,7 @@ test('media timing review pauses playback, resolves exact timing, and restores p }), getCurrentMediaPath: () => '/video/show.mkv', getMpvExecutablePath: () => 'mpv', + generateWaveform: async () => [], createPreviewSession: () => ({ start: async () => undefined, play: async (startTime, endTime) => { @@ -125,6 +126,69 @@ test('media timing review pauses playback, resolves exact timing, and restores p assert.deepEqual(previewCalls, [[9.5, 12.5]]); }); +test('media timing review analyzes the visible range on the selected audio stream', async () => { + const waveformCalls: Array<{ + mediaPath: string; + startTime: number; + endTime: number; + audioStreamIndex?: number; + }> = []; + let runtime: ReturnType; + runtime = createMediaTimingReviewRuntime({ + getMpvClient: () => ({ + connected: true, + currentVideoPath: '/video/show.mkv', + currentAudioStreamIndex: 4, + requestProperty: async (name) => (name === 'duration' ? 100 : name === 'pause' ? true : null), + send: () => undefined, + }), + getCurrentMediaPath: () => '/video/show.mkv', + getMpvExecutablePath: () => 'mpv', + generateWaveform: async (options) => { + waveformCalls.push(options); + return [0.1, 0.8, 0.2]; + }, + createPreviewSession: () => ({ + start: async () => undefined, + play: async () => undefined, + stop: async () => undefined, + dispose: () => undefined, + }), + openModal: async (payload) => { + const waveform = await runtime.getWaveform({ + reviewId: payload.reviewId, + startTime: payload.timelineStartTime, + endTime: payload.timelineEndTime, + }); + assert.deepEqual(waveform, { ok: true, peaks: [0.1, 0.8, 0.2] }); + runtime.resolveReview({ + reviewId: payload.reviewId, + decision: { action: 'use-original' }, + }); + return true; + }, + showStatus: () => undefined, + }); + + await runtime.requestReview({ + kind: 'sentence', + text: '字幕', + startTime: 10, + endTime: 12, + audioPadding: 0.5, + maxMediaDuration: 30, + }); + + assert.deepEqual(waveformCalls, [ + { + mediaPath: '/video/show.mkv', + startTime: 7.5, + endTime: 14.5, + audioStreamIndex: 4, + }, + ]); +}); + test('media timing review does not resume playback when the prior state is unavailable', async () => { const commands: Array> = []; let runtime: ReturnType; @@ -137,6 +201,7 @@ test('media timing review does not resume playback when the prior state is unava }), getCurrentMediaPath: () => '/video/show.mkv', getMpvExecutablePath: () => '', + generateWaveform: async () => [], createPreviewSession: () => ({ start: async () => { throw new Error('preview unavailable'); @@ -182,6 +247,7 @@ test('media timing review restores playback when setup fails after pausing', asy }), getCurrentMediaPath: () => '/video/show.mkv', getMpvExecutablePath: () => 'mpv', + generateWaveform: async () => [], createPreviewSession: () => { throw new Error('preview setup failed'); }, @@ -217,6 +283,7 @@ test('disposing an open review settles it with original timing and restores play }), getCurrentMediaPath: () => '/video/show.mkv', getMpvExecutablePath: () => 'mpv', + generateWaveform: async () => [], createPreviewSession: () => ({ start: async () => undefined, play: async () => undefined, diff --git a/src/main/runtime/media-timing-review.ts b/src/main/runtime/media-timing-review.ts index 989ba1a9..1eda8e6c 100644 --- a/src/main/runtime/media-timing-review.ts +++ b/src/main/runtime/media-timing-review.ts @@ -6,13 +6,17 @@ import type { MediaTimingReviewPreviewRequest, MediaTimingReviewRequest, MediaTimingReviewResolveRequest, + MediaTimingReviewWaveformRequest, + MediaTimingReviewWaveformResult, } from '../../types/anki'; +import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform'; const INITIAL_TIMELINE_MARGIN_SECONDS = 2; interface ReviewMpvClient { connected: boolean; currentVideoPath: string; + currentAudioStreamIndex?: number | null; requestProperty?: (name: string) => Promise; send: (payload: { command: Array }) => void; } @@ -31,6 +35,8 @@ interface PreviewSession { interface ActiveReview { payload: MediaTimingReviewOpenPayload; + mediaPath: string; + audioStreamIndex?: number; mpvClient: ReviewMpvClient; restorePlayback: boolean; preview: Promise; @@ -42,6 +48,7 @@ export interface MediaTimingReviewRuntimeDeps { getCurrentMediaPath: () => string | null; getMpvExecutablePath: () => string; createPreviewSession: () => PreviewSession; + generateWaveform: (options: SpeechWaveformOptions) => Promise; openModal: (payload: MediaTimingReviewOpenPayload) => Promise; showStatus: (message: string) => void; } @@ -152,6 +159,11 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep }); active = { payload, + mediaPath, + ...(mpvClient.currentAudioStreamIndex !== null && + mpvClient.currentAudioStreamIndex !== undefined + ? { audioStreamIndex: mpvClient.currentAudioStreamIndex } + : {}), mpvClient, restorePlayback: pendingPauseRestore === mpvClient, preview, @@ -222,6 +234,42 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep } } + async function getWaveform( + request: MediaTimingReviewWaveformRequest, + ): Promise { + const current = active; + if (!current || request.reviewId !== current.payload.reviewId) { + return { ok: false, message: 'This timing review is no longer active.' }; + } + if ( + !Number.isFinite(request.startTime) || + !Number.isFinite(request.endTime) || + request.startTime < 0 || + request.endTime <= request.startTime || + (current.payload.mediaDuration !== undefined && + request.endTime > current.payload.mediaDuration + 0.001) + ) { + return { ok: false, message: 'The waveform range is invalid.' }; + } + + try { + const peaks = await deps.generateWaveform({ + mediaPath: current.mediaPath, + startTime: request.startTime, + endTime: request.endTime, + ...(current.audioStreamIndex !== undefined + ? { audioStreamIndex: current.audioStreamIndex } + : {}), + }); + if (active !== current || peaks.length < 2 || peaks.some((peak) => !Number.isFinite(peak))) { + return { ok: false, message: 'Timing waveform is unavailable.' }; + } + return { ok: true, peaks }; + } catch { + return { ok: false, message: 'Timing waveform is unavailable.' }; + } + } + async function stopPreview(reviewId: string): Promise { const current = active; if (!current || reviewId !== current.payload.reviewId) { @@ -282,6 +330,7 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep return { requestReview, previewRange, + getWaveform, stopPreview, resolveReview, dispose, diff --git a/src/preload.ts b/src/preload.ts index f36a5a3a..9615f989 100644 --- a/src/preload.ts +++ b/src/preload.ts @@ -72,6 +72,7 @@ import type { MediaTimingReviewOpenPayload, MediaTimingReviewPreviewRequest, MediaTimingReviewResolveRequest, + MediaTimingReviewWaveformRequest, } from './types'; import { IPC_CHANNELS } from './shared/ipc/contracts'; @@ -469,6 +470,8 @@ const electronAPI: ElectronAPI = { onOpenMediaTimingReview: onOpenMediaTimingReviewEvent, previewMediaTimingReview: (request: MediaTimingReviewPreviewRequest) => ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewPreview, request), + getMediaTimingReviewWaveform: (request: MediaTimingReviewWaveformRequest) => + ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewWaveform, request), stopMediaTimingReviewPreview: (reviewId: string) => ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewStopPreview, reviewId), resolveMediaTimingReview: (request: MediaTimingReviewResolveRequest) => diff --git a/src/renderer/index.html b/src/renderer/index.html index 217195bf..ff17628c 100644 --- a/src/renderer/index.html +++ b/src/renderer/index.html @@ -208,13 +208,11 @@ aria-labelledby="mediaTimingReviewTitle" >
-
-
- Sentence card -
+
Review media timing
+
Sentence card
- Drag either edge to set the exact clip. + Drag an edge to trim, or drag the middle to slide the clip.
@@ -293,14 +320,14 @@ type="button" aria-label="Move start backward 100 milliseconds" > - − 0.1s + −0.1s
@@ -310,14 +337,14 @@ type="button" aria-label="Move end backward 100 milliseconds" > - − 0.1s + −0.1s
@@ -335,7 +362,8 @@ class="media-timing-review-play-button" type="button" > - Play selection + + Play selection