feat(anki): add waveform-guided timing controls

- Add speech-weighted waveform analysis and playback playhead
- Support dragging, sliding, and keyboard nudging for clip timing

src/core/services/ipc.ts

@@ -23,6 +23,8 @@ import type {
   MediaTimingReviewActionResult,
   MediaTimingReviewPreviewRequest,
   MediaTimingReviewResolveRequest,
+  MediaTimingReviewWaveformRequest,
+  MediaTimingReviewWaveformResult,
 } from '../../types/anki';
 import { IPC_CHANNELS, type OverlayHostedModal } from '../../shared/ipc/contracts';
 import {
@@ -106,6 +108,9 @@ export interface IpcServiceDeps {
   previewMediaTimingReview?: (
     request: MediaTimingReviewPreviewRequest,
   ) => Promise<MediaTimingReviewActionResult>;
+  getMediaTimingReviewWaveform?: (
+    request: MediaTimingReviewWaveformRequest,
+  ) => Promise<MediaTimingReviewWaveformResult>;
   stopMediaTimingReviewPreview?: (reviewId: string) => Promise<MediaTimingReviewActionResult>;
   resolveMediaTimingReview?: (
     request: MediaTimingReviewResolveRequest,
@@ -255,6 +260,12 @@ function parseMediaTimingReviewPreviewRequest(
   };
 }
 
+function parseMediaTimingReviewWaveformRequest(
+  payload: unknown,
+): MediaTimingReviewWaveformRequest | null {
+  return parseMediaTimingReviewPreviewRequest(payload);
+}
+
 function parseMediaTimingReviewResolveRequest(
   payload: unknown,
 ): MediaTimingReviewResolveRequest | null {
@@ -343,6 +354,7 @@ export interface IpcDepsRuntimeOptions {
     request: YoutubePickerResolveRequest,
   ) => Promise<YoutubePickerResolveResult>;
   previewMediaTimingReview?: IpcServiceDeps['previewMediaTimingReview'];
+  getMediaTimingReviewWaveform?: IpcServiceDeps['getMediaTimingReviewWaveform'];
   stopMediaTimingReviewPreview?: IpcServiceDeps['stopMediaTimingReviewPreview'];
   resolveMediaTimingReview?: IpcServiceDeps['resolveMediaTimingReview'];
   getAnkiConnectStatus: () => boolean;
@@ -439,6 +451,7 @@ export function createIpcDepsRuntime(options: IpcDepsRuntimeOptions): IpcService
     runSubsyncManual: options.runSubsyncManual,
     onYoutubePickerResolve: options.onYoutubePickerResolve,
     previewMediaTimingReview: options.previewMediaTimingReview,
+    getMediaTimingReviewWaveform: options.getMediaTimingReviewWaveform,
     stopMediaTimingReviewPreview: options.stopMediaTimingReviewPreview,
     resolveMediaTimingReview: options.resolveMediaTimingReview,
     getAnkiConnectStatus: options.getAnkiConnectStatus,
@@ -572,6 +585,16 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar
       return await deps.previewMediaTimingReview(request);
     },
   );
+  ipc.handle(
+    IPC_CHANNELS.request.mediaTimingReviewWaveform,
+    async (_event: unknown, payload: unknown) => {
+      const request = parseMediaTimingReviewWaveformRequest(payload);
+      if (!request || !deps.getMediaTimingReviewWaveform) {
+        return { ok: false, message: 'Timing waveform is unavailable.' };
+      }
+      return await deps.getMediaTimingReviewWaveform(request);
+    },
+  );
   ipc.handle(
     IPC_CHANNELS.request.mediaTimingReviewStopPreview,
     async (_event: unknown, reviewId: unknown) => {

src/core/services/media-timing-waveform.test.ts

@@ -0,0 +1,74 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import {
+  buildSpeechWaveformArgs,
+  computeWaveformPeaks,
+  generateSpeechWaveform,
+} from './media-timing-waveform';
+
+function pcm(samples: number[]): Buffer {
+  const result = Buffer.alloc(samples.length * 2);
+  samples.forEach((sample, index) => result.writeInt16LE(sample, index * 2));
+  return result;
+}
+
+test('speech waveform maps the selected FFmpeg stream and visible range', () => {
+  const args = buildSpeechWaveformArgs(
+    {
+      mediaPath: '/video/show.mkv',
+      startTime: 8,
+      endTime: 15,
+      audioStreamIndex: 3,
+    },
+    'center',
+  );
+
+  assert.deepEqual(args.slice(args.indexOf('-ss'), args.indexOf('-t') + 2), [
+    '-ss',
+    '8',
+    '-i',
+    '/video/show.mkv',
+    '-t',
+    '7',
+  ]);
+  assert.deepEqual(args.slice(args.indexOf('-map'), args.indexOf('-map') + 2), ['-map', '0:3']);
+  assert.match(args[args.indexOf('-af') + 1] ?? '', /c0=FC/);
+});
+
+test('waveform peaks are normalized without flattening quieter sections', () => {
+  const peaks = computeWaveformPeaks(pcm([0, 1_000, -2_000, 4_000, -8_000, 16_000]), 3);
+
+  assert.equal(peaks.length, 3);
+  assert.ok((peaks[0] ?? 0) > 0);
+  assert.ok((peaks[0] ?? 0) < (peaks[1] ?? 0));
+  assert.ok((peaks[1] ?? 0) < (peaks[2] ?? 0));
+  assert.equal(peaks[2], 1);
+});
+
+test('speech waveform uses a mono downmix when the source has no center activity', async () => {
+  const calls: string[][] = [];
+  const peaks = await generateSpeechWaveform(
+    { mediaPath: '/video/show.mkv', startTime: 0, endTime: 2 },
+    async (args) => {
+      calls.push(args);
+      return calls.length === 1 ? pcm([0, 0, 0, 0]) : pcm([0, 4_000, -8_000, 16_000]);
+    },
+  );
+
+  assert.equal(calls.length, 2);
+  assert.match(calls[1]?.[calls[1].indexOf('-af') + 1] ?? '', /channel_layouts=mono/);
+  assert.equal(Math.max(...peaks), 1);
+});
+
+test('speech waveform keeps an active center channel without doing a second decode', async () => {
+  let calls = 0;
+  await generateSpeechWaveform(
+    { mediaPath: '/video/show.mkv', startTime: 0, endTime: 2 },
+    async () => {
+      calls += 1;
+      return pcm([0, 4_000, -8_000, 16_000]);
+    },
+  );
+
+  assert.equal(calls, 1);
+});

src/core/services/media-timing-waveform.ts

@@ -0,0 +1,158 @@
+import { spawn } from 'node:child_process';
+
+const WAVEFORM_SAMPLE_RATE = 8_000;
+const WAVEFORM_POINT_COUNT = 480;
+const WAVEFORM_TIMEOUT_MS = 15_000;
+const MAX_WAVEFORM_BYTES = 16 * 1024 * 1024;
+const SPEECH_FILTER = 'highpass=f=120,lowpass=f=4000';
+const CENTER_CHANNEL_FILTER = `pan=mono|c0=FC,${SPEECH_FILTER}`;
+const DOWNMIX_FILTER = `aformat=channel_layouts=mono,${SPEECH_FILTER}`;
+
+export interface SpeechWaveformOptions {
+  mediaPath: string;
+  startTime: number;
+  endTime: number;
+  audioStreamIndex?: number;
+}
+
+type RunFfmpeg = (args: string[]) => Promise<Buffer>;
+
+export function buildSpeechWaveformArgs(
+  options: SpeechWaveformOptions,
+  mode: 'center' | 'downmix',
+): string[] {
+  const duration = options.endTime - options.startTime;
+  const args = [
+    '-hide_banner',
+    '-nostdin',
+    '-loglevel',
+    'error',
+    '-ss',
+    String(options.startTime),
+    '-i',
+    options.mediaPath,
+    '-t',
+    String(duration),
+  ];
+  if (
+    options.audioStreamIndex !== undefined &&
+    Number.isInteger(options.audioStreamIndex) &&
+    options.audioStreamIndex >= 0
+  ) {
+    args.push('-map', `0:${options.audioStreamIndex}`);
+  }
+  args.push(
+    '-vn',
+    '-sn',
+    '-dn',
+    '-af',
+    mode === 'center' ? CENTER_CHANNEL_FILTER : DOWNMIX_FILTER,
+    '-ac',
+    '1',
+    '-ar',
+    String(WAVEFORM_SAMPLE_RATE),
+    '-f',
+    's16le',
+    'pipe:1',
+  );
+  return args;
+}
+
+function runFfmpeg(args: string[]): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    const child = spawn('ffmpeg', args, { stdio: ['ignore', 'pipe', 'pipe'] });
+    const chunks: Buffer[] = [];
+    let byteLength = 0;
+    let stderr = '';
+    let settled = false;
+    const timeout = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      child.kill('SIGKILL');
+      reject(new Error(`FFmpeg waveform analysis timed out after ${WAVEFORM_TIMEOUT_MS}ms`));
+    }, WAVEFORM_TIMEOUT_MS);
+
+    const settle = (callback: () => void): void => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      callback();
+    };
+
+    child.stdout.on('data', (chunk: Buffer) => {
+      if (settled) return;
+      byteLength += chunk.byteLength;
+      if (byteLength > MAX_WAVEFORM_BYTES) {
+        settle(() => {
+          child.kill('SIGKILL');
+          reject(new Error('The visible waveform range is too large to analyze.'));
+        });
+        return;
+      }
+      chunks.push(chunk);
+    });
+    child.stderr.setEncoding('utf8');
+    child.stderr.on('data', (chunk) => {
+      if (stderr.length < 4_000) stderr += String(chunk);
+    });
+    child.once('error', (error) => settle(() => reject(error)));
+    child.once('close', (code) => {
+      settle(() => {
+        if (code === 0) {
+          resolve(Buffer.concat(chunks, byteLength));
+          return;
+        }
+        reject(new Error(stderr.trim() || `FFmpeg exited with status ${code ?? 'unknown'}`));
+      });
+    });
+  });
+}
+
+export function computeWaveformPeaks(pcm: Buffer, pointCount = WAVEFORM_POINT_COUNT): number[] {
+  const sampleCount = Math.floor(pcm.byteLength / 2);
+  if (sampleCount === 0 || pointCount <= 0) return [];
+  const resolvedPointCount = Math.min(pointCount, sampleCount);
+  const peaks = Array.from({ length: resolvedPointCount }, () => 0);
+
+  for (let point = 0; point < resolvedPointCount; point += 1) {
+    const sampleStart = Math.floor((point * sampleCount) / resolvedPointCount);
+    const sampleEnd = Math.max(
+      sampleStart + 1,
+      Math.floor(((point + 1) * sampleCount) / resolvedPointCount),
+    );
+    let peak = 0;
+    for (let sample = sampleStart; sample < sampleEnd; sample += 1) {
+      peak = Math.max(peak, Math.abs(pcm.readInt16LE(sample * 2)) / 32_768);
+    }
+    peaks[point] = peak;
+  }
+
+  const sortedPeaks = [...peaks].sort((left, right) => left - right);
+  const referenceIndex = Math.min(sortedPeaks.length - 1, Math.floor(sortedPeaks.length * 0.95));
+  const referencePeak = Math.max(sortedPeaks[referenceIndex] ?? 0, 0.01);
+  return peaks.map(
+    (peak) => Math.round(Math.sqrt(Math.min(1, peak / referencePeak)) * 1_000) / 1_000,
+  );
+}
+
+function hasAudibleSamples(pcm: Buffer): boolean {
+  for (let offset = 0; offset + 1 < pcm.byteLength; offset += 2) {
+    if (Math.abs(pcm.readInt16LE(offset)) >= 164) return true;
+  }
+  return false;
+}
+
+export async function generateSpeechWaveform(
+  options: SpeechWaveformOptions,
+  execute: RunFfmpeg = runFfmpeg,
+): Promise<number[]> {
+  try {
+    const centerPcm = await execute(buildSpeechWaveformArgs(options, 'center'));
+    if (hasAudibleSamples(centerPcm)) return computeWaveformPeaks(centerPcm);
+  } catch {
+    // Sources without a named center channel can reject the center-only filter.
+  }
+
+  const downmixPcm = await execute(buildSpeechWaveformArgs(options, 'downmix'));
+  return computeWaveformPeaks(downmixPcm);
+}