feat(mining): add a screenshot frame picker to media review (#254)

src/main/dependencies.ts

@@ -63,6 +63,7 @@ export interface MainIpcRuntimeServiceDepsParams {
   handleOverlayNotificationAction?: IpcDepsRuntimeOptions['handleOverlayNotificationAction'];
   onYoutubePickerResolve: IpcDepsRuntimeOptions['onYoutubePickerResolve'];
   previewMediaTimingReview?: IpcDepsRuntimeOptions['previewMediaTimingReview'];
+  getMediaTimingReviewFrame?: IpcDepsRuntimeOptions['getMediaTimingReviewFrame'];
   getMediaTimingReviewWaveform?: IpcDepsRuntimeOptions['getMediaTimingReviewWaveform'];
   stopMediaTimingReviewPreview?: IpcDepsRuntimeOptions['stopMediaTimingReviewPreview'];
   resolveMediaTimingReview?: IpcDepsRuntimeOptions['resolveMediaTimingReview'];
@@ -263,6 +264,7 @@ export function createMainIpcRuntimeServiceDeps(
     handleOverlayNotificationAction: params.handleOverlayNotificationAction,
     onYoutubePickerResolve: params.onYoutubePickerResolve,
     previewMediaTimingReview: params.previewMediaTimingReview,
+    getMediaTimingReviewFrame: params.getMediaTimingReviewFrame,
     getMediaTimingReviewWaveform: params.getMediaTimingReviewWaveform,
     stopMediaTimingReviewPreview: params.stopMediaTimingReviewPreview,
     resolveMediaTimingReview: params.resolveMediaTimingReview,

src/main/runtime/media-timing-review-frame.test.ts

@@ -0,0 +1,161 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import type { MediaTimingReviewOpenPayload } from '../../types/anki';
+import type { MediaTimingFrameOptions } from '../../core/services/media-timing-frame';
+import {
+  createMediaTimingReviewRuntime,
+  type MediaTimingReviewRuntimeDeps,
+} from './media-timing-review';
+
+async function start(
+  options: Partial<MediaTimingReviewRuntimeDeps> = {},
+  screenshotEnabled = true,
+) {
+  let opened!: (payload: MediaTimingReviewOpenPayload) => void;
+  const payloadPromise = new Promise<MediaTimingReviewOpenPayload>((resolve) => {
+    opened = resolve;
+  });
+  const frames: MediaTimingFrameOptions[] = [];
+  const runtime = createMediaTimingReviewRuntime({
+    getMpvClient: () => ({
+      connected: true,
+      currentVideoPath: '/video.mkv',
+      send: () => {},
+      requestProperty: async (name) => (name === 'duration' ? 100 : true),
+    }),
+    getCurrentMediaPath: () => '/video.mkv',
+    getMpvExecutablePath: () => '',
+    createPreviewSession: () => ({
+      start: async () => {},
+      play: async () => {},
+      stop: async () => {},
+      onPlaybackEnded: () => {},
+      dispose: () => {},
+    }),
+    generateWaveform: async () => [0, 1],
+    resolveVideoSource: async () => ({ path: '/video.mkv' }),
+    generateFrame: async (options) => {
+      frames.push(options);
+      return { dataUrl: 'data:image/jpeg;base64,AA==', timestamp: options.timestamp };
+    },
+    openModal: async (payload) => {
+      opened(payload);
+      return true;
+    },
+    showStatus: () => {},
+    ...options,
+  });
+  const decision = runtime.requestReview({
+    kind: 'word',
+    text: '字幕',
+    startTime: 10,
+    endTime: 12,
+    audioPadding: 0,
+    maxMediaDuration: 30,
+    screenshotEnabled,
+  });
+  return { runtime, decision, payload: await payloadPromise, frames };
+}
+
+test('review accepts a screenshot outside the audio range and validates media bounds', async () => {
+  const { runtime, decision, payload, frames } = await start();
+  for (const timestamp of [NaN, Infinity, -1, 100]) {
+    assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp })).ok, false);
+    assert.equal(
+      runtime.resolveReview({
+        reviewId: payload.reviewId,
+        decision: { action: 'confirm', startTime: 10, endTime: 11, screenshotTime: timestamp },
+      }).ok,
+      false,
+    );
+  }
+  assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 13 })).ok, true);
+  assert.equal(frames[0]?.timestamp, 13);
+  assert.equal(
+    runtime.resolveReview({
+      reviewId: payload.reviewId,
+      decision: { action: 'confirm', startTime: 10, endTime: 11, screenshotTime: 13 },
+    }).ok,
+    true,
+  );
+  assert.deepEqual(await decision, {
+    action: 'confirm',
+    startTime: 10,
+    endTime: 11,
+    screenshotTime: 13,
+  });
+});
+
+test('screenshot preview reuses the remote audio window and its absolute timestamps', async () => {
+  let downloads = 0;
+  const remote = 'https://example.test/video';
+  const { runtime, payload, decision, frames } = await start({
+    resolveMediaSource: async () => ({ path: remote }),
+    resolveVideoSource: async () => ({ path: remote }),
+    acquireMediaWindow: async () => {
+      downloads += 1;
+      return {
+        path: '/window.mkv',
+        sourcePath: remote,
+        startTime: 7,
+        endTime: 15,
+        audioStreamIndex: null,
+        media: { path: '/window.mkv', absoluteTimestamps: true },
+      };
+    },
+  });
+  await runtime.getWaveform({ reviewId: payload.reviewId, startTime: 8, endTime: 14 });
+  assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 })).ok, true);
+  assert.equal(downloads, 1);
+  assert.deepEqual(frames[0]?.media, { path: '/window.mkv', absoluteTimestamps: true });
+  await runtime.dispose();
+  await decision;
+});
+
+test('split video streams never read an audio-only cache as a video source', async () => {
+  const video = {
+    path: 'https://example.test/video',
+    inputOptions: { headers: { 'X-Test': 'value' } },
+  };
+  const { runtime, payload, decision, frames } = await start({
+    resolveMediaSource: async () => ({ path: 'https://example.test/audio' }),
+    resolveVideoSource: async () => video,
+  });
+  await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 });
+  assert.deepEqual(frames[0]?.media, video);
+  await runtime.dispose();
+  await decision;
+});
+
+test('disabled screenshots and missing video inputs do not trigger extraction', async () => {
+  for (const [enabled, options] of [
+    [false, {}],
+    [true, { resolveVideoSource: async () => null }],
+  ] as const) {
+    const { runtime, payload, decision, frames } = await start(options, enabled);
+    assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 })).ok, false);
+    assert.equal(frames.length, 0);
+    await runtime.dispose();
+    await decision;
+  }
+});
+
+test('closing a review invalidates in-flight frame results and clears the frame index', async () => {
+  let finish!: (value: { dataUrl: string; timestamp: number }) => void;
+  let clearCount = 0;
+  const { runtime, payload, decision } = await start({
+    generateFrame: () =>
+      new Promise((resolve) => {
+        finish = resolve;
+      }),
+    clearFrameCache: () => {
+      clearCount += 1;
+    },
+  });
+  const frame = runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 });
+  await runtime.dispose();
+  finish({ dataUrl: 'data:image/jpeg;base64,AA==', timestamp: 11 });
+  assert.equal((await frame).stale, true);
+  await decision;
+  assert.equal(clearCount, 1);
+});

src/main/runtime/media-timing-review.ts

@@ -7,10 +7,13 @@ import type {
   MediaTimingReviewPreviewRequest,
   MediaTimingReviewRequest,
   MediaTimingReviewResolveRequest,
+  MediaTimingReviewFrameRequest,
+  MediaTimingReviewFrameResult,
   MediaTimingReviewWaveformRequest,
   MediaTimingReviewWaveformResult,
 } from '../../types/anki';
 import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform';
+import type { MediaTimingFrameOptions } from '../../core/services/media-timing-frame';
 import {
   isRemoteMediaWindowSourcePath,
   type RemoteMediaWindow,
@@ -59,6 +62,8 @@ interface ActiveReview {
   mediaPath: string;
   /** What the waveform reads when no cached window is available. */
   waveformMedia: MediaInput;
+  videoSource: ReviewMediaSource | null;
+  frameInFlight: boolean;
   audioStreamIndex?: number;
   /** Remote source to download windows of; null for local media or without a cache. */
   windowSource: RemoteMediaWindowSource | null;
@@ -81,6 +86,11 @@ export interface MediaTimingReviewRuntimeDeps {
   generateWaveform: (options: SpeechWaveformOptions) => Promise<number[]>;
   /** Resolves the FFmpeg-readable stream URL and headers behind the current media path. */
   resolveMediaSource?: () => Promise<ReviewMediaSource | null>;
+  resolveVideoSource?: () => Promise<ReviewMediaSource | null>;
+  generateFrame?: (
+    options: MediaTimingFrameOptions,
+  ) => Promise<{ dataUrl: string; timestamp: number }>;
+  clearFrameCache?: () => void;
   /** Downloads (or reuses) a local window of a remote source covering the range. */
   acquireMediaWindow?: (
     source: RemoteMediaWindowSource,
@@ -227,6 +237,9 @@ export function buildMediaTimingReviewPayload(
     timelineEndTime,
     ...(duration !== null && duration > 0 ? { mediaDuration: duration } : {}),
     maxMediaDuration,
+    ...(request.screenshotEnabled !== undefined
+      ? { screenshotEnabled: request.screenshotEnabled }
+      : {}),
   };
 }
 
@@ -335,13 +348,15 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
       return { action: 'use-original' };
     }
 
-    const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw, resolvedSource] = await Promise.all([
-      mpvClient.requestProperty?.('pause').catch(() => null) ?? null,
-      mpvClient.requestProperty?.('duration').catch(() => null) ?? null,
-      mpvClient.requestProperty?.('aid').catch(() => null) ?? null,
-      mpvClient.requestProperty?.('volume').catch(() => null) ?? null,
-      deps.resolveMediaSource?.().catch(() => null) ?? null,
-    ]);
+    const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw, resolvedSource, videoSource] =
+      await Promise.all([
+        mpvClient.requestProperty?.('pause').catch(() => null) ?? null,
+        mpvClient.requestProperty?.('duration').catch(() => null) ?? null,
+        mpvClient.requestProperty?.('aid').catch(() => null) ?? null,
+        mpvClient.requestProperty?.('volume').catch(() => null) ?? null,
+        deps.resolveMediaSource?.().catch(() => null) ?? null,
+        request.screenshotEnabled ? (deps.resolveVideoSource?.().catch(() => null) ?? null) : null,
+      ]);
     const pauseState = booleanProperty(pauseRaw);
     mpvClient.send({ command: ['set_property', 'pause', 'yes'] });
     pendingPauseRestore = pauseState === false ? mpvClient : null;
@@ -383,6 +398,8 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
       payload,
       mediaPath,
       waveformMedia: inputOptions ? { path: sourcePath, inputOptions } : sourcePath,
+      videoSource,
+      frameInFlight: false,
       ...(audioStreamIndex !== undefined ? { audioStreamIndex } : {}),
       windowSource,
       window: null,
@@ -527,6 +544,58 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
     }
   }
 
+  async function getFrame(
+    request: MediaTimingReviewFrameRequest,
+  ): Promise<MediaTimingReviewFrameResult> {
+    const current = active;
+    if (!current || request.reviewId !== current.payload.reviewId) return staleReviewResult();
+    if (!current.payload.screenshotEnabled || !deps.generateFrame || !current.videoSource) {
+      return { ok: false, message: 'Screenshot preview is unavailable for this media.' };
+    }
+    if (
+      !Number.isFinite(request.timestamp) ||
+      request.timestamp < 0 ||
+      (current.payload.mediaDuration !== undefined &&
+        request.timestamp >= current.payload.mediaDuration) ||
+      (request.direction !== undefined && request.direction !== -1 && request.direction !== 1)
+    ) {
+      return { ok: false, message: 'The screenshot time is invalid.' };
+    }
+    if (current.frameInFlight)
+      return { ok: false, message: 'A screenshot preview is already loading.' };
+    current.frameInFlight = true;
+    try {
+      // Reuse the audio window only when it contains this same video source (not split streams).
+      const range = {
+        startTime: Math.max(0, Math.min(current.payload.timelineStartTime, request.timestamp - 2)),
+        endTime: Math.min(
+          current.payload.mediaDuration ?? Infinity,
+          Math.max(current.payload.timelineEndTime, request.timestamp + 2),
+        ),
+      };
+      const window =
+        current.windowSource?.path === current.videoSource.path
+          ? await ensureWindow(current, range)
+          : null;
+      if (active !== current) return staleReviewResult();
+      const frame = await deps.generateFrame({
+        media: window?.media ?? current.videoSource,
+        timestamp: request.timestamp,
+        ...(request.direction !== undefined ? { direction: request.direction } : {}),
+      });
+      if (active !== current) return staleReviewResult();
+      return { ok: true, ...frame };
+    } catch {
+      if (active !== current) return staleReviewResult();
+      return {
+        ok: false,
+        message: 'Screenshot preview unavailable. Try another time or reset to the midpoint.',
+      };
+    } finally {
+      current.frameInFlight = false;
+    }
+  }
+
   async function stopPreview(reviewId: string): Promise<MediaTimingReviewActionResult> {
     const current = active;
     if (!current || reviewId !== current.payload.reviewId) {
@@ -550,13 +619,23 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
       return staleReviewResult();
     }
     if (request.decision.action === 'confirm') {
-      const { startTime, endTime, text } = request.decision;
+      const { startTime, endTime, text, screenshotTime } = request.decision;
       if (!isValidMediaTimingRange(current.payload, startTime, endTime)) {
         return { ok: false, message: 'The selected timing range is invalid.' };
       }
       if (text !== undefined && (typeof text !== 'string' || text.trim().length === 0)) {
         return { ok: false, message: 'The combined sentence text is invalid.' };
       }
+      if (
+        screenshotTime !== undefined &&
+        (!current.payload.screenshotEnabled ||
+          !Number.isFinite(screenshotTime) ||
+          screenshotTime < 0 ||
+          (current.payload.mediaDuration !== undefined &&
+            screenshotTime >= current.payload.mediaDuration))
+      ) {
+        return { ok: false, message: 'The screenshot time is invalid.' };
+      }
     }
     current.resolve(request.decision);
     return { ok: true };
@@ -566,6 +645,7 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
     const current = active;
     active = null;
     if (!current) return;
+    deps.clearFrameCache?.();
     void current.preview?.session.then((session) => session.dispose()).catch(() => {});
     if (current.restorePlayback && current.mpvClient.connected) {
       current.mpvClient.send({ command: ['set_property', 'pause', 'no'] });
@@ -582,6 +662,7 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
     requestReview,
     previewRange,
     getWaveform,
+    getFrame,
     stopPreview,
     resolveReview,
     dispose,