feat(mining): add a screenshot frame picker to media review (#254)

src/core/services/ipc.test.ts

@@ -671,7 +671,34 @@ test('registerIpcHandlers accepts the keep-without-media timing decision', async
   assert.deepEqual(requests, [{ reviewId: 'review-1', decision: { action: 'skip-media' } }]);
 });
 
-test('registerIpcHandlers validates and forwards combined timing review text', async () => {
+test('frame IPC validates timestamps and directions', async () => {
+  const { registrar, handlers } = createFakeIpcRegistrar();
+  const requests: unknown[] = [];
+  registerIpcHandlers(
+    createRegisterIpcDeps({
+      getMediaTimingReviewFrame: async (request) => {
+        requests.push(request);
+        return { ok: true };
+      },
+    }),
+    registrar,
+  );
+  const frame = handlers.handle.get(IPC_CHANNELS.request.mediaTimingReviewFrame)!;
+  const valid = { reviewId: 'r', timestamp: 13, direction: 1 };
+  assert.deepEqual(await frame({}, valid), { ok: true });
+  for (const invalid of [
+    null,
+    {},
+    { ...valid, timestamp: NaN },
+    { ...valid, timestamp: '13' },
+    { ...valid, direction: 2 },
+  ]) {
+    assert.equal(((await frame({}, invalid)) as { ok: boolean }).ok, false);
+  }
+  assert.deepEqual(requests, [valid]);
+});
+
+test('registerIpcHandlers validates and forwards timing review text and screenshot selection', async () => {
   const { registrar, handlers } = createFakeIpcRegistrar();
   const requests: unknown[] = [];
   registerIpcHandlers(
@@ -696,6 +723,7 @@ test('registerIpcHandlers validates and forwards combined timing review text', a
           startTime: 10,
           endTime: 12,
           text: '前の行 対象の行',
+          screenshotTime: 13,
         },
       },
     ),
@@ -709,20 +737,23 @@ test('registerIpcHandlers validates and forwards combined timing review text', a
         startTime: 10,
         endTime: 12,
         text: '前の行 対象の行',
+        screenshotTime: 13,
       },
     },
   ]);
 
-  assert.deepEqual(
-    await handler!(
-      {},
-      {
-        reviewId: 'review-1',
-        decision: { action: 'confirm', startTime: 10, endTime: 12, text: '   ' },
-      },
-    ),
-    { ok: false, message: 'Timing review is unavailable.' },
-  );
+  for (const invalid of [{ text: '   ' }, { screenshotTime: Infinity }]) {
+    assert.deepEqual(
+      await handler!(
+        {},
+        {
+          reviewId: 'review-1',
+          decision: { action: 'confirm', startTime: 10, endTime: 12, ...invalid },
+        },
+      ),
+      { ok: false, message: 'Timing review is unavailable.' },
+    );
+  }
   assert.equal(requests.length, 1);
 });
 

src/core/services/ipc.ts

@@ -24,6 +24,8 @@ import type {
   MediaTimingReviewActionResult,
   MediaTimingReviewPreviewRequest,
   MediaTimingReviewResolveRequest,
+  MediaTimingReviewFrameRequest,
+  MediaTimingReviewFrameResult,
   MediaTimingReviewWaveformRequest,
   MediaTimingReviewWaveformResult,
 } from '../../types/anki';
@@ -111,6 +113,9 @@ export interface IpcServiceDeps {
   previewMediaTimingReview?: (
     request: MediaTimingReviewPreviewRequest,
   ) => Promise<MediaTimingReviewActionResult>;
+  getMediaTimingReviewFrame?: (
+    request: MediaTimingReviewFrameRequest,
+  ) => Promise<MediaTimingReviewFrameResult>;
   getMediaTimingReviewWaveform?: (
     request: MediaTimingReviewWaveformRequest,
   ) => Promise<MediaTimingReviewWaveformResult>;
@@ -269,6 +274,26 @@ function parseMediaTimingReviewWaveformRequest(
   return parseMediaTimingReviewPreviewRequest(payload);
 }
 
+function parseMediaTimingReviewFrameRequest(
+  payload: unknown,
+): MediaTimingReviewFrameRequest | null {
+  if (!payload || typeof payload !== 'object') return null;
+  const record = payload as Record<string, unknown>;
+  if (
+    typeof record.reviewId !== 'string' ||
+    !record.reviewId ||
+    typeof record.timestamp !== 'number' ||
+    !Number.isFinite(record.timestamp) ||
+    (record.direction !== undefined && record.direction !== -1 && record.direction !== 1)
+  )
+    return null;
+  return {
+    reviewId: record.reviewId,
+    timestamp: record.timestamp,
+    ...(record.direction !== undefined ? { direction: record.direction as -1 | 1 } : {}),
+  };
+}
+
 function parseMediaTimingReviewResolveRequest(
   payload: unknown,
 ): MediaTimingReviewResolveRequest | null {
@@ -291,6 +316,9 @@ function parseMediaTimingReviewResolveRequest(
     Number.isFinite(decisionRecord.startTime) &&
     typeof decisionRecord.endTime === 'number' &&
     Number.isFinite(decisionRecord.endTime) &&
+    (decisionRecord.screenshotTime === undefined ||
+      (typeof decisionRecord.screenshotTime === 'number' &&
+        Number.isFinite(decisionRecord.screenshotTime))) &&
     (decisionRecord.text === undefined ||
       (typeof decisionRecord.text === 'string' && decisionRecord.text.trim().length > 0))
   ) {
@@ -300,6 +328,9 @@ function parseMediaTimingReviewResolveRequest(
         action: 'confirm',
         startTime: decisionRecord.startTime,
         endTime: decisionRecord.endTime,
+        ...(decisionRecord.screenshotTime === undefined
+          ? {}
+          : { screenshotTime: decisionRecord.screenshotTime as number }),
         ...(decisionRecord.text === undefined ? {} : { text: decisionRecord.text }),
       },
     };
@@ -365,6 +396,7 @@ export interface IpcDepsRuntimeOptions {
     request: YoutubePickerResolveRequest,
   ) => Promise<YoutubePickerResolveResult>;
   previewMediaTimingReview?: IpcServiceDeps['previewMediaTimingReview'];
+  getMediaTimingReviewFrame?: IpcServiceDeps['getMediaTimingReviewFrame'];
   getMediaTimingReviewWaveform?: IpcServiceDeps['getMediaTimingReviewWaveform'];
   stopMediaTimingReviewPreview?: IpcServiceDeps['stopMediaTimingReviewPreview'];
   resolveMediaTimingReview?: IpcServiceDeps['resolveMediaTimingReview'];
@@ -463,6 +495,7 @@ export function createIpcDepsRuntime(options: IpcDepsRuntimeOptions): IpcService
     runSubsyncManual: options.runSubsyncManual,
     onYoutubePickerResolve: options.onYoutubePickerResolve,
     previewMediaTimingReview: options.previewMediaTimingReview,
+    getMediaTimingReviewFrame: options.getMediaTimingReviewFrame,
     getMediaTimingReviewWaveform: options.getMediaTimingReviewWaveform,
     stopMediaTimingReviewPreview: options.stopMediaTimingReviewPreview,
     resolveMediaTimingReview: options.resolveMediaTimingReview,
@@ -613,6 +646,16 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar
       return await deps.getMediaTimingReviewWaveform(request);
     },
   );
+  ipc.handle(
+    IPC_CHANNELS.request.mediaTimingReviewFrame,
+    async (_event: unknown, payload: unknown) => {
+      const request = parseMediaTimingReviewFrameRequest(payload);
+      if (!request || !deps.getMediaTimingReviewFrame) {
+        return { ok: false, message: 'Screenshot preview is unavailable.' };
+      }
+      return await deps.getMediaTimingReviewFrame(request);
+    },
+  );
   ipc.handle(
     IPC_CHANNELS.request.mediaTimingReviewStopPreview,
     async (_event: unknown, reviewId: unknown) => {

src/core/services/media-timing-frame.test.ts

@@ -0,0 +1,70 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { createMediaTimingFrameExtractor, selectMediaTimingFrame } from './media-timing-frame';
+
+test('frame stepping follows decoded timestamps with variable frame durations', () => {
+  const times = [10, 10.041667, 10.125, 10.166667];
+  assert.equal(selectMediaTimingFrame(times, 10.05), 10.125);
+  assert.equal(selectMediaTimingFrame(times, 10.125 - 0.000001, 1), 10.166667);
+  assert.equal(selectMediaTimingFrame(times, 10.125 - 0.000001, -1), 10.041667);
+  assert.equal(selectMediaTimingFrame(times, 10, -1), undefined);
+  assert.equal(selectMediaTimingFrame(times, 10.166667, 1), undefined);
+  assert.equal(selectMediaTimingFrame([], 10), undefined);
+});
+
+function fixture(startTime = 0) {
+  const calls: Array<{ file: string; args: string[] }> = [];
+  const extractor = createMediaTimingFrameExtractor(async (file, args) => {
+    calls.push({ file, args });
+    if (file === 'ffmpeg') return Buffer.from('image');
+    if (args.includes('format=start_time'))
+      return Buffer.from(JSON.stringify({ format: { start_time: String(startTime) } }));
+    return Buffer.from(
+      JSON.stringify({
+        frames: [10, 10.04, 10.12, 10.16].map((time) => ({
+          best_effort_timestamp_time: String(time + startTime),
+        })),
+      }),
+    );
+  });
+  return { calls, extractor };
+}
+
+test('frame extraction normalizes nonzero source start times and reuses the frame index', async () => {
+  const { calls, extractor } = fixture(5);
+  const first = await extractor.generate({ media: '/movie.mkv', timestamp: 10.05 });
+  assert.ok(Math.abs(first.timestamp - 10.119999) < 0.000001);
+  assert.equal(first.dataUrl, 'data:image/jpeg;base64,aW1hZ2U=');
+  await extractor.generate({ media: '/movie.mkv', timestamp: first.timestamp, direction: 1 });
+  assert.equal(calls.filter((call) => call.file === 'ffprobe').length, 2);
+  assert.ok(calls[1]!.args.includes('13.05%17.05'));
+  extractor.clear();
+  await extractor.generate({ media: '/movie.mkv', timestamp: 10.05 });
+  assert.equal(calls.filter((call) => call.file === 'ffprobe').length, 4);
+});
+
+test('cached windows keep source timestamps for ffmpeg and use absolute ffprobe intervals', async () => {
+  const { calls, extractor } = fixture();
+  await extractor.generate({
+    media: { path: '/window.mkv', absoluteTimestamps: true },
+    timestamp: 10.05,
+  });
+  assert.ok(calls[1]!.args.includes('8.05%12.05'));
+  assert.equal(calls[1]!.args.includes('-seek_timestamp'), false);
+  assert.ok(calls[2]!.args.includes('-seek_timestamp'));
+});
+
+test('remote frame reads carry source headers and do not silently reuse a different input', async () => {
+  const { calls, extractor } = fixture();
+  const media = {
+    path: 'https://example.test/video',
+    inputOptions: { headers: { 'X-Emby-Token': 'test-token' } },
+  };
+  await extractor.generate({ media, timestamp: 10.05 });
+  assert.ok(calls.every((call) => call.args.includes('X-Emby-Token: test-token\r\n')));
+  await extractor.generate({
+    media: { ...media, path: 'https://example.test/other' },
+    timestamp: 10.05,
+  });
+  assert.equal(calls.filter((call) => call.file === 'ffprobe').length, 4);
+});

src/core/services/media-timing-frame.ts

@@ -0,0 +1,159 @@
+import { execFile } from 'node:child_process';
+import { normalizeMediaInput, type MediaInput } from '../../media-input';
+
+export interface MediaTimingFrameOptions {
+  media: MediaInput;
+  timestamp: number;
+  direction?: -1 | 1;
+}
+
+const FRAME_EPSILON = 0.00001;
+const PROBE_RADIUS_SECONDS = 2;
+
+function run(file: string, args: string[]): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    execFile(
+      file,
+      args,
+      { encoding: 'buffer', timeout: 30_000, maxBuffer: 8 * 1024 * 1024, windowsHide: true },
+      (error, stdout) => {
+        // Child-process errors contain the input URL, which may contain authentication tokens.
+        if (error)
+          reject(
+            new Error('Screenshot preview is unavailable. Check FFmpeg and the video source.'),
+          );
+        else resolve(stdout);
+      },
+    );
+  });
+}
+
+/** Uses decoded timestamps, rather than an assumed FPS, including for variable-rate video. */
+export function selectMediaTimingFrame(
+  times: readonly number[],
+  timestamp: number,
+  direction?: -1 | 1,
+): number | undefined {
+  if (direction === -1)
+    return [...times].reverse().find((time) => time < timestamp - FRAME_EPSILON);
+  if (direction === 1) return times.find((time) => time > timestamp + FRAME_EPSILON);
+  return times.find((time) => time >= timestamp - FRAME_EPSILON) ?? times.at(-1);
+}
+
+export function createMediaTimingFrameExtractor(execute: typeof run = run) {
+  let cached: { key: string; start: number; end: number; times: number[] } | null = null;
+  let source: { key: string; offset: number } | null = null;
+  let generation = 0;
+
+  async function generate(
+    options: MediaTimingFrameOptions,
+  ): Promise<{ dataUrl: string; timestamp: number }> {
+    const input = normalizeMediaInput(options.media);
+    const key = JSON.stringify(options.media);
+    const currentGeneration = generation;
+    const absolute = typeof options.media !== 'string' && options.media.absoluteTimestamps;
+    // -seek_timestamp is an ffmpeg option; ffprobe intervals already use stream timestamps.
+    const probeInputArgs = normalizeMediaInput({
+      path: input.path,
+      ...(typeof options.media !== 'string' ? { inputOptions: options.media.inputOptions } : {}),
+    }).inputArgs;
+    let offset = source?.key === key ? source.offset : undefined;
+    if (offset === undefined) {
+      const metadata = JSON.parse(
+        (
+          await execute('ffprobe', [
+            '-v',
+            'error',
+            ...probeInputArgs,
+            '-show_entries',
+            'format=start_time',
+            '-of',
+            'json',
+            input.path,
+          ])
+        ).toString(),
+      ) as { format?: { start_time?: string } };
+      const start = Number(metadata.format?.start_time ?? 0);
+      offset = absolute || !Number.isFinite(start) ? 0 : start;
+      if (generation === currentGeneration) source = { key, offset };
+    }
+    let times: number[];
+    if (
+      cached?.key === key &&
+      options.timestamp > cached.start + 0.5 &&
+      options.timestamp < cached.end - 0.5
+    ) {
+      times = cached.times;
+    } else {
+      const start = Math.max(0, options.timestamp - PROBE_RADIUS_SECONDS);
+      const end = options.timestamp + PROBE_RADIUS_SECONDS;
+      const result = JSON.parse(
+        (
+          await execute('ffprobe', [
+            '-v',
+            'error',
+            ...probeInputArgs,
+            '-read_intervals',
+            `${start + offset}%${end + offset}`,
+            '-select_streams',
+            'v:0',
+            '-show_entries',
+            'frame=best_effort_timestamp_time',
+            '-of',
+            'json',
+            input.path,
+          ])
+        ).toString(),
+      ) as { frames?: { best_effort_timestamp_time?: string }[] };
+      times = [
+        ...new Set(
+          (result.frames ?? [])
+            .map((frame) => Number(frame.best_effort_timestamp_time) - offset)
+            .filter((time) => Number.isFinite(time) && time >= 0 && time >= start && time <= end),
+        ),
+      ].sort((a, b) => a - b);
+      if (generation === currentGeneration) cached = { key, start, end, times };
+    }
+    const timestamp = selectMediaTimingFrame(times, options.timestamp, options.direction);
+    if (timestamp === undefined) throw new Error('No adjacent video frame is available here.');
+    // Round down by one microsecond so decimal timestamp rounding cannot skip the chosen frame.
+    const seekTime = Math.max(0, timestamp - 0.000001);
+    const image = await execute('ffmpeg', [
+      '-hide_banner',
+      '-nostdin',
+      '-loglevel',
+      'error',
+      '-ss',
+      String(seekTime),
+      ...input.inputArgs,
+      '-i',
+      input.path,
+      '-map',
+      '0:v:0',
+      '-frames:v',
+      '1',
+      '-an',
+      '-sn',
+      '-vf',
+      'scale=w=640:h=360:force_original_aspect_ratio=decrease',
+      '-c:v',
+      'mjpeg',
+      '-q:v',
+      '3',
+      '-f',
+      'image2pipe',
+      'pipe:1',
+    ]);
+    if (!image.length) throw new Error('No video frame is available here.');
+    return { dataUrl: `data:image/jpeg;base64,${image.toString('base64')}`, timestamp: seekTime };
+  }
+
+  return {
+    generate,
+    clear: () => {
+      generation += 1;
+      cached = null;
+      source = null;
+    },
+  };
+}