feat(mining): add a screenshot frame picker to media review (#254)

src/renderer/modals/media-timing-frame-picker.test.ts

@@ -0,0 +1,116 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { setTimeout as tick } from 'node:timers/promises';
+import { createMediaTimingFramePicker } from './media-timing-frame-picker';
+import type { MediaTimingReviewFrameRequest, MediaTimingReviewFrameResult } from '../../types/anki';
+
+function fixture() {
+  const requests: Array<{
+    request: MediaTimingReviewFrameRequest;
+    resolve: (result: MediaTimingReviewFrameResult) => void;
+  }> = [];
+  let stale = false;
+  const picker = createMediaTimingFramePicker({
+    debounceMs: 0,
+    load: (request) => new Promise((resolve) => requests.push({ request, resolve })),
+    onChange: () => {},
+    onStale: () => {
+      stale = true;
+      picker.close();
+    },
+  });
+  const complete = async (index: number, result: number | MediaTimingReviewFrameResult) => {
+    requests[index]!.resolve(
+      typeof result === 'number'
+        ? { ok: true, timestamp: result, dataUrl: `data:image/jpeg;base64,${result}` }
+        : result,
+    );
+    await tick(0);
+  };
+  return { picker, requests, complete, isStale: () => stale };
+}
+
+test('a chosen frame stays fixed until Reset restores midpoint tracking', async () => {
+  const { picker, requests, complete } = fixture();
+  picker.open('r', true, 11);
+  await tick(5);
+  await complete(0, 11);
+  assert.equal(picker.getScreenshotTime(), undefined);
+  picker.choose(13);
+  await tick(5);
+  await complete(1, 13);
+  picker.updateMidpoint(12);
+  await tick(5);
+  assert.equal(picker.getScreenshotTime(), 13);
+  assert.equal(requests.length, 2);
+  picker.reset();
+  await tick(5);
+  assert.equal(requests[2]?.request.timestamp, 12);
+  await complete(2, 12);
+  assert.equal(picker.getScreenshotTime(), undefined);
+  picker.updateMidpoint(14);
+  await tick(5);
+  assert.equal(requests[3]?.request.timestamp, 14);
+  await complete(3, 14);
+  picker.close();
+});
+
+test('scrubbing keeps only the latest requested frame', async () => {
+  const { picker, requests, complete } = fixture();
+  picker.open('r', true, 11);
+  await tick(5);
+  picker.choose(12);
+  picker.choose(13);
+  picker.choose(14);
+  await tick(5);
+  assert.equal(requests.length, 1);
+  assert.equal(picker.getState().blockConfirm, true);
+  await complete(0, 11);
+  await tick(5);
+  assert.equal(picker.getState().timestamp, undefined);
+  assert.equal(requests[1]?.request.timestamp, 14);
+  await complete(1, 14);
+  assert.equal(picker.getScreenshotTime(), 14);
+  assert.equal(picker.getState().blockConfirm, false);
+  picker.close();
+});
+
+test('failed manual previews block confirmation; Reset allows the default fallback', async () => {
+  const { picker, complete } = fixture();
+  picker.open('r', true, 11);
+  await tick(5);
+  await complete(0, { ok: false });
+  assert.equal(picker.getState().blockConfirm, false);
+  picker.choose(12);
+  await tick(5);
+  await complete(1, { ok: false });
+  assert.equal(picker.getState().blockConfirm, true);
+  picker.reset();
+  assert.equal(picker.getState().blockConfirm, false);
+  picker.close();
+});
+
+test('replacing a review ignores old frames; stale responses close the current review', async () => {
+  const { picker, requests, complete, isStale } = fixture();
+  picker.open('old', true, 11);
+  await tick(5);
+  picker.close();
+  picker.open('new', true, 22);
+  await tick(5);
+  await complete(0, 11);
+  await tick(5);
+  assert.equal(picker.getState().timestamp, undefined);
+  assert.equal(requests[1]?.request.reviewId, 'new');
+  await complete(1, { ok: false, stale: true });
+  assert.equal(isStale(), true);
+  assert.equal(picker.getState().enabled, false);
+});
+
+test('disabled screenshots perform no extraction', async () => {
+  const { picker, requests } = fixture();
+  picker.open('r', false, 11);
+  picker.updateMidpoint(12);
+  await tick(5);
+  assert.equal(requests.length, 0);
+  picker.close();
+});

src/renderer/modals/media-timing-frame-picker.ts

@@ -0,0 +1,129 @@
+import type { MediaTimingReviewFrameRequest, MediaTimingReviewFrameResult } from '../../types/anki';
+
+export interface MediaTimingFramePickerState {
+  enabled: boolean;
+  manual: boolean;
+  loading: boolean;
+  timestamp?: number;
+  requestedTime?: number;
+  dataUrl?: string;
+  message: string;
+  blockConfirm: boolean;
+}
+
+/** Coalesces scrubbing into one active extraction and the latest requested frame. */
+export function createMediaTimingFramePicker(options: {
+  load: (request: MediaTimingReviewFrameRequest) => Promise<MediaTimingReviewFrameResult>;
+  onChange: (state: MediaTimingFramePickerState) => void;
+  onStale: () => void;
+  debounceMs?: number;
+}) {
+  let reviewId: string | null = null;
+  let midpoint = 0;
+  let sequence = 0;
+  let inFlight = false;
+  let pending: (MediaTimingReviewFrameRequest & { sequence: number }) | null = null;
+  let timer: ReturnType<typeof setTimeout> | null = null;
+  let state: MediaTimingFramePickerState = emptyState();
+
+  function emptyState(): MediaTimingFramePickerState {
+    return { enabled: false, manual: false, loading: false, message: '', blockConfirm: false };
+  }
+
+  function publish(): void {
+    options.onChange({ ...state });
+  }
+
+  async function drain(): Promise<void> {
+    if (inFlight || !pending) return;
+    const request = pending;
+    pending = null;
+    inFlight = true;
+    try {
+      const result = await options.load(request);
+      if (reviewId !== request.reviewId || sequence !== request.sequence) return;
+      if (result.stale) {
+        options.onStale();
+        return;
+      }
+      if (
+        !result.ok ||
+        !Number.isFinite(result.timestamp) ||
+        !result.dataUrl?.startsWith('data:image/jpeg;base64,')
+      ) {
+        throw new Error(result.message ?? 'Screenshot preview is unavailable.');
+      }
+      state.timestamp = result.timestamp;
+      state.dataUrl = result.dataUrl;
+      state.blockConfirm = false;
+      state.message = state.manual
+        ? 'Selected frame stays fixed when you trim audio.'
+        : 'Following the audio midpoint.';
+    } catch (error) {
+      if (reviewId !== request.reviewId || sequence !== request.sequence) return;
+      state.message = error instanceof Error ? error.message : 'Screenshot preview is unavailable.';
+      state.blockConfirm = state.manual;
+    } finally {
+      inFlight = false;
+      if (reviewId === request.reviewId && sequence === request.sequence) {
+        state.loading = false;
+        publish();
+      }
+      if (timer === null) void drain();
+    }
+  }
+
+  function request(timestamp: number, direction?: -1 | 1): void {
+    if (!reviewId || !state.enabled) return;
+    sequence += 1;
+    pending = { reviewId, timestamp, ...(direction ? { direction } : {}), sequence };
+    state.requestedTime = timestamp;
+    state.loading = true;
+    state.blockConfirm = state.manual;
+    state.message = 'Loading screenshot…';
+    publish();
+    if (timer !== null) clearTimeout(timer);
+    timer = setTimeout(() => {
+      timer = null;
+      void drain();
+    }, options.debounceMs ?? 120);
+  }
+
+  function close(): void {
+    sequence += 1;
+    reviewId = null;
+    pending = null;
+    if (timer !== null) clearTimeout(timer);
+    timer = null;
+    state = emptyState();
+    publish();
+  }
+
+  return {
+    open(id: string, enabled: boolean, time: number) {
+      close();
+      reviewId = id;
+      midpoint = time;
+      state.enabled = enabled;
+      publish();
+      if (enabled) request(time);
+    },
+    updateMidpoint(time: number) {
+      if (midpoint === time) return;
+      midpoint = time;
+      if (!state.manual) request(time);
+    },
+    choose(time: number, direction?: -1 | 1) {
+      if (!Number.isFinite(time) || time < 0) return;
+      state.manual = true;
+      request(time, direction);
+    },
+    reset() {
+      state.manual = false;
+      request(midpoint);
+    },
+    close,
+    getState: () => ({ ...state }),
+    getScreenshotTime: () => (state.manual && !state.blockConfirm ? state.timestamp : undefined),
+  };
+}

src/renderer/modals/media-timing-review.ts

@@ -1,3 +1,4 @@
+import { createMediaTimingFramePicker } from './media-timing-frame-picker';
 import type {
   MediaTimingReviewContextLine,
   MediaTimingReviewDecision,
@@ -252,6 +253,45 @@ export function createMediaTimingReviewModal(
     isModalLayer: ctx.platform.isModalLayer,
   });
   const previewRequest = createMediaTimingPreviewRequestGuard();
+  const framePicker = createMediaTimingFramePicker({
+    load: (request) => window.electronAPI.getMediaTimingReviewFrame(request),
+    onChange: () => renderFramePicker(),
+    onStale: () => closeResolvedReview(),
+  });
+
+  function renderFramePicker(): void {
+    const state = framePicker.getState();
+    const dom = ctx.dom;
+    dom.mediaTimingReviewFramePicker.classList.toggle('hidden', !state.enabled);
+    dom.mediaTimingReviewFramePicker.setAttribute('aria-busy', String(state.loading));
+    dom.mediaTimingReviewFrameSlider.min = String(timelineStart);
+    dom.mediaTimingReviewFrameSlider.max = String(Math.max(timelineStart, timelineEnd - 0.001));
+    const shownTime = state.loading ? state.requestedTime : state.timestamp;
+    if (shownTime !== undefined) dom.mediaTimingReviewFrameSlider.value = String(shownTime);
+    dom.mediaTimingReviewFrameTime.textContent =
+      shownTime === undefined ? '—' : formatMediaTimingTimestamp(shownTime);
+    dom.mediaTimingReviewFrameSlider.setAttribute(
+      'aria-valuetext',
+      dom.mediaTimingReviewFrameTime.textContent,
+    );
+    dom.mediaTimingReviewFrameStatus.textContent = state.message;
+    dom.mediaTimingReviewFrameImage.classList.toggle('hidden', !state.dataUrl);
+    if (state.dataUrl) dom.mediaTimingReviewFrameImage.src = state.dataUrl;
+    else dom.mediaTimingReviewFrameImage.removeAttribute('src');
+    dom.mediaTimingReviewFramePrevious.disabled =
+      resolveInFlight ||
+      state.loading ||
+      state.timestamp === undefined ||
+      state.timestamp <= timelineStart;
+    dom.mediaTimingReviewFrameNext.disabled =
+      resolveInFlight ||
+      state.loading ||
+      state.timestamp === undefined ||
+      state.timestamp >= timelineEnd - 0.001;
+    dom.mediaTimingReviewFrameSlider.disabled = resolveInFlight;
+    dom.mediaTimingReviewFrameReset.disabled = resolveInFlight || !state.manual;
+    dom.mediaTimingReviewConfirm.disabled = resolveInFlight || state.blockConfirm;
+  }
 
   function setStatus(message: string, isError = false): void {
     ctx.dom.mediaTimingReviewStatus.textContent = message;
@@ -308,6 +348,8 @@ export function createMediaTimingReviewModal(
   }
 
   function renderSelection(): void {
+    framePicker.updateMidpoint(selectionStart + (selectionEnd - selectionStart) / 2);
+    renderFramePicker();
     const span = Math.max(MINIMUM_CLIP_SECONDS, timelineEnd - timelineStart);
     const startPercent = ((selectionStart - timelineStart) / span) * 100;
     const endPercent = ((selectionEnd - timelineStart) / span) * 100;
@@ -690,6 +732,7 @@ export function createMediaTimingReviewModal(
     window.electronAPI.notifyOverlayModalClosed('media-timing-review');
     options.syncSettingsModalSubtitleSuppression();
     payload = null;
+    framePicker.close();
     if (!options.modalStateReader.isAnyModalOpen()) {
       ctx.dom.overlay.classList.remove('interactive');
       if (ctx.platform.shouldToggleMouseIgnore) {
@@ -701,6 +744,7 @@ export function createMediaTimingReviewModal(
   async function resolveReview(decision: MediaTimingReviewDecision): Promise<void> {
     if (!payload || resolveInFlight) return;
     resolveInFlight = true;
+    renderFramePicker();
     stopPreview();
     const controls = ctx.dom.mediaTimingReviewModal.querySelectorAll<HTMLButtonElement>('button');
     controls.forEach((button) => {
@@ -733,12 +777,14 @@ export function createMediaTimingReviewModal(
   }
 
   function confirmSelection(): void {
-    if (!payload) return;
+    if (!payload || framePicker.getState().blockConfirm) return;
+    const screenshotTime = framePicker.getScreenshotTime();
     const includesAdjacentLines = previousCount > 0 || nextCount > 0;
     void resolveReview({
       action: 'confirm',
       startTime: selectionStart,
       endTime: selectionEnd,
+      ...(screenshotTime !== undefined ? { screenshotTime } : {}),
       ...(includesAdjacentLines ? { text: currentLineSelection().sentence } : {}),
     });
   }
@@ -818,6 +864,11 @@ export function createMediaTimingReviewModal(
       nextPayload.noteId !== undefined ? 'Delete card' : "Don't create card";
     setStatus('');
     showEditor();
+    framePicker.open(
+      nextPayload.reviewId,
+      nextPayload.screenshotEnabled === true,
+      selectionStart + (selectionEnd - selectionStart) / 2,
+    );
     renderSelection();
     renderSentence();
     ctx.state.mediaTimingReviewModalOpen = true;
@@ -892,6 +943,28 @@ export function createMediaTimingReviewModal(
   }
 
   function wireDomEvents(): void {
+    ctx.dom.mediaTimingReviewFrameSlider.addEventListener('input', () => {
+      if (!resolveInFlight) framePicker.choose(Number(ctx.dom.mediaTimingReviewFrameSlider.value));
+    });
+    const stepFrame = (direction: -1 | 1) => {
+      const state = framePicker.getState();
+      if (!resolveInFlight && !state.loading && state.timestamp !== undefined)
+        framePicker.choose(state.timestamp, direction);
+    };
+    ctx.dom.mediaTimingReviewFrameSlider.addEventListener('keydown', (event) => {
+      if (event.key === 'ArrowLeft' || event.key === 'ArrowDown') {
+        event.preventDefault();
+        stepFrame(-1);
+      } else if (event.key === 'ArrowRight' || event.key === 'ArrowUp') {
+        event.preventDefault();
+        stepFrame(1);
+      }
+    });
+    ctx.dom.mediaTimingReviewFramePrevious.addEventListener('click', () => stepFrame(-1));
+    ctx.dom.mediaTimingReviewFrameNext.addEventListener('click', () => stepFrame(1));
+    ctx.dom.mediaTimingReviewFrameReset.addEventListener('click', () => {
+      if (!resolveInFlight) framePicker.reset();
+    });
     const track = ctx.dom.mediaTimingReviewSelectionTrack;
     track.addEventListener('pointerdown', beginDrag);
     track.addEventListener('pointermove', (event) => {