feat(mining): add a screenshot frame picker to media review (#254)

src/renderer/index.html

@@ -297,6 +297,57 @@
             </div>
             <blockquote id="mediaTimingReviewText" class="media-timing-review-text"></blockquote>
 
+            <div id="mediaTimingReviewFramePicker" class="media-timing-frame-picker hidden">
+              <div class="media-timing-frame-image-shell">
+                <img
+                  id="mediaTimingReviewFrameImage"
+                  class="hidden"
+                  alt="Selected card screenshot"
+                />
+              </div>
+              <div class="media-timing-frame-controls">
+                <div class="media-timing-frame-heading">
+                  <label for="mediaTimingReviewFrameSlider">Card screenshot</label>
+                  <output id="mediaTimingReviewFrameTime" for="mediaTimingReviewFrameSlider"
+                    >—</output
+                  >
+                </div>
+                <input
+                  id="mediaTimingReviewFrameSlider"
+                  type="range"
+                  min="0"
+                  max="1"
+                  step="0.001"
+                  value="0"
+                  aria-label="Screenshot time"
+                />
+                <div class="media-timing-frame-buttons">
+                  <button
+                    id="mediaTimingReviewFramePrevious"
+                    type="button"
+                    aria-label="Previous video frame"
+                  >
+                    ← Frame
+                  </button>
+                  <button
+                    id="mediaTimingReviewFrameNext"
+                    type="button"
+                    aria-label="Next video frame"
+                  >
+                    Frame →
+                  </button>
+                  <button
+                    id="mediaTimingReviewFrameReset"
+                    type="button"
+                    aria-label="Reset screenshot"
+                  >
+                    Reset
+                  </button>
+                </div>
+                <p id="mediaTimingReviewFrameStatus" role="status"></p>
+              </div>
+            </div>
+
             <div class="media-timing-review-readout" aria-live="polite">
               <div>
                 <span>Starts</span>

src/renderer/modals/media-timing-frame-picker.test.ts

@@ -0,0 +1,116 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { setTimeout as tick } from 'node:timers/promises';
+import { createMediaTimingFramePicker } from './media-timing-frame-picker';
+import type { MediaTimingReviewFrameRequest, MediaTimingReviewFrameResult } from '../../types/anki';
+
+function fixture() {
+  const requests: Array<{
+    request: MediaTimingReviewFrameRequest;
+    resolve: (result: MediaTimingReviewFrameResult) => void;
+  }> = [];
+  let stale = false;
+  const picker = createMediaTimingFramePicker({
+    debounceMs: 0,
+    load: (request) => new Promise((resolve) => requests.push({ request, resolve })),
+    onChange: () => {},
+    onStale: () => {
+      stale = true;
+      picker.close();
+    },
+  });
+  const complete = async (index: number, result: number | MediaTimingReviewFrameResult) => {
+    requests[index]!.resolve(
+      typeof result === 'number'
+        ? { ok: true, timestamp: result, dataUrl: `data:image/jpeg;base64,${result}` }
+        : result,
+    );
+    await tick(0);
+  };
+  return { picker, requests, complete, isStale: () => stale };
+}
+
+test('a chosen frame stays fixed until Reset restores midpoint tracking', async () => {
+  const { picker, requests, complete } = fixture();
+  picker.open('r', true, 11);
+  await tick(5);
+  await complete(0, 11);
+  assert.equal(picker.getScreenshotTime(), undefined);
+  picker.choose(13);
+  await tick(5);
+  await complete(1, 13);
+  picker.updateMidpoint(12);
+  await tick(5);
+  assert.equal(picker.getScreenshotTime(), 13);
+  assert.equal(requests.length, 2);
+  picker.reset();
+  await tick(5);
+  assert.equal(requests[2]?.request.timestamp, 12);
+  await complete(2, 12);
+  assert.equal(picker.getScreenshotTime(), undefined);
+  picker.updateMidpoint(14);
+  await tick(5);
+  assert.equal(requests[3]?.request.timestamp, 14);
+  await complete(3, 14);
+  picker.close();
+});
+
+test('scrubbing keeps only the latest requested frame', async () => {
+  const { picker, requests, complete } = fixture();
+  picker.open('r', true, 11);
+  await tick(5);
+  picker.choose(12);
+  picker.choose(13);
+  picker.choose(14);
+  await tick(5);
+  assert.equal(requests.length, 1);
+  assert.equal(picker.getState().blockConfirm, true);
+  await complete(0, 11);
+  await tick(5);
+  assert.equal(picker.getState().timestamp, undefined);
+  assert.equal(requests[1]?.request.timestamp, 14);
+  await complete(1, 14);
+  assert.equal(picker.getScreenshotTime(), 14);
+  assert.equal(picker.getState().blockConfirm, false);
+  picker.close();
+});
+
+test('failed manual previews block confirmation; Reset allows the default fallback', async () => {
+  const { picker, complete } = fixture();
+  picker.open('r', true, 11);
+  await tick(5);
+  await complete(0, { ok: false });
+  assert.equal(picker.getState().blockConfirm, false);
+  picker.choose(12);
+  await tick(5);
+  await complete(1, { ok: false });
+  assert.equal(picker.getState().blockConfirm, true);
+  picker.reset();
+  assert.equal(picker.getState().blockConfirm, false);
+  picker.close();
+});
+
+test('replacing a review ignores old frames; stale responses close the current review', async () => {
+  const { picker, requests, complete, isStale } = fixture();
+  picker.open('old', true, 11);
+  await tick(5);
+  picker.close();
+  picker.open('new', true, 22);
+  await tick(5);
+  await complete(0, 11);
+  await tick(5);
+  assert.equal(picker.getState().timestamp, undefined);
+  assert.equal(requests[1]?.request.reviewId, 'new');
+  await complete(1, { ok: false, stale: true });
+  assert.equal(isStale(), true);
+  assert.equal(picker.getState().enabled, false);
+});
+
+test('disabled screenshots perform no extraction', async () => {
+  const { picker, requests } = fixture();
+  picker.open('r', false, 11);
+  picker.updateMidpoint(12);
+  await tick(5);
+  assert.equal(requests.length, 0);
+  picker.close();
+});

src/renderer/modals/media-timing-frame-picker.ts

@@ -0,0 +1,129 @@
+import type { MediaTimingReviewFrameRequest, MediaTimingReviewFrameResult } from '../../types/anki';
+
+export interface MediaTimingFramePickerState {
+  enabled: boolean;
+  manual: boolean;
+  loading: boolean;
+  timestamp?: number;
+  requestedTime?: number;
+  dataUrl?: string;
+  message: string;
+  blockConfirm: boolean;
+}
+
+/** Coalesces scrubbing into one active extraction and the latest requested frame. */
+export function createMediaTimingFramePicker(options: {
+  load: (request: MediaTimingReviewFrameRequest) => Promise<MediaTimingReviewFrameResult>;
+  onChange: (state: MediaTimingFramePickerState) => void;
+  onStale: () => void;
+  debounceMs?: number;
+}) {
+  let reviewId: string | null = null;
+  let midpoint = 0;
+  let sequence = 0;
+  let inFlight = false;
+  let pending: (MediaTimingReviewFrameRequest & { sequence: number }) | null = null;
+  let timer: ReturnType<typeof setTimeout> | null = null;
+  let state: MediaTimingFramePickerState = emptyState();
+
+  function emptyState(): MediaTimingFramePickerState {
+    return { enabled: false, manual: false, loading: false, message: '', blockConfirm: false };
+  }
+
+  function publish(): void {
+    options.onChange({ ...state });
+  }
+
+  async function drain(): Promise<void> {
+    if (inFlight || !pending) return;
+    const request = pending;
+    pending = null;
+    inFlight = true;
+    try {
+      const result = await options.load(request);
+      if (reviewId !== request.reviewId || sequence !== request.sequence) return;
+      if (result.stale) {
+        options.onStale();
+        return;
+      }
+      if (
+        !result.ok ||
+        !Number.isFinite(result.timestamp) ||
+        !result.dataUrl?.startsWith('data:image/jpeg;base64,')
+      ) {
+        throw new Error(result.message ?? 'Screenshot preview is unavailable.');
+      }
+      state.timestamp = result.timestamp;
+      state.dataUrl = result.dataUrl;
+      state.blockConfirm = false;
+      state.message = state.manual
+        ? 'Selected frame stays fixed when you trim audio.'
+        : 'Following the audio midpoint.';
+    } catch (error) {
+      if (reviewId !== request.reviewId || sequence !== request.sequence) return;
+      state.message = error instanceof Error ? error.message : 'Screenshot preview is unavailable.';
+      state.blockConfirm = state.manual;
+    } finally {
+      inFlight = false;
+      if (reviewId === request.reviewId && sequence === request.sequence) {
+        state.loading = false;
+        publish();
+      }
+      if (timer === null) void drain();
+    }
+  }
+
+  function request(timestamp: number, direction?: -1 | 1): void {
+    if (!reviewId || !state.enabled) return;
+    sequence += 1;
+    pending = { reviewId, timestamp, ...(direction ? { direction } : {}), sequence };
+    state.requestedTime = timestamp;
+    state.loading = true;
+    state.blockConfirm = state.manual;
+    state.message = 'Loading screenshot…';
+    publish();
+    if (timer !== null) clearTimeout(timer);
+    timer = setTimeout(() => {
+      timer = null;
+      void drain();
+    }, options.debounceMs ?? 120);
+  }
+
+  function close(): void {
+    sequence += 1;
+    reviewId = null;
+    pending = null;
+    if (timer !== null) clearTimeout(timer);
+    timer = null;
+    state = emptyState();
+    publish();
+  }
+
+  return {
+    open(id: string, enabled: boolean, time: number) {
+      close();
+      reviewId = id;
+      midpoint = time;
+      state.enabled = enabled;
+      publish();
+      if (enabled) request(time);
+    },
+    updateMidpoint(time: number) {
+      if (midpoint === time) return;
+      midpoint = time;
+      if (!state.manual) request(time);
+    },
+    choose(time: number, direction?: -1 | 1) {
+      if (!Number.isFinite(time) || time < 0) return;
+      state.manual = true;
+      request(time, direction);
+    },
+    reset() {
+      state.manual = false;
+      request(midpoint);
+    },
+    close,
+    getState: () => ({ ...state }),
+    getScreenshotTime: () => (state.manual && !state.blockConfirm ? state.timestamp : undefined),
+  };
+}

src/renderer/modals/media-timing-review.ts

@@ -1,3 +1,4 @@
+import { createMediaTimingFramePicker } from './media-timing-frame-picker';
 import type {
   MediaTimingReviewContextLine,
   MediaTimingReviewDecision,
@@ -252,6 +253,45 @@ export function createMediaTimingReviewModal(
     isModalLayer: ctx.platform.isModalLayer,
   });
   const previewRequest = createMediaTimingPreviewRequestGuard();
+  const framePicker = createMediaTimingFramePicker({
+    load: (request) => window.electronAPI.getMediaTimingReviewFrame(request),
+    onChange: () => renderFramePicker(),
+    onStale: () => closeResolvedReview(),
+  });
+
+  function renderFramePicker(): void {
+    const state = framePicker.getState();
+    const dom = ctx.dom;
+    dom.mediaTimingReviewFramePicker.classList.toggle('hidden', !state.enabled);
+    dom.mediaTimingReviewFramePicker.setAttribute('aria-busy', String(state.loading));
+    dom.mediaTimingReviewFrameSlider.min = String(timelineStart);
+    dom.mediaTimingReviewFrameSlider.max = String(Math.max(timelineStart, timelineEnd - 0.001));
+    const shownTime = state.loading ? state.requestedTime : state.timestamp;
+    if (shownTime !== undefined) dom.mediaTimingReviewFrameSlider.value = String(shownTime);
+    dom.mediaTimingReviewFrameTime.textContent =
+      shownTime === undefined ? '—' : formatMediaTimingTimestamp(shownTime);
+    dom.mediaTimingReviewFrameSlider.setAttribute(
+      'aria-valuetext',
+      dom.mediaTimingReviewFrameTime.textContent,
+    );
+    dom.mediaTimingReviewFrameStatus.textContent = state.message;
+    dom.mediaTimingReviewFrameImage.classList.toggle('hidden', !state.dataUrl);
+    if (state.dataUrl) dom.mediaTimingReviewFrameImage.src = state.dataUrl;
+    else dom.mediaTimingReviewFrameImage.removeAttribute('src');
+    dom.mediaTimingReviewFramePrevious.disabled =
+      resolveInFlight ||
+      state.loading ||
+      state.timestamp === undefined ||
+      state.timestamp <= timelineStart;
+    dom.mediaTimingReviewFrameNext.disabled =
+      resolveInFlight ||
+      state.loading ||
+      state.timestamp === undefined ||
+      state.timestamp >= timelineEnd - 0.001;
+    dom.mediaTimingReviewFrameSlider.disabled = resolveInFlight;
+    dom.mediaTimingReviewFrameReset.disabled = resolveInFlight || !state.manual;
+    dom.mediaTimingReviewConfirm.disabled = resolveInFlight || state.blockConfirm;
+  }
 
   function setStatus(message: string, isError = false): void {
     ctx.dom.mediaTimingReviewStatus.textContent = message;
@@ -308,6 +348,8 @@ export function createMediaTimingReviewModal(
   }
 
   function renderSelection(): void {
+    framePicker.updateMidpoint(selectionStart + (selectionEnd - selectionStart) / 2);
+    renderFramePicker();
     const span = Math.max(MINIMUM_CLIP_SECONDS, timelineEnd - timelineStart);
     const startPercent = ((selectionStart - timelineStart) / span) * 100;
     const endPercent = ((selectionEnd - timelineStart) / span) * 100;
@@ -690,6 +732,7 @@ export function createMediaTimingReviewModal(
     window.electronAPI.notifyOverlayModalClosed('media-timing-review');
     options.syncSettingsModalSubtitleSuppression();
     payload = null;
+    framePicker.close();
     if (!options.modalStateReader.isAnyModalOpen()) {
       ctx.dom.overlay.classList.remove('interactive');
       if (ctx.platform.shouldToggleMouseIgnore) {
@@ -701,6 +744,7 @@ export function createMediaTimingReviewModal(
   async function resolveReview(decision: MediaTimingReviewDecision): Promise<void> {
     if (!payload || resolveInFlight) return;
     resolveInFlight = true;
+    renderFramePicker();
     stopPreview();
     const controls = ctx.dom.mediaTimingReviewModal.querySelectorAll<HTMLButtonElement>('button');
     controls.forEach((button) => {
@@ -733,12 +777,14 @@ export function createMediaTimingReviewModal(
   }
 
   function confirmSelection(): void {
-    if (!payload) return;
+    if (!payload || framePicker.getState().blockConfirm) return;
+    const screenshotTime = framePicker.getScreenshotTime();
     const includesAdjacentLines = previousCount > 0 || nextCount > 0;
     void resolveReview({
       action: 'confirm',
       startTime: selectionStart,
       endTime: selectionEnd,
+      ...(screenshotTime !== undefined ? { screenshotTime } : {}),
       ...(includesAdjacentLines ? { text: currentLineSelection().sentence } : {}),
     });
   }
@@ -818,6 +864,11 @@ export function createMediaTimingReviewModal(
       nextPayload.noteId !== undefined ? 'Delete card' : "Don't create card";
     setStatus('');
     showEditor();
+    framePicker.open(
+      nextPayload.reviewId,
+      nextPayload.screenshotEnabled === true,
+      selectionStart + (selectionEnd - selectionStart) / 2,
+    );
     renderSelection();
     renderSentence();
     ctx.state.mediaTimingReviewModalOpen = true;
@@ -892,6 +943,28 @@ export function createMediaTimingReviewModal(
   }
 
   function wireDomEvents(): void {
+    ctx.dom.mediaTimingReviewFrameSlider.addEventListener('input', () => {
+      if (!resolveInFlight) framePicker.choose(Number(ctx.dom.mediaTimingReviewFrameSlider.value));
+    });
+    const stepFrame = (direction: -1 | 1) => {
+      const state = framePicker.getState();
+      if (!resolveInFlight && !state.loading && state.timestamp !== undefined)
+        framePicker.choose(state.timestamp, direction);
+    };
+    ctx.dom.mediaTimingReviewFrameSlider.addEventListener('keydown', (event) => {
+      if (event.key === 'ArrowLeft' || event.key === 'ArrowDown') {
+        event.preventDefault();
+        stepFrame(-1);
+      } else if (event.key === 'ArrowRight' || event.key === 'ArrowUp') {
+        event.preventDefault();
+        stepFrame(1);
+      }
+    });
+    ctx.dom.mediaTimingReviewFramePrevious.addEventListener('click', () => stepFrame(-1));
+    ctx.dom.mediaTimingReviewFrameNext.addEventListener('click', () => stepFrame(1));
+    ctx.dom.mediaTimingReviewFrameReset.addEventListener('click', () => {
+      if (!resolveInFlight) framePicker.reset();
+    });
     const track = ctx.dom.mediaTimingReviewSelectionTrack;
     track.addEventListener('pointerdown', beginDrag);
     track.addEventListener('pointermove', (event) => {

src/renderer/style.css

@@ -1347,7 +1347,7 @@ body:focus-visible,
 
 .media-timing-review-content {
   width: min(720px, calc(100vw - 32px));
-  max-height: min(700px, calc(100vh - 32px));
+  max-height: min(900px, calc(100vh - 32px));
   overflow: auto;
   gap: 0;
   padding: 0;
@@ -1425,6 +1425,86 @@ body:focus-visible,
   padding: 14px 20px 18px;
 }
 
+.media-timing-frame-picker {
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
+  gap: 16px;
+  align-items: center;
+  margin: 0 0 16px;
+  padding: 12px;
+  border: 1px solid var(--ctp-surface1);
+  border-radius: 10px;
+  background: var(--ctp-mantle);
+}
+
+.media-timing-frame-picker.hidden {
+  display: none;
+}
+.media-timing-frame-image-shell {
+  aspect-ratio: 16 / 9;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  overflow: hidden;
+  border-radius: 6px;
+  background: var(--ctp-crust);
+}
+.media-timing-frame-image-shell img {
+  width: 100%;
+  height: 100%;
+  object-fit: contain;
+}
+.media-timing-frame-picker[aria-busy='true'] img {
+  opacity: 0.55;
+}
+.media-timing-frame-heading {
+  display: flex;
+  justify-content: space-between;
+  gap: 8px;
+  font-size: 12px;
+}
+.media-timing-frame-heading output {
+  color: var(--ctp-teal);
+  font-variant-numeric: tabular-nums;
+}
+.media-timing-frame-controls input {
+  width: 100%;
+  margin: 12px 0;
+  accent-color: var(--ctp-teal);
+}
+.media-timing-frame-buttons {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 6px;
+}
+.media-timing-frame-buttons button {
+  padding: 5px 8px;
+  border: 1px solid var(--ctp-surface2);
+  border-radius: 6px;
+  background: var(--ctp-surface0);
+  color: var(--ctp-text);
+  font-size: 11px;
+  cursor: pointer;
+}
+.media-timing-frame-buttons button:disabled {
+  opacity: 0.45;
+  cursor: default;
+}
+.media-timing-frame-controls p {
+  margin: 8px 0 0;
+  min-height: 2.6em;
+  color: var(--ctp-subtext0);
+  font-size: 11px;
+}
+@media (max-width: 520px) {
+  .media-timing-frame-picker {
+    grid-template-columns: 1fr;
+  }
+  .media-timing-frame-image-shell {
+    max-height: 180px;
+  }
+}
+
 .media-timing-review-sentence-header {
   display: flex;
   align-items: center;

src/renderer/utils/dom.ts

@@ -46,6 +46,14 @@ export type RendererDom = {
   youtubePickerStatus: HTMLDivElement;
   youtubePickerTracks: HTMLUListElement;
 
+  mediaTimingReviewFramePicker: HTMLDivElement;
+  mediaTimingReviewFrameImage: HTMLImageElement;
+  mediaTimingReviewFrameSlider: HTMLInputElement;
+  mediaTimingReviewFrameTime: HTMLElement;
+  mediaTimingReviewFrameStatus: HTMLElement;
+  mediaTimingReviewFramePrevious: HTMLButtonElement;
+  mediaTimingReviewFrameNext: HTMLButtonElement;
+  mediaTimingReviewFrameReset: HTMLButtonElement;
   mediaTimingReviewModal: HTMLDivElement;
   mediaTimingReviewKind: HTMLDivElement;
   mediaTimingReviewText: HTMLElement;
@@ -249,6 +257,24 @@ export function resolveRendererDom(): RendererDom {
     youtubePickerStatus: getRequiredElement<HTMLDivElement>('youtubePickerStatus'),
     youtubePickerTracks: getRequiredElement<HTMLUListElement>('youtubePickerTracks'),
 
+    mediaTimingReviewFramePicker: getRequiredElement<HTMLDivElement>(
+      'mediaTimingReviewFramePicker',
+    ),
+    mediaTimingReviewFrameImage: getRequiredElement<HTMLImageElement>(
+      'mediaTimingReviewFrameImage',
+    ),
+    mediaTimingReviewFrameSlider: getRequiredElement<HTMLInputElement>(
+      'mediaTimingReviewFrameSlider',
+    ),
+    mediaTimingReviewFrameTime: getRequiredElement<HTMLElement>('mediaTimingReviewFrameTime'),
+    mediaTimingReviewFrameStatus: getRequiredElement<HTMLElement>('mediaTimingReviewFrameStatus'),
+    mediaTimingReviewFramePrevious: getRequiredElement<HTMLButtonElement>(
+      'mediaTimingReviewFramePrevious',
+    ),
+    mediaTimingReviewFrameNext: getRequiredElement<HTMLButtonElement>('mediaTimingReviewFrameNext'),
+    mediaTimingReviewFrameReset: getRequiredElement<HTMLButtonElement>(
+      'mediaTimingReviewFrameReset',
+    ),
     mediaTimingReviewModal: getRequiredElement<HTMLDivElement>('mediaTimingReviewModal'),
     mediaTimingReviewKind: getRequiredElement<HTMLDivElement>('mediaTimingReviewKind'),
     mediaTimingReviewText: getRequiredElement<HTMLElement>('mediaTimingReviewText'),