feat(mining): add a screenshot frame picker to media review (#254)

src/anki-integration/card-creation.test.ts

@@ -290,6 +290,9 @@ test('CardCreationService keeps updating after recordCardsMinedCallback throws',
 });
 
 test('CardCreationService uses stream-open-filename for remote media generation', async () => {
+  let reviewing = false;
+  const audioRanges: number[][] = [];
+  const imageTimes: number[] = [];
   const audioPaths: string[] = [];
   const imagePaths: string[] = [];
   const recordMediaPath = (mediaInput: MediaInput): string =>
@@ -319,6 +322,10 @@ test('CardCreationService uses stream-open-filename for remote media generation'
         behavior: {},
         ai: false,
       }) as AnkiConnectConfig,
+    reviewMediaTiming: async () =>
+      reviewing
+        ? { action: 'confirm', startTime: 0.2, endTime: 0.8, screenshotTime: 3.125 }
+        : { action: 'use-original' },
     getAiConfig: () => ({}),
     getTimingTracker: () => ({}) as never,
     getMpvClient: () =>
@@ -349,16 +356,18 @@ test('CardCreationService uses stream-open-filename for remote media generation'
       ],
       updateNoteFields: async () => undefined,
       storeMediaFile: async () => undefined,
-      findNotes: async () => [],
+      findNotes: async () => [42],
       retrieveMediaFile: async () => '',
       deleteNotes: async () => undefined,
     },
     mediaGenerator: {
-      generateAudio: async (path) => {
+      generateAudio: async (path, start, end, padding) => {
+        audioRanges.push([start, end, padding ?? -1]);
         audioPaths.push(recordMediaPath(path));
         return Buffer.from('audio');
       },
-      generateScreenshot: async (path) => {
+      generateScreenshot: async (path, timestamp) => {
+        imageTimes.push(timestamp);
         imagePaths.push(recordMediaPath(path));
         return Buffer.from('image');
       },
@@ -406,6 +415,14 @@ test('CardCreationService uses stream-open-filename for remote media generation'
   assert.equal(created, true);
   assert.deepEqual(audioPaths, [audioUrl]);
   assert.deepEqual(imagePaths, [videoUrl]);
+  reviewing = true;
+  assert.equal(await service.createSentenceCard('テスト', 0, 1), true);
+  assert.deepEqual(audioRanges.at(-1), [0.2, 0.8, 0]);
+  assert.equal(imageTimes.at(-1), 3.125);
+  await service.markLastCardAsAudioCard();
+  assert.equal(imageTimes.length, 3);
+  assert.deepEqual(audioRanges.at(-1), [0.2, 0.8, 0]);
+  assert.equal(imageTimes.at(-1), 3.125);
 });
 
 test('CardCreationService does not use mpv stream indexes for ready cached YouTube media', async () => {

src/anki-integration/card-creation.ts

@@ -536,6 +536,7 @@ export class CardCreationService {
               endTime,
               animatedLeadInSeconds,
               exactReviewedRange,
+              timingDecision.action === 'confirm' ? timingDecision.screenshotTime : undefined,
             );
 
             const imageField = this.deps.getConfig().fields?.image;
@@ -796,6 +797,9 @@ export class CardCreationService {
             generateImage,
             volumeScale,
             ...(exactReviewedRange ? { mediaPaddingSeconds: 0 } : {}),
+            ...(timingDecision.action === 'confirm' && timingDecision.screenshotTime !== undefined
+              ? { screenshotTime: timingDecision.screenshotTime }
+              : {}),
           });
           await this.deps.showNotification(noteId, label, 'media queued');
           return true;
@@ -840,6 +844,7 @@ export class CardCreationService {
               endTime,
               0,
               exactReviewedRange,
+              timingDecision.action === 'confirm' ? timingDecision.screenshotTime : undefined,
             );
 
             const imageField = config.fields?.image;
@@ -922,15 +927,16 @@ export class CardCreationService {
     endTime: number,
     animatedLeadInSeconds = 0,
     exactReviewedRange = false,
+    screenshotTime?: number,
   ): Promise<Buffer | null> {
     const mpvClient = this.deps.getMpvClient();
     if (!mpvClient) {
       return null;
     }
 
-    const timestamp = exactReviewedRange
-      ? startTime + (endTime - startTime) / 2
-      : mpvClient.currentTimePos || 0;
+    const timestamp =
+      screenshotTime ??
+      (exactReviewedRange ? startTime + (endTime - startTime) / 2 : mpvClient.currentTimePos || 0);
 
     if (this.deps.getConfig().media?.imageType === 'avif') {
       let imageStart = startTime;

src/anki-integration/note-update-workflow.test.ts

@@ -729,6 +729,7 @@ test('NoteUpdateWorkflow uses the combined review sentence for the card and medi
     startTime: 2,
     endTime: 7,
     text: 'previous-line current-line next-line',
+    screenshotTime: 8,
   });
   harness.deps.generateAudio = async (context) => {
     audioContexts.push(context);
@@ -745,6 +746,7 @@ test('NoteUpdateWorkflow uses the combined review sentence for the card and medi
   assert.equal(audioContexts[0]?.startTime, 2);
   assert.equal(audioContexts[0]?.endTime, 7);
   assert.equal(audioContexts[0]?.mediaPaddingSeconds, 0);
+  assert.equal(audioContexts[0]?.screenshotTime, 8);
 });
 
 test('NoteUpdateWorkflow keeps cache unchanged and reports when deletion fails', async () => {

src/anki-integration/note-update-workflow.ts

@@ -253,6 +253,9 @@ export class NoteUpdateWorkflow {
             startTime: timingDecision.startTime,
             endTime: timingDecision.endTime,
             mediaPaddingSeconds: 0,
+            ...(timingDecision.screenshotTime !== undefined
+              ? { screenshotTime: timingDecision.screenshotTime }
+              : {}),
           };
         } else if (timingDecision.action === 'skip-media') {
           skipMedia = true;

src/anki-integration/pending-youtube-media-queue.test.ts

@@ -51,6 +51,41 @@ function createDeps(
   return deps;
 }
 
+test('queued media keeps a chosen screenshot separate from the reviewed audio range', async () => {
+  const screenshots: number[] = [];
+  const audioRanges: number[][] = [];
+  const deps = createDeps();
+  deps.client.notesInfo = async () => [{ noteId: 42, fields: { Picture: { value: '' } } }];
+  deps.mediaGenerator.generateScreenshot = async (_media, time) => {
+    screenshots.push(time);
+    return Buffer.from('image');
+  };
+  deps.mediaGenerator.generateAudio = async (_media, start, end, padding) => {
+    audioRanges.push([start, end, padding ?? -1]);
+    return Buffer.from('audio');
+  };
+  const queue = new PendingYoutubeMediaQueue(deps);
+  assert.equal(
+    await queue.queueFromNote({
+      noteId: 42,
+      noteInfo: { noteId: 42, fields: {} },
+      label: 'test',
+      context: {
+        source: 'overlay',
+        text: '字幕',
+        startTime: 1,
+        endTime: 2,
+        mediaPaddingSeconds: 0,
+        screenshotTime: 3.125,
+      },
+    }),
+    true,
+  );
+  await queue.handleReady('https://youtu.be/abc123', '/cache/video.mkv');
+  assert.deepEqual(screenshots, [3.125]);
+  assert.deepEqual(audioRanges, [[1, 2, 0]]);
+});
+
 test('PendingYoutubeMediaQueue treats cache lookup failures as an immediate generation fallback', async () => {
   const deps = createDeps({
     getCachedMediaPath: async () => {

src/anki-integration/pending-youtube-media-queue.ts

@@ -147,6 +147,9 @@ export class PendingYoutubeMediaQueue {
       generateAudio: shouldGenerateAudio(config),
       generateImage: shouldGenerateImage(config),
       volumeScale,
+      ...(job.context?.screenshotTime !== undefined
+        ? { screenshotTime: job.context.screenshotTime }
+        : {}),
       ...(job.context?.mediaPaddingSeconds !== undefined
         ? { mediaPaddingSeconds: job.context.mediaPaddingSeconds }
         : {}),
@@ -320,6 +323,7 @@ export class PendingYoutubeMediaQueue {
           job.endTime,
           animatedLeadInSeconds,
           job.mediaPaddingSeconds,
+          job.screenshotTime,
         );
         if (imageBuffer) {
           await this.deps.client.storeMediaFile(imageFilename, imageBuffer);
@@ -381,6 +385,7 @@ export class PendingYoutubeMediaQueue {
     endTime: number,
     animatedLeadInSeconds = 0,
     mediaPaddingSeconds?: number,
+    screenshotTime?: number,
   ): Promise<Buffer | null> {
     const config = this.deps.getConfig();
     if (config.media?.imageType === 'avif') {
@@ -399,7 +404,7 @@ export class PendingYoutubeMediaQueue {
       );
     }
 
-    const timestamp = startTime + (endTime - startTime) / 2;
+    const timestamp = screenshotTime ?? startTime + (endTime - startTime) / 2;
     return this.deps.mediaGenerator.generateScreenshot(videoPath, timestamp, {
       format: config.media?.imageFormat as 'jpg' | 'png' | 'webp',
       quality: config.media?.imageQuality,

src/anki-integration/pending-youtube-media.ts

@@ -11,6 +11,7 @@ export interface PendingYoutubeMediaUpdate {
   generateImage: boolean;
   volumeScale?: number;
   mediaPaddingSeconds?: number;
+  screenshotTime?: number;
 }
 
 function trimToNonEmptyString(value: unknown): string | null {