fix(anki): snapshot mining media clip timing (#197)

src/anki-integration.ts

@@ -42,6 +42,7 @@ import {
   getPreferredWordValueFromExtractedFields,
 } from './anki-field-config';
 import { createLogger } from './logger';
+import { captureLiveSubtitleMiningContext } from './core/services/mining';
 import {
   createUiFeedbackState,
   beginUpdateProgress,
@@ -669,6 +670,7 @@ export class AnkiIntegration {
       formatMiscInfoPattern: (fallbackFilename, startTimeSeconds) =>
         this.formatMiscInfoPattern(fallbackFilename, startTimeSeconds),
       consumeSubtitleMiningContext: () => this.consumeSubtitleMiningContext(),
+      captureSubtitleMediaContext: () => captureLiveSubtitleMiningContext(this.mpvClient),
       queuePendingYoutubeMediaUpdate: (job) => this.queuePendingYoutubeMediaUpdateForNote(job),
       addConfiguredTagsToNote: (noteId) => this.addConfiguredTagsToNote(noteId),
       showNotification: (noteId, label) => this.showNotification(noteId, label),

src/anki-integration/note-update-workflow.test.ts

@@ -472,6 +472,71 @@ test('NoteUpdateWorkflow uses subtitle sidebar context for sentence media timing
   assert.equal(miscInfoStartTime, 10);
 });
 
+test('NoteUpdateWorkflow snapshots one media range for audio and image without a mining context', async () => {
+  const harness = createWorkflowHarness();
+  const capturedContext: SubtitleMiningContext = {
+    source: 'overlay',
+    text: 'subtitle-text',
+    startTime: 31.5,
+    endTime: 34.25,
+  };
+  let captureCalls = 0;
+  let audioContext: SubtitleMiningContext | null = null;
+  let imageContext: SubtitleMiningContext | null = null;
+  let miscInfoStartTime: number | undefined;
+
+  harness.deps.client.notesInfo = async () =>
+    [
+      {
+        noteId: 42,
+        fields: {
+          Expression: { value: 'taberu' },
+          Sentence: { value: '' },
+          SentenceAudio: { value: '' },
+          Picture: { value: '' },
+          MiscInfo: { value: '' },
+        },
+      },
+    ] satisfies NoteUpdateWorkflowNoteInfo[];
+  harness.deps.getConfig = () => ({
+    fields: {
+      sentence: 'Sentence',
+      image: 'Picture',
+      miscInfo: 'MiscInfo',
+    },
+    media: {
+      generateAudio: true,
+      generateImage: true,
+      imageType: 'avif',
+    },
+    behavior: {},
+  });
+  harness.deps.getResolvedSentenceAudioFieldName = () => 'SentenceAudio';
+  harness.deps.captureSubtitleMediaContext = () => {
+    captureCalls += 1;
+    return capturedContext;
+  };
+  harness.deps.generateAudio = async (context?: SubtitleMiningContext) => {
+    audioContext = context ?? null;
+    return Buffer.from('audio');
+  };
+  harness.deps.generateImage = async (_leadInSeconds?: number, context?: SubtitleMiningContext) => {
+    imageContext = context ?? null;
+    return Buffer.from('image');
+  };
+  harness.deps.formatMiscInfoPattern = (_fallbackFilename, startTimeSeconds) => {
+    miscInfoStartTime = startTimeSeconds;
+    return `start:${startTimeSeconds}`;
+  };
+
+  await harness.workflow.execute(42);
+
+  assert.equal(captureCalls, 1);
+  assert.deepEqual(audioContext, capturedContext);
+  assert.deepEqual(imageContext, capturedContext);
+  assert.equal(miscInfoStartTime, 31.5);
+});
+
 test('NoteUpdateWorkflow queues media updates when YouTube cache is pending', async () => {
   const harness = createWorkflowHarness();
   const queuedUpdates: Array<{

src/anki-integration/note-update-workflow.ts

@@ -87,6 +87,7 @@ export interface NoteUpdateWorkflowDeps {
   ) => Promise<Buffer | null>;
   formatMiscInfoPattern: (fallbackFilename: string, startTimeSeconds?: number) => string;
   consumeSubtitleMiningContext?: () => SubtitleMiningContext | null;
+  captureSubtitleMediaContext?: () => SubtitleMiningContext | null;
   queuePendingYoutubeMediaUpdate?: (job: {
     noteId: number;
     noteInfo: NoteUpdateWorkflowNoteInfo;
@@ -203,6 +204,11 @@ export class NoteUpdateWorkflow {
         sentenceField,
         config.fields?.sentence,
       );
+      // Audio and image generation run sequentially and audio extraction can take tens of
+      // seconds, so resolve the clip range exactly once up front; reading live mpv sub
+      // timings per generator clips whichever line is on screen when each one starts.
+      const mediaTimingContext =
+        subtitleMiningContext ?? this.deps.captureSubtitleMediaContext?.() ?? null;
       const noteLabel = hasExpressionText ? expressionText : noteId;
 
       const currentSubtitleText = subtitleMiningContext?.text ?? this.deps.getCurrentSubtitleText();
@@ -240,7 +246,7 @@ export class NoteUpdateWorkflow {
           ? await this.deps.queuePendingYoutubeMediaUpdate({
               noteId,
               noteInfo,
-              context: subtitleMiningContext ?? undefined,
+              context: mediaTimingContext ?? undefined,
               label: noteLabel,
             })
           : false;
@@ -248,7 +254,7 @@ export class NoteUpdateWorkflow {
       if (!mediaCacheQueued && generateAudio) {
         try {
           const audioFilename = this.deps.generateAudioFilename();
-          const audioBuffer = await this.deps.generateAudio(subtitleMiningContext ?? undefined);
+          const audioBuffer = await this.deps.generateAudio(mediaTimingContext ?? undefined);
 
           if (audioBuffer) {
             await this.deps.client.storeMediaFile(audioFilename, audioBuffer);
@@ -276,7 +282,7 @@ export class NoteUpdateWorkflow {
           const imageFilename = this.deps.generateImageFilename();
           const imageBuffer = await this.deps.generateImage(
             animatedLeadInSeconds,
-            subtitleMiningContext ?? undefined,
+            mediaTimingContext ?? undefined,
           );
 
           if (imageBuffer) {
@@ -308,7 +314,7 @@ export class NoteUpdateWorkflow {
       if (!mediaCacheQueued && config.fields?.miscInfo) {
         const miscInfo = this.deps.formatMiscInfoPattern(
           miscInfoFilename || '',
-          subtitleMiningContext?.startTime ?? this.deps.getCurrentSubtitleStart(),
+          mediaTimingContext?.startTime ?? this.deps.getCurrentSubtitleStart(),
         );
         const miscInfoField = this.deps.resolveConfiguredFieldName(
           noteInfo,

src/core/services/index.ts

@@ -16,6 +16,7 @@ export {
 export { createOverlayShortcutRuntimeHandlers } from './overlay-shortcut-handler';
 export { createCliCommandDepsRuntime, handleCliCommand } from './cli-command';
 export {
+  captureLiveSubtitleMiningContext,
   copyCurrentSubtitle,
   handleMineSentenceDigit,
   handleMultiCopyDigit,

src/core/services/mining.test.ts

@@ -1,6 +1,7 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import {
+  captureLiveSubtitleMiningContext,
   copyCurrentSubtitle,
   handleMineSentenceDigit,
   handleMultiCopyDigit,
@@ -345,3 +346,46 @@ test('handleMineSentenceDigit joins per-entry secondary subtitles when available
     tracker.destroy();
   }
 });
+
+test('captureLiveSubtitleMiningContext snapshots the current line and timings', () => {
+  const context = captureLiveSubtitleMiningContext(
+    { currentSubText: ' 食べる ', currentSubStart: 12.5, currentSubEnd: 15.25 },
+    () => 1234,
+  );
+
+  assert.deepEqual(context, {
+    source: 'overlay',
+    text: '食べる',
+    startTime: 12.5,
+    endTime: 15.25,
+    capturedAtMs: 1234,
+  });
+});
+
+test('captureLiveSubtitleMiningContext rejects missing client, empty text and bad timings', () => {
+  assert.equal(captureLiveSubtitleMiningContext(null), null);
+  assert.equal(
+    captureLiveSubtitleMiningContext({
+      currentSubText: '  ',
+      currentSubStart: 1,
+      currentSubEnd: 2,
+    }),
+    null,
+  );
+  assert.equal(
+    captureLiveSubtitleMiningContext({
+      currentSubText: 'line',
+      currentSubStart: 5,
+      currentSubEnd: 5,
+    }),
+    null,
+  );
+  assert.equal(
+    captureLiveSubtitleMiningContext({
+      currentSubText: 'line',
+      currentSubStart: NaN,
+      currentSubEnd: 2,
+    }),
+    null,
+  );
+});

src/core/services/mining.ts

@@ -1,4 +1,5 @@
 import type { SubtitleTimingBlock } from '../../subtitle-timing-tracker';
+import type { SubtitleMiningContext } from '../../types/subtitle';
 
 interface SubtitleTimingTrackerLike {
   getRecentBlocks: (count: number) => string[];
@@ -54,6 +55,27 @@ export function handleMultiCopyDigit(
   }
 }
 
+/**
+ * Snapshot the live mpv subtitle line and its timings as a mining context. Media
+ * generation can run tens of seconds after the user mines (slow audio extraction),
+ * so anything that reads `currentSubStart`/`currentSubEnd` lazily clips whichever
+ * line is on screen by then; callers capture here at mining time instead.
+ */
+export function captureLiveSubtitleMiningContext(
+  client: Pick<MpvClientLike, 'currentSubText' | 'currentSubStart' | 'currentSubEnd'> | null,
+  now: () => number = Date.now,
+): SubtitleMiningContext | null {
+  if (!client) {
+    return null;
+  }
+  const text = client.currentSubText?.trim();
+  const { currentSubStart: startTime, currentSubEnd: endTime } = client;
+  if (!text || !Number.isFinite(startTime) || !Number.isFinite(endTime) || endTime <= startTime) {
+    return null;
+  }
+  return { source: 'overlay', text, startTime, endTime, capturedAtMs: now() };
+}
+
 export function copyCurrentSubtitle(deps: {
   subtitleTimingTracker: SubtitleTimingTrackerLike | null;
   writeClipboardText: (text: string) => void;

src/main.ts

@@ -275,6 +275,7 @@ import {
   applyMpvSubtitleRenderMetricsPatch,
   authenticateWithPasswordRuntime,
   broadcastRuntimeOptionsChangedRuntime,
+  captureLiveSubtitleMiningContext,
   copyCurrentSubtitle as copyCurrentSubtitleCore,
   createConfigHotReloadRuntime,
   createDiscordPresenceService,
@@ -5535,7 +5536,13 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
       },
       onYoutubePickerResolve: (request) => youtubeFlowRuntime.resolveActivePicker(request),
       openYomitanSettings: () => openYomitanSettings(),
-      recordSubtitleMiningContext: (context) => recordSubtitleMiningContext(context),
+      // Overlay lookups carry no cue context of their own; fall back to snapshotting the
+      // live mpv sub timings at lookup time so media generation clips the mined line even
+      // when extraction finishes long after playback has moved on.
+      recordSubtitleMiningContext: (context) =>
+        recordSubtitleMiningContext(
+          context ?? captureLiveSubtitleMiningContext(appState.mpvClient),
+        ),
       quitApp: () => requestAppQuit(),
       toggleVisibleOverlay: () => toggleVisibleOverlay(),
       tokenizeCurrentSubtitle: async () => {

src/types/subtitle.ts

@@ -238,7 +238,7 @@ export interface SubtitleSidebarSnapshot {
 }
 
 export interface SubtitleMiningContext {
-  source: 'subtitle-sidebar';
+  source: 'subtitle-sidebar' | 'overlay';
   text: string;
   startTime: number;
   endTime: number;