fix(anki): snapshot mining media clip timing (#197)

changes/mining-clip-range-snapshot.md

@@ -0,0 +1,4 @@
+type: fixed
+area: anki
+
+- Mined audio and animated AVIF clips now capture the subtitle line that was actually mined. The clip range is snapshotted once at Yomitan lookup time (and reused for both audio and image), instead of each generator reading the live mpv subtitle when it starts — which clipped whatever line was on screen after slow audio extraction finished, producing too-short or misaligned AVIF clips.

src/anki-integration.ts

@@ -42,6 +42,7 @@ import {
   getPreferredWordValueFromExtractedFields,
 } from './anki-field-config';
 import { createLogger } from './logger';
+import { captureLiveSubtitleMiningContext } from './core/services/mining';
 import {
   createUiFeedbackState,
   beginUpdateProgress,
@@ -669,6 +670,7 @@ export class AnkiIntegration {
       formatMiscInfoPattern: (fallbackFilename, startTimeSeconds) =>
         this.formatMiscInfoPattern(fallbackFilename, startTimeSeconds),
       consumeSubtitleMiningContext: () => this.consumeSubtitleMiningContext(),
+      captureSubtitleMediaContext: () => captureLiveSubtitleMiningContext(this.mpvClient),
       queuePendingYoutubeMediaUpdate: (job) => this.queuePendingYoutubeMediaUpdateForNote(job),
       addConfiguredTagsToNote: (noteId) => this.addConfiguredTagsToNote(noteId),
       showNotification: (noteId, label) => this.showNotification(noteId, label),

src/anki-integration/note-update-workflow.test.ts

@@ -472,6 +472,71 @@ test('NoteUpdateWorkflow uses subtitle sidebar context for sentence media timing
   assert.equal(miscInfoStartTime, 10);
 });
 
+test('NoteUpdateWorkflow snapshots one media range for audio and image without a mining context', async () => {
+  const harness = createWorkflowHarness();
+  const capturedContext: SubtitleMiningContext = {
+    source: 'overlay',
+    text: 'subtitle-text',
+    startTime: 31.5,
+    endTime: 34.25,
+  };
+  let captureCalls = 0;
+  let audioContext: SubtitleMiningContext | null = null;
+  let imageContext: SubtitleMiningContext | null = null;
+  let miscInfoStartTime: number | undefined;
+
+  harness.deps.client.notesInfo = async () =>
+    [
+      {
+        noteId: 42,
+        fields: {
+          Expression: { value: 'taberu' },
+          Sentence: { value: '' },
+          SentenceAudio: { value: '' },
+          Picture: { value: '' },
+          MiscInfo: { value: '' },
+        },
+      },
+    ] satisfies NoteUpdateWorkflowNoteInfo[];
+  harness.deps.getConfig = () => ({
+    fields: {
+      sentence: 'Sentence',
+      image: 'Picture',
+      miscInfo: 'MiscInfo',
+    },
+    media: {
+      generateAudio: true,
+      generateImage: true,
+      imageType: 'avif',
+    },
+    behavior: {},
+  });
+  harness.deps.getResolvedSentenceAudioFieldName = () => 'SentenceAudio';
+  harness.deps.captureSubtitleMediaContext = () => {
+    captureCalls += 1;
+    return capturedContext;
+  };
+  harness.deps.generateAudio = async (context?: SubtitleMiningContext) => {
+    audioContext = context ?? null;
+    return Buffer.from('audio');
+  };
+  harness.deps.generateImage = async (_leadInSeconds?: number, context?: SubtitleMiningContext) => {
+    imageContext = context ?? null;
+    return Buffer.from('image');
+  };
+  harness.deps.formatMiscInfoPattern = (_fallbackFilename, startTimeSeconds) => {
+    miscInfoStartTime = startTimeSeconds;
+    return `start:${startTimeSeconds}`;
+  };
+
+  await harness.workflow.execute(42);
+
+  assert.equal(captureCalls, 1);
+  assert.deepEqual(audioContext, capturedContext);
+  assert.deepEqual(imageContext, capturedContext);
+  assert.equal(miscInfoStartTime, 31.5);
+});
+
 test('NoteUpdateWorkflow queues media updates when YouTube cache is pending', async () => {
   const harness = createWorkflowHarness();
   const queuedUpdates: Array<{

src/anki-integration/note-update-workflow.ts

@@ -87,6 +87,7 @@ export interface NoteUpdateWorkflowDeps {
   ) => Promise<Buffer | null>;
   formatMiscInfoPattern: (fallbackFilename: string, startTimeSeconds?: number) => string;
   consumeSubtitleMiningContext?: () => SubtitleMiningContext | null;
+  captureSubtitleMediaContext?: () => SubtitleMiningContext | null;
   queuePendingYoutubeMediaUpdate?: (job: {
     noteId: number;
     noteInfo: NoteUpdateWorkflowNoteInfo;
@@ -203,6 +204,11 @@ export class NoteUpdateWorkflow {
         sentenceField,
         config.fields?.sentence,
       );
+      // Audio and image generation run sequentially and audio extraction can take tens of
+      // seconds, so resolve the clip range exactly once up front; reading live mpv sub
+      // timings per generator clips whichever line is on screen when each one starts.
+      const mediaTimingContext =
+        subtitleMiningContext ?? this.deps.captureSubtitleMediaContext?.() ?? null;
       const noteLabel = hasExpressionText ? expressionText : noteId;
 
       const currentSubtitleText = subtitleMiningContext?.text ?? this.deps.getCurrentSubtitleText();
@@ -240,7 +246,7 @@ export class NoteUpdateWorkflow {
           ? await this.deps.queuePendingYoutubeMediaUpdate({
               noteId,
               noteInfo,
-              context: subtitleMiningContext ?? undefined,
+              context: mediaTimingContext ?? undefined,
               label: noteLabel,
             })
           : false;
@@ -248,7 +254,7 @@ export class NoteUpdateWorkflow {
       if (!mediaCacheQueued && generateAudio) {
         try {
           const audioFilename = this.deps.generateAudioFilename();
-          const audioBuffer = await this.deps.generateAudio(subtitleMiningContext ?? undefined);
+          const audioBuffer = await this.deps.generateAudio(mediaTimingContext ?? undefined);
 
           if (audioBuffer) {
             await this.deps.client.storeMediaFile(audioFilename, audioBuffer);
@@ -276,7 +282,7 @@ export class NoteUpdateWorkflow {
           const imageFilename = this.deps.generateImageFilename();
           const imageBuffer = await this.deps.generateImage(
             animatedLeadInSeconds,
-            subtitleMiningContext ?? undefined,
+            mediaTimingContext ?? undefined,
           );
 
           if (imageBuffer) {
@@ -308,7 +314,7 @@ export class NoteUpdateWorkflow {
       if (!mediaCacheQueued && config.fields?.miscInfo) {
         const miscInfo = this.deps.formatMiscInfoPattern(
           miscInfoFilename || '',
-          subtitleMiningContext?.startTime ?? this.deps.getCurrentSubtitleStart(),
+          mediaTimingContext?.startTime ?? this.deps.getCurrentSubtitleStart(),
         );
         const miscInfoField = this.deps.resolveConfiguredFieldName(
           noteInfo,

src/core/services/index.ts

@@ -16,6 +16,7 @@ export {
 export { createOverlayShortcutRuntimeHandlers } from './overlay-shortcut-handler';
 export { createCliCommandDepsRuntime, handleCliCommand } from './cli-command';
 export {
+  captureLiveSubtitleMiningContext,
   copyCurrentSubtitle,
   handleMineSentenceDigit,
   handleMultiCopyDigit,

src/core/services/mining.test.ts

@@ -1,6 +1,7 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import {
+  captureLiveSubtitleMiningContext,
   copyCurrentSubtitle,
   handleMineSentenceDigit,
   handleMultiCopyDigit,
@@ -345,3 +346,46 @@ test('handleMineSentenceDigit joins per-entry secondary subtitles when available
     tracker.destroy();
   }
 });
+
+test('captureLiveSubtitleMiningContext snapshots the current line and timings', () => {
+  const context = captureLiveSubtitleMiningContext(
+    { currentSubText: ' 食べる ', currentSubStart: 12.5, currentSubEnd: 15.25 },
+    () => 1234,
+  );
+
+  assert.deepEqual(context, {
+    source: 'overlay',
+    text: '食べる',
+    startTime: 12.5,
+    endTime: 15.25,
+    capturedAtMs: 1234,
+  });
+});
+
+test('captureLiveSubtitleMiningContext rejects missing client, empty text and bad timings', () => {
+  assert.equal(captureLiveSubtitleMiningContext(null), null);
+  assert.equal(
+    captureLiveSubtitleMiningContext({
+      currentSubText: '  ',
+      currentSubStart: 1,
+      currentSubEnd: 2,
+    }),
+    null,
+  );
+  assert.equal(
+    captureLiveSubtitleMiningContext({
+      currentSubText: 'line',
+      currentSubStart: 5,
+      currentSubEnd: 5,
+    }),
+    null,
+  );
+  assert.equal(
+    captureLiveSubtitleMiningContext({
+      currentSubText: 'line',
+      currentSubStart: NaN,
+      currentSubEnd: 2,
+    }),
+    null,
+  );
+});

src/core/services/mining.ts

@@ -1,4 +1,5 @@
 import type { SubtitleTimingBlock } from '../../subtitle-timing-tracker';
+import type { SubtitleMiningContext } from '../../types/subtitle';
 
 interface SubtitleTimingTrackerLike {
   getRecentBlocks: (count: number) => string[];
@@ -54,6 +55,27 @@ export function handleMultiCopyDigit(
   }
 }
 
+/**
+ * Snapshot the live mpv subtitle line and its timings as a mining context. Media
+ * generation can run tens of seconds after the user mines (slow audio extraction),
+ * so anything that reads `currentSubStart`/`currentSubEnd` lazily clips whichever
+ * line is on screen by then; callers capture here at mining time instead.
+ */
+export function captureLiveSubtitleMiningContext(
+  client: Pick<MpvClientLike, 'currentSubText' | 'currentSubStart' | 'currentSubEnd'> | null,
+  now: () => number = Date.now,
+): SubtitleMiningContext | null {
+  if (!client) {
+    return null;
+  }
+  const text = client.currentSubText?.trim();
+  const { currentSubStart: startTime, currentSubEnd: endTime } = client;
+  if (!text || !Number.isFinite(startTime) || !Number.isFinite(endTime) || endTime <= startTime) {
+    return null;
+  }
+  return { source: 'overlay', text, startTime, endTime, capturedAtMs: now() };
+}
+
 export function copyCurrentSubtitle(deps: {
   subtitleTimingTracker: SubtitleTimingTrackerLike | null;
   writeClipboardText: (text: string) => void;

src/main.ts

@@ -275,6 +275,7 @@ import {
   applyMpvSubtitleRenderMetricsPatch,
   authenticateWithPasswordRuntime,
   broadcastRuntimeOptionsChangedRuntime,
+  captureLiveSubtitleMiningContext,
   copyCurrentSubtitle as copyCurrentSubtitleCore,
   createConfigHotReloadRuntime,
   createDiscordPresenceService,
@@ -5535,7 +5536,13 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
       },
       onYoutubePickerResolve: (request) => youtubeFlowRuntime.resolveActivePicker(request),
       openYomitanSettings: () => openYomitanSettings(),
-      recordSubtitleMiningContext: (context) => recordSubtitleMiningContext(context),
+      // Overlay lookups carry no cue context of their own; fall back to snapshotting the
+      // live mpv sub timings at lookup time so media generation clips the mined line even
+      // when extraction finishes long after playback has moved on.
+      recordSubtitleMiningContext: (context) =>
+        recordSubtitleMiningContext(
+          context ?? captureLiveSubtitleMiningContext(appState.mpvClient),
+        ),
       quitApp: () => requestAppQuit(),
       toggleVisibleOverlay: () => toggleVisibleOverlay(),
       tokenizeCurrentSubtitle: async () => {

src/types/subtitle.ts

@@ -238,7 +238,7 @@ export interface SubtitleSidebarSnapshot {
 }
 
 export interface SubtitleMiningContext {
-  source: 'subtitle-sidebar';
+  source: 'subtitle-sidebar' | 'overlay';
   text: string;
   startTime: number;
   endTime: number;