From c5a77ac0678f3a70daabea00412588fa3ec67be2 Mon Sep 17 00:00:00 2001 From: sudacode Date: Fri, 14 Aug 2026 21:50:32 -0700 Subject: [PATCH] fix(anki): snapshot mining media clip timing (#197) --- changes/mining-clip-range-snapshot.md | 4 ++ src/anki-integration.ts | 2 + .../note-update-workflow.test.ts | 65 +++++++++++++++++++ src/anki-integration/note-update-workflow.ts | 14 ++-- src/core/services/index.ts | 1 + src/core/services/mining.test.ts | 44 +++++++++++++ src/core/services/mining.ts | 22 +++++++ src/main.ts | 9 ++- src/types/subtitle.ts | 2 +- 9 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 changes/mining-clip-range-snapshot.md diff --git a/changes/mining-clip-range-snapshot.md b/changes/mining-clip-range-snapshot.md new file mode 100644 index 00000000..d31d6176 --- /dev/null +++ b/changes/mining-clip-range-snapshot.md @@ -0,0 +1,4 @@ +type: fixed +area: anki + +- Mined audio and animated AVIF clips now capture the subtitle line that was actually mined. The clip range is snapshotted once at Yomitan lookup time (and reused for both audio and image), instead of each generator reading the live mpv subtitle when it starts — which clipped whatever line was on screen after slow audio extraction finished, producing too-short or misaligned AVIF clips. diff --git a/src/anki-integration.ts b/src/anki-integration.ts index 1263be07..fcc118e1 100644 --- a/src/anki-integration.ts +++ b/src/anki-integration.ts @@ -42,6 +42,7 @@ import { getPreferredWordValueFromExtractedFields, } from './anki-field-config'; import { createLogger } from './logger'; +import { captureLiveSubtitleMiningContext } from './core/services/mining'; import { createUiFeedbackState, beginUpdateProgress, @@ -669,6 +670,7 @@ export class AnkiIntegration { formatMiscInfoPattern: (fallbackFilename, startTimeSeconds) => this.formatMiscInfoPattern(fallbackFilename, startTimeSeconds), consumeSubtitleMiningContext: () => this.consumeSubtitleMiningContext(), + captureSubtitleMediaContext: () => captureLiveSubtitleMiningContext(this.mpvClient), queuePendingYoutubeMediaUpdate: (job) => this.queuePendingYoutubeMediaUpdateForNote(job), addConfiguredTagsToNote: (noteId) => this.addConfiguredTagsToNote(noteId), showNotification: (noteId, label) => this.showNotification(noteId, label), diff --git a/src/anki-integration/note-update-workflow.test.ts b/src/anki-integration/note-update-workflow.test.ts index 54ca44cd..2138b538 100644 --- a/src/anki-integration/note-update-workflow.test.ts +++ b/src/anki-integration/note-update-workflow.test.ts @@ -472,6 +472,71 @@ test('NoteUpdateWorkflow uses subtitle sidebar context for sentence media timing assert.equal(miscInfoStartTime, 10); }); +test('NoteUpdateWorkflow snapshots one media range for audio and image without a mining context', async () => { + const harness = createWorkflowHarness(); + const capturedContext: SubtitleMiningContext = { + source: 'overlay', + text: 'subtitle-text', + startTime: 31.5, + endTime: 34.25, + }; + let captureCalls = 0; + let audioContext: SubtitleMiningContext | null = null; + let imageContext: SubtitleMiningContext | null = null; + let miscInfoStartTime: number | undefined; + + harness.deps.client.notesInfo = async () => + [ + { + noteId: 42, + fields: { + Expression: { value: 'taberu' }, + Sentence: { value: '' }, + SentenceAudio: { value: '' }, + Picture: { value: '' }, + MiscInfo: { value: '' }, + }, + }, + ] satisfies NoteUpdateWorkflowNoteInfo[]; + harness.deps.getConfig = () => ({ + fields: { + sentence: 'Sentence', + image: 'Picture', + miscInfo: 'MiscInfo', + }, + media: { + generateAudio: true, + generateImage: true, + imageType: 'avif', + }, + behavior: {}, + }); + harness.deps.getResolvedSentenceAudioFieldName = () => 'SentenceAudio'; + harness.deps.captureSubtitleMediaContext = () => { + captureCalls += 1; + return capturedContext; + }; + harness.deps.generateAudio = async (context?: SubtitleMiningContext) => { + audioContext = context ?? null; + return Buffer.from('audio'); + }; + harness.deps.generateImage = async (_leadInSeconds?: number, context?: SubtitleMiningContext) => { + imageContext = context ?? null; + return Buffer.from('image'); + }; + harness.deps.formatMiscInfoPattern = (_fallbackFilename, startTimeSeconds) => { + miscInfoStartTime = startTimeSeconds; + return `start:${startTimeSeconds}`; + }; + + await harness.workflow.execute(42); + + assert.equal(captureCalls, 1); + assert.deepEqual(audioContext, capturedContext); + assert.deepEqual(imageContext, capturedContext); + assert.equal(miscInfoStartTime, 31.5); +}); + test('NoteUpdateWorkflow queues media updates when YouTube cache is pending', async () => { const harness = createWorkflowHarness(); const queuedUpdates: Array<{ diff --git a/src/anki-integration/note-update-workflow.ts b/src/anki-integration/note-update-workflow.ts index ff1aec6d..0348bdeb 100644 --- a/src/anki-integration/note-update-workflow.ts +++ b/src/anki-integration/note-update-workflow.ts @@ -87,6 +87,7 @@ export interface NoteUpdateWorkflowDeps { ) => Promise; formatMiscInfoPattern: (fallbackFilename: string, startTimeSeconds?: number) => string; consumeSubtitleMiningContext?: () => SubtitleMiningContext | null; + captureSubtitleMediaContext?: () => SubtitleMiningContext | null; queuePendingYoutubeMediaUpdate?: (job: { noteId: number; noteInfo: NoteUpdateWorkflowNoteInfo; @@ -203,6 +204,11 @@ export class NoteUpdateWorkflow { sentenceField, config.fields?.sentence, ); + // Audio and image generation run sequentially and audio extraction can take tens of + // seconds, so resolve the clip range exactly once up front; reading live mpv sub + // timings per generator clips whichever line is on screen when each one starts. + const mediaTimingContext = + subtitleMiningContext ?? this.deps.captureSubtitleMediaContext?.() ?? null; const noteLabel = hasExpressionText ? expressionText : noteId; const currentSubtitleText = subtitleMiningContext?.text ?? this.deps.getCurrentSubtitleText(); @@ -240,7 +246,7 @@ export class NoteUpdateWorkflow { ? await this.deps.queuePendingYoutubeMediaUpdate({ noteId, noteInfo, - context: subtitleMiningContext ?? undefined, + context: mediaTimingContext ?? undefined, label: noteLabel, }) : false; @@ -248,7 +254,7 @@ export class NoteUpdateWorkflow { if (!mediaCacheQueued && generateAudio) { try { const audioFilename = this.deps.generateAudioFilename(); - const audioBuffer = await this.deps.generateAudio(subtitleMiningContext ?? undefined); + const audioBuffer = await this.deps.generateAudio(mediaTimingContext ?? undefined); if (audioBuffer) { await this.deps.client.storeMediaFile(audioFilename, audioBuffer); @@ -276,7 +282,7 @@ export class NoteUpdateWorkflow { const imageFilename = this.deps.generateImageFilename(); const imageBuffer = await this.deps.generateImage( animatedLeadInSeconds, - subtitleMiningContext ?? undefined, + mediaTimingContext ?? undefined, ); if (imageBuffer) { @@ -308,7 +314,7 @@ export class NoteUpdateWorkflow { if (!mediaCacheQueued && config.fields?.miscInfo) { const miscInfo = this.deps.formatMiscInfoPattern( miscInfoFilename || '', - subtitleMiningContext?.startTime ?? this.deps.getCurrentSubtitleStart(), + mediaTimingContext?.startTime ?? this.deps.getCurrentSubtitleStart(), ); const miscInfoField = this.deps.resolveConfiguredFieldName( noteInfo, diff --git a/src/core/services/index.ts b/src/core/services/index.ts index 149d7248..ecaeb03a 100644 --- a/src/core/services/index.ts +++ b/src/core/services/index.ts @@ -16,6 +16,7 @@ export { export { createOverlayShortcutRuntimeHandlers } from './overlay-shortcut-handler'; export { createCliCommandDepsRuntime, handleCliCommand } from './cli-command'; export { + captureLiveSubtitleMiningContext, copyCurrentSubtitle, handleMineSentenceDigit, handleMultiCopyDigit, diff --git a/src/core/services/mining.test.ts b/src/core/services/mining.test.ts index d98e2060..7431ed6b 100644 --- a/src/core/services/mining.test.ts +++ b/src/core/services/mining.test.ts @@ -1,6 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { + captureLiveSubtitleMiningContext, copyCurrentSubtitle, handleMineSentenceDigit, handleMultiCopyDigit, @@ -345,3 +346,46 @@ test('handleMineSentenceDigit joins per-entry secondary subtitles when available tracker.destroy(); } }); + +test('captureLiveSubtitleMiningContext snapshots the current line and timings', () => { + const context = captureLiveSubtitleMiningContext( + { currentSubText: ' 食べる ', currentSubStart: 12.5, currentSubEnd: 15.25 }, + () => 1234, + ); + + assert.deepEqual(context, { + source: 'overlay', + text: '食べる', + startTime: 12.5, + endTime: 15.25, + capturedAtMs: 1234, + }); +}); + +test('captureLiveSubtitleMiningContext rejects missing client, empty text and bad timings', () => { + assert.equal(captureLiveSubtitleMiningContext(null), null); + assert.equal( + captureLiveSubtitleMiningContext({ + currentSubText: ' ', + currentSubStart: 1, + currentSubEnd: 2, + }), + null, + ); + assert.equal( + captureLiveSubtitleMiningContext({ + currentSubText: 'line', + currentSubStart: 5, + currentSubEnd: 5, + }), + null, + ); + assert.equal( + captureLiveSubtitleMiningContext({ + currentSubText: 'line', + currentSubStart: NaN, + currentSubEnd: 2, + }), + null, + ); +}); diff --git a/src/core/services/mining.ts b/src/core/services/mining.ts index 4871df7f..9dac45de 100644 --- a/src/core/services/mining.ts +++ b/src/core/services/mining.ts @@ -1,4 +1,5 @@ import type { SubtitleTimingBlock } from '../../subtitle-timing-tracker'; +import type { SubtitleMiningContext } from '../../types/subtitle'; interface SubtitleTimingTrackerLike { getRecentBlocks: (count: number) => string[]; @@ -54,6 +55,27 @@ export function handleMultiCopyDigit( } } +/** + * Snapshot the live mpv subtitle line and its timings as a mining context. Media + * generation can run tens of seconds after the user mines (slow audio extraction), + * so anything that reads `currentSubStart`/`currentSubEnd` lazily clips whichever + * line is on screen by then; callers capture here at mining time instead. + */ +export function captureLiveSubtitleMiningContext( + client: Pick | null, + now: () => number = Date.now, +): SubtitleMiningContext | null { + if (!client) { + return null; + } + const text = client.currentSubText?.trim(); + const { currentSubStart: startTime, currentSubEnd: endTime } = client; + if (!text || !Number.isFinite(startTime) || !Number.isFinite(endTime) || endTime <= startTime) { + return null; + } + return { source: 'overlay', text, startTime, endTime, capturedAtMs: now() }; +} + export function copyCurrentSubtitle(deps: { subtitleTimingTracker: SubtitleTimingTrackerLike | null; writeClipboardText: (text: string) => void; diff --git a/src/main.ts b/src/main.ts index 0d405a54..8ca54a75 100644 --- a/src/main.ts +++ b/src/main.ts @@ -275,6 +275,7 @@ import { applyMpvSubtitleRenderMetricsPatch, authenticateWithPasswordRuntime, broadcastRuntimeOptionsChangedRuntime, + captureLiveSubtitleMiningContext, copyCurrentSubtitle as copyCurrentSubtitleCore, createConfigHotReloadRuntime, createDiscordPresenceService, @@ -5535,7 +5536,13 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({ }, onYoutubePickerResolve: (request) => youtubeFlowRuntime.resolveActivePicker(request), openYomitanSettings: () => openYomitanSettings(), - recordSubtitleMiningContext: (context) => recordSubtitleMiningContext(context), + // Overlay lookups carry no cue context of their own; fall back to snapshotting the + // live mpv sub timings at lookup time so media generation clips the mined line even + // when extraction finishes long after playback has moved on. + recordSubtitleMiningContext: (context) => + recordSubtitleMiningContext( + context ?? captureLiveSubtitleMiningContext(appState.mpvClient), + ), quitApp: () => requestAppQuit(), toggleVisibleOverlay: () => toggleVisibleOverlay(), tokenizeCurrentSubtitle: async () => { diff --git a/src/types/subtitle.ts b/src/types/subtitle.ts index 332af1d8..409d0e2e 100644 --- a/src/types/subtitle.ts +++ b/src/types/subtitle.ts @@ -238,7 +238,7 @@ export interface SubtitleSidebarSnapshot { } export interface SubtitleMiningContext { - source: 'subtitle-sidebar'; + source: 'subtitle-sidebar' | 'overlay'; text: string; startTime: number; endTime: number;