feat(mining): add a screenshot frame picker to media review (#254)

This commit is contained in:
Abdulrazzaq Alhendi
2026-09-20 19:39:30 -07:00
committed by GitHub
parent 026d495fac
commit 4f0762e840
30 changed files with 1185 additions and 32 deletions
@@ -0,0 +1,4 @@
type: docs
area: mining
- Document independent screenshot selection in media timing review.
+4
View File
@@ -0,0 +1,4 @@
type: added
area: mining
- Choose a still screenshot independently of the audio in media timing review, with a live preview, a separate time slider, and decoded-frame stepping. Local video and seekable remote streams such as Jellyfin are supported.
+4
View File
@@ -189,6 +189,10 @@ The review opens on the subtitle range plus your configured audio padding. Subti
**Adjusting the clip.** Drag either edge to trim, drag the middle to slide the whole clip without changing its length, or click anywhere on the waveform to snap the nearer edge there. A focused edge also moves with the arrow keys: 100 ms per press, or 500 ms with Shift. The 100 ms buttons do the same thing. Earlier and Later each reveal two more seconds of timeline without moving the selection.
**Choosing the screenshot.** With still images enabled, drag the screenshot slider or use the Frame buttons to choose a video frame independently of the audio. Earlier and Later reveal more time for both sliders. The image follows the audio midpoint until you pick a frame, then stays fixed while you trim audio. Reset below the image restores automatic selection; the audio Reset affects only the audio. The screenshot slider also supports arrow keys, Home, and End.
The picker supports local video and seekable remote streams, including Jellyfin, without seeking the main player. A selected frame must load before you can confirm; if it fails, choose another frame or Reset. The picker is hidden when image generation is disabled or animated AVIF is selected. Animated images continue to use the reviewed audio range.
**Keys.** Space previews the selection with a playhead sweeping the clip. The preview ends when the hidden player has actually played the last sample, so Bluetooth output latency does not clip the tail. Enter confirms and Escape cancels.
**The waveform.** SubMiner reads a center channel when one carries dialogue and falls back to a mono mix otherwise, keeps only the 250 to 3500 Hz speech band, and draws each slice's loudness against the clip's own noise floor. Steady background music flattens out and dialogue stands up, which makes it much easier to tell adjacent lines apart. The mined subtitle appears as a tinted band with labeled line-start and line-end rails. If waveform analysis fails, the timing controls still work.
+7 -3
View File
@@ -1118,9 +1118,11 @@ export class AnkiIntegration {
return null;
}
const mediaRange = this.getSubtitleMediaRange(context);
const timestamp = context
? mediaRange.startTime + (mediaRange.endTime - mediaRange.startTime) / 2
: this.mpvClient.currentTimePos || 0;
const timestamp =
context?.screenshotTime ??
(context
? mediaRange.startTime + (mediaRange.endTime - mediaRange.startTime) / 2
: this.mpvClient.currentTimePos || 0);
if (this.config.media?.imageType === 'avif') {
return this.mediaGenerator.generateAnimatedImage(
@@ -1797,6 +1799,8 @@ export class AnkiIntegration {
...request,
audioPadding: Math.max(0, this.config.media.audioPadding ?? 0),
maxMediaDuration: Math.max(0, this.config.media.maxMediaDuration ?? 30),
screenshotEnabled:
this.config.media.generateImage !== false && this.config.media.imageType !== 'avif',
});
}
+20 -3
View File
@@ -290,6 +290,9 @@ test('CardCreationService keeps updating after recordCardsMinedCallback throws',
});
test('CardCreationService uses stream-open-filename for remote media generation', async () => {
let reviewing = false;
const audioRanges: number[][] = [];
const imageTimes: number[] = [];
const audioPaths: string[] = [];
const imagePaths: string[] = [];
const recordMediaPath = (mediaInput: MediaInput): string =>
@@ -319,6 +322,10 @@ test('CardCreationService uses stream-open-filename for remote media generation'
behavior: {},
ai: false,
}) as AnkiConnectConfig,
reviewMediaTiming: async () =>
reviewing
? { action: 'confirm', startTime: 0.2, endTime: 0.8, screenshotTime: 3.125 }
: { action: 'use-original' },
getAiConfig: () => ({}),
getTimingTracker: () => ({}) as never,
getMpvClient: () =>
@@ -349,16 +356,18 @@ test('CardCreationService uses stream-open-filename for remote media generation'
],
updateNoteFields: async () => undefined,
storeMediaFile: async () => undefined,
findNotes: async () => [],
findNotes: async () => [42],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
mediaGenerator: {
generateAudio: async (path) => {
generateAudio: async (path, start, end, padding) => {
audioRanges.push([start, end, padding ?? -1]);
audioPaths.push(recordMediaPath(path));
return Buffer.from('audio');
},
generateScreenshot: async (path) => {
generateScreenshot: async (path, timestamp) => {
imageTimes.push(timestamp);
imagePaths.push(recordMediaPath(path));
return Buffer.from('image');
},
@@ -406,6 +415,14 @@ test('CardCreationService uses stream-open-filename for remote media generation'
assert.equal(created, true);
assert.deepEqual(audioPaths, [audioUrl]);
assert.deepEqual(imagePaths, [videoUrl]);
reviewing = true;
assert.equal(await service.createSentenceCard('テスト', 0, 1), true);
assert.deepEqual(audioRanges.at(-1), [0.2, 0.8, 0]);
assert.equal(imageTimes.at(-1), 3.125);
await service.markLastCardAsAudioCard();
assert.equal(imageTimes.length, 3);
assert.deepEqual(audioRanges.at(-1), [0.2, 0.8, 0]);
assert.equal(imageTimes.at(-1), 3.125);
});
test('CardCreationService does not use mpv stream indexes for ready cached YouTube media', async () => {
+9 -3
View File
@@ -536,6 +536,7 @@ export class CardCreationService {
endTime,
animatedLeadInSeconds,
exactReviewedRange,
timingDecision.action === 'confirm' ? timingDecision.screenshotTime : undefined,
);
const imageField = this.deps.getConfig().fields?.image;
@@ -796,6 +797,9 @@ export class CardCreationService {
generateImage,
volumeScale,
...(exactReviewedRange ? { mediaPaddingSeconds: 0 } : {}),
...(timingDecision.action === 'confirm' && timingDecision.screenshotTime !== undefined
? { screenshotTime: timingDecision.screenshotTime }
: {}),
});
await this.deps.showNotification(noteId, label, 'media queued');
return true;
@@ -840,6 +844,7 @@ export class CardCreationService {
endTime,
0,
exactReviewedRange,
timingDecision.action === 'confirm' ? timingDecision.screenshotTime : undefined,
);
const imageField = config.fields?.image;
@@ -922,15 +927,16 @@ export class CardCreationService {
endTime: number,
animatedLeadInSeconds = 0,
exactReviewedRange = false,
screenshotTime?: number,
): Promise<Buffer | null> {
const mpvClient = this.deps.getMpvClient();
if (!mpvClient) {
return null;
}
const timestamp = exactReviewedRange
? startTime + (endTime - startTime) / 2
: mpvClient.currentTimePos || 0;
const timestamp =
screenshotTime ??
(exactReviewedRange ? startTime + (endTime - startTime) / 2 : mpvClient.currentTimePos || 0);
if (this.deps.getConfig().media?.imageType === 'avif') {
let imageStart = startTime;
@@ -729,6 +729,7 @@ test('NoteUpdateWorkflow uses the combined review sentence for the card and medi
startTime: 2,
endTime: 7,
text: 'previous-line current-line next-line',
screenshotTime: 8,
});
harness.deps.generateAudio = async (context) => {
audioContexts.push(context);
@@ -745,6 +746,7 @@ test('NoteUpdateWorkflow uses the combined review sentence for the card and medi
assert.equal(audioContexts[0]?.startTime, 2);
assert.equal(audioContexts[0]?.endTime, 7);
assert.equal(audioContexts[0]?.mediaPaddingSeconds, 0);
assert.equal(audioContexts[0]?.screenshotTime, 8);
});
test('NoteUpdateWorkflow keeps cache unchanged and reports when deletion fails', async () => {
@@ -253,6 +253,9 @@ export class NoteUpdateWorkflow {
startTime: timingDecision.startTime,
endTime: timingDecision.endTime,
mediaPaddingSeconds: 0,
...(timingDecision.screenshotTime !== undefined
? { screenshotTime: timingDecision.screenshotTime }
: {}),
};
} else if (timingDecision.action === 'skip-media') {
skipMedia = true;
@@ -51,6 +51,41 @@ function createDeps(
return deps;
}
test('queued media keeps a chosen screenshot separate from the reviewed audio range', async () => {
const screenshots: number[] = [];
const audioRanges: number[][] = [];
const deps = createDeps();
deps.client.notesInfo = async () => [{ noteId: 42, fields: { Picture: { value: '' } } }];
deps.mediaGenerator.generateScreenshot = async (_media, time) => {
screenshots.push(time);
return Buffer.from('image');
};
deps.mediaGenerator.generateAudio = async (_media, start, end, padding) => {
audioRanges.push([start, end, padding ?? -1]);
return Buffer.from('audio');
};
const queue = new PendingYoutubeMediaQueue(deps);
assert.equal(
await queue.queueFromNote({
noteId: 42,
noteInfo: { noteId: 42, fields: {} },
label: 'test',
context: {
source: 'overlay',
text: '字幕',
startTime: 1,
endTime: 2,
mediaPaddingSeconds: 0,
screenshotTime: 3.125,
},
}),
true,
);
await queue.handleReady('https://youtu.be/abc123', '/cache/video.mkv');
assert.deepEqual(screenshots, [3.125]);
assert.deepEqual(audioRanges, [[1, 2, 0]]);
});
test('PendingYoutubeMediaQueue treats cache lookup failures as an immediate generation fallback', async () => {
const deps = createDeps({
getCachedMediaPath: async () => {
@@ -147,6 +147,9 @@ export class PendingYoutubeMediaQueue {
generateAudio: shouldGenerateAudio(config),
generateImage: shouldGenerateImage(config),
volumeScale,
...(job.context?.screenshotTime !== undefined
? { screenshotTime: job.context.screenshotTime }
: {}),
...(job.context?.mediaPaddingSeconds !== undefined
? { mediaPaddingSeconds: job.context.mediaPaddingSeconds }
: {}),
@@ -320,6 +323,7 @@ export class PendingYoutubeMediaQueue {
job.endTime,
animatedLeadInSeconds,
job.mediaPaddingSeconds,
job.screenshotTime,
);
if (imageBuffer) {
await this.deps.client.storeMediaFile(imageFilename, imageBuffer);
@@ -381,6 +385,7 @@ export class PendingYoutubeMediaQueue {
endTime: number,
animatedLeadInSeconds = 0,
mediaPaddingSeconds?: number,
screenshotTime?: number,
): Promise<Buffer | null> {
const config = this.deps.getConfig();
if (config.media?.imageType === 'avif') {
@@ -399,7 +404,7 @@ export class PendingYoutubeMediaQueue {
);
}
const timestamp = startTime + (endTime - startTime) / 2;
const timestamp = screenshotTime ?? startTime + (endTime - startTime) / 2;
return this.deps.mediaGenerator.generateScreenshot(videoPath, timestamp, {
format: config.media?.imageFormat as 'jpg' | 'png' | 'webp',
quality: config.media?.imageQuality,
@@ -11,6 +11,7 @@ export interface PendingYoutubeMediaUpdate {
generateImage: boolean;
volumeScale?: number;
mediaPaddingSeconds?: number;
screenshotTime?: number;
}
function trimToNonEmptyString(value: unknown): string | null {
+42 -11
View File
@@ -671,7 +671,34 @@ test('registerIpcHandlers accepts the keep-without-media timing decision', async
assert.deepEqual(requests, [{ reviewId: 'review-1', decision: { action: 'skip-media' } }]);
});
test('registerIpcHandlers validates and forwards combined timing review text', async () => {
test('frame IPC validates timestamps and directions', async () => {
const { registrar, handlers } = createFakeIpcRegistrar();
const requests: unknown[] = [];
registerIpcHandlers(
createRegisterIpcDeps({
getMediaTimingReviewFrame: async (request) => {
requests.push(request);
return { ok: true };
},
}),
registrar,
);
const frame = handlers.handle.get(IPC_CHANNELS.request.mediaTimingReviewFrame)!;
const valid = { reviewId: 'r', timestamp: 13, direction: 1 };
assert.deepEqual(await frame({}, valid), { ok: true });
for (const invalid of [
null,
{},
{ ...valid, timestamp: NaN },
{ ...valid, timestamp: '13' },
{ ...valid, direction: 2 },
]) {
assert.equal(((await frame({}, invalid)) as { ok: boolean }).ok, false);
}
assert.deepEqual(requests, [valid]);
});
test('registerIpcHandlers validates and forwards timing review text and screenshot selection', async () => {
const { registrar, handlers } = createFakeIpcRegistrar();
const requests: unknown[] = [];
registerIpcHandlers(
@@ -696,6 +723,7 @@ test('registerIpcHandlers validates and forwards combined timing review text', a
startTime: 10,
endTime: 12,
text: '前の行 対象の行',
screenshotTime: 13,
},
},
),
@@ -709,20 +737,23 @@ test('registerIpcHandlers validates and forwards combined timing review text', a
startTime: 10,
endTime: 12,
text: '前の行 対象の行',
screenshotTime: 13,
},
},
]);
assert.deepEqual(
await handler!(
{},
{
reviewId: 'review-1',
decision: { action: 'confirm', startTime: 10, endTime: 12, text: ' ' },
},
),
{ ok: false, message: 'Timing review is unavailable.' },
);
for (const invalid of [{ text: ' ' }, { screenshotTime: Infinity }]) {
assert.deepEqual(
await handler!(
{},
{
reviewId: 'review-1',
decision: { action: 'confirm', startTime: 10, endTime: 12, ...invalid },
},
),
{ ok: false, message: 'Timing review is unavailable.' },
);
}
assert.equal(requests.length, 1);
});
+43
View File
@@ -24,6 +24,8 @@ import type {
MediaTimingReviewActionResult,
MediaTimingReviewPreviewRequest,
MediaTimingReviewResolveRequest,
MediaTimingReviewFrameRequest,
MediaTimingReviewFrameResult,
MediaTimingReviewWaveformRequest,
MediaTimingReviewWaveformResult,
} from '../../types/anki';
@@ -111,6 +113,9 @@ export interface IpcServiceDeps {
previewMediaTimingReview?: (
request: MediaTimingReviewPreviewRequest,
) => Promise<MediaTimingReviewActionResult>;
getMediaTimingReviewFrame?: (
request: MediaTimingReviewFrameRequest,
) => Promise<MediaTimingReviewFrameResult>;
getMediaTimingReviewWaveform?: (
request: MediaTimingReviewWaveformRequest,
) => Promise<MediaTimingReviewWaveformResult>;
@@ -269,6 +274,26 @@ function parseMediaTimingReviewWaveformRequest(
return parseMediaTimingReviewPreviewRequest(payload);
}
function parseMediaTimingReviewFrameRequest(
payload: unknown,
): MediaTimingReviewFrameRequest | null {
if (!payload || typeof payload !== 'object') return null;
const record = payload as Record<string, unknown>;
if (
typeof record.reviewId !== 'string' ||
!record.reviewId ||
typeof record.timestamp !== 'number' ||
!Number.isFinite(record.timestamp) ||
(record.direction !== undefined && record.direction !== -1 && record.direction !== 1)
)
return null;
return {
reviewId: record.reviewId,
timestamp: record.timestamp,
...(record.direction !== undefined ? { direction: record.direction as -1 | 1 } : {}),
};
}
function parseMediaTimingReviewResolveRequest(
payload: unknown,
): MediaTimingReviewResolveRequest | null {
@@ -291,6 +316,9 @@ function parseMediaTimingReviewResolveRequest(
Number.isFinite(decisionRecord.startTime) &&
typeof decisionRecord.endTime === 'number' &&
Number.isFinite(decisionRecord.endTime) &&
(decisionRecord.screenshotTime === undefined ||
(typeof decisionRecord.screenshotTime === 'number' &&
Number.isFinite(decisionRecord.screenshotTime))) &&
(decisionRecord.text === undefined ||
(typeof decisionRecord.text === 'string' && decisionRecord.text.trim().length > 0))
) {
@@ -300,6 +328,9 @@ function parseMediaTimingReviewResolveRequest(
action: 'confirm',
startTime: decisionRecord.startTime,
endTime: decisionRecord.endTime,
...(decisionRecord.screenshotTime === undefined
? {}
: { screenshotTime: decisionRecord.screenshotTime as number }),
...(decisionRecord.text === undefined ? {} : { text: decisionRecord.text }),
},
};
@@ -365,6 +396,7 @@ export interface IpcDepsRuntimeOptions {
request: YoutubePickerResolveRequest,
) => Promise<YoutubePickerResolveResult>;
previewMediaTimingReview?: IpcServiceDeps['previewMediaTimingReview'];
getMediaTimingReviewFrame?: IpcServiceDeps['getMediaTimingReviewFrame'];
getMediaTimingReviewWaveform?: IpcServiceDeps['getMediaTimingReviewWaveform'];
stopMediaTimingReviewPreview?: IpcServiceDeps['stopMediaTimingReviewPreview'];
resolveMediaTimingReview?: IpcServiceDeps['resolveMediaTimingReview'];
@@ -463,6 +495,7 @@ export function createIpcDepsRuntime(options: IpcDepsRuntimeOptions): IpcService
runSubsyncManual: options.runSubsyncManual,
onYoutubePickerResolve: options.onYoutubePickerResolve,
previewMediaTimingReview: options.previewMediaTimingReview,
getMediaTimingReviewFrame: options.getMediaTimingReviewFrame,
getMediaTimingReviewWaveform: options.getMediaTimingReviewWaveform,
stopMediaTimingReviewPreview: options.stopMediaTimingReviewPreview,
resolveMediaTimingReview: options.resolveMediaTimingReview,
@@ -613,6 +646,16 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar
return await deps.getMediaTimingReviewWaveform(request);
},
);
ipc.handle(
IPC_CHANNELS.request.mediaTimingReviewFrame,
async (_event: unknown, payload: unknown) => {
const request = parseMediaTimingReviewFrameRequest(payload);
if (!request || !deps.getMediaTimingReviewFrame) {
return { ok: false, message: 'Screenshot preview is unavailable.' };
}
return await deps.getMediaTimingReviewFrame(request);
},
);
ipc.handle(
IPC_CHANNELS.request.mediaTimingReviewStopPreview,
async (_event: unknown, reviewId: unknown) => {
@@ -0,0 +1,70 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { createMediaTimingFrameExtractor, selectMediaTimingFrame } from './media-timing-frame';
test('frame stepping follows decoded timestamps with variable frame durations', () => {
const times = [10, 10.041667, 10.125, 10.166667];
assert.equal(selectMediaTimingFrame(times, 10.05), 10.125);
assert.equal(selectMediaTimingFrame(times, 10.125 - 0.000001, 1), 10.166667);
assert.equal(selectMediaTimingFrame(times, 10.125 - 0.000001, -1), 10.041667);
assert.equal(selectMediaTimingFrame(times, 10, -1), undefined);
assert.equal(selectMediaTimingFrame(times, 10.166667, 1), undefined);
assert.equal(selectMediaTimingFrame([], 10), undefined);
});
function fixture(startTime = 0) {
const calls: Array<{ file: string; args: string[] }> = [];
const extractor = createMediaTimingFrameExtractor(async (file, args) => {
calls.push({ file, args });
if (file === 'ffmpeg') return Buffer.from('image');
if (args.includes('format=start_time'))
return Buffer.from(JSON.stringify({ format: { start_time: String(startTime) } }));
return Buffer.from(
JSON.stringify({
frames: [10, 10.04, 10.12, 10.16].map((time) => ({
best_effort_timestamp_time: String(time + startTime),
})),
}),
);
});
return { calls, extractor };
}
test('frame extraction normalizes nonzero source start times and reuses the frame index', async () => {
const { calls, extractor } = fixture(5);
const first = await extractor.generate({ media: '/movie.mkv', timestamp: 10.05 });
assert.ok(Math.abs(first.timestamp - 10.119999) < 0.000001);
assert.equal(first.dataUrl, 'data:image/jpeg;base64,aW1hZ2U=');
await extractor.generate({ media: '/movie.mkv', timestamp: first.timestamp, direction: 1 });
assert.equal(calls.filter((call) => call.file === 'ffprobe').length, 2);
assert.ok(calls[1]!.args.includes('13.05%17.05'));
extractor.clear();
await extractor.generate({ media: '/movie.mkv', timestamp: 10.05 });
assert.equal(calls.filter((call) => call.file === 'ffprobe').length, 4);
});
test('cached windows keep source timestamps for ffmpeg and use absolute ffprobe intervals', async () => {
const { calls, extractor } = fixture();
await extractor.generate({
media: { path: '/window.mkv', absoluteTimestamps: true },
timestamp: 10.05,
});
assert.ok(calls[1]!.args.includes('8.05%12.05'));
assert.equal(calls[1]!.args.includes('-seek_timestamp'), false);
assert.ok(calls[2]!.args.includes('-seek_timestamp'));
});
test('remote frame reads carry source headers and do not silently reuse a different input', async () => {
const { calls, extractor } = fixture();
const media = {
path: 'https://example.test/video',
inputOptions: { headers: { 'X-Emby-Token': 'test-token' } },
};
await extractor.generate({ media, timestamp: 10.05 });
assert.ok(calls.every((call) => call.args.includes('X-Emby-Token: test-token\r\n')));
await extractor.generate({
media: { ...media, path: 'https://example.test/other' },
timestamp: 10.05,
});
assert.equal(calls.filter((call) => call.file === 'ffprobe').length, 4);
});
+159
View File
@@ -0,0 +1,159 @@
import { execFile } from 'node:child_process';
import { normalizeMediaInput, type MediaInput } from '../../media-input';
export interface MediaTimingFrameOptions {
media: MediaInput;
timestamp: number;
direction?: -1 | 1;
}
const FRAME_EPSILON = 0.00001;
const PROBE_RADIUS_SECONDS = 2;
function run(file: string, args: string[]): Promise<Buffer> {
return new Promise((resolve, reject) => {
execFile(
file,
args,
{ encoding: 'buffer', timeout: 30_000, maxBuffer: 8 * 1024 * 1024, windowsHide: true },
(error, stdout) => {
// Child-process errors contain the input URL, which may contain authentication tokens.
if (error)
reject(
new Error('Screenshot preview is unavailable. Check FFmpeg and the video source.'),
);
else resolve(stdout);
},
);
});
}
/** Uses decoded timestamps, rather than an assumed FPS, including for variable-rate video. */
export function selectMediaTimingFrame(
times: readonly number[],
timestamp: number,
direction?: -1 | 1,
): number | undefined {
if (direction === -1)
return [...times].reverse().find((time) => time < timestamp - FRAME_EPSILON);
if (direction === 1) return times.find((time) => time > timestamp + FRAME_EPSILON);
return times.find((time) => time >= timestamp - FRAME_EPSILON) ?? times.at(-1);
}
export function createMediaTimingFrameExtractor(execute: typeof run = run) {
let cached: { key: string; start: number; end: number; times: number[] } | null = null;
let source: { key: string; offset: number } | null = null;
let generation = 0;
async function generate(
options: MediaTimingFrameOptions,
): Promise<{ dataUrl: string; timestamp: number }> {
const input = normalizeMediaInput(options.media);
const key = JSON.stringify(options.media);
const currentGeneration = generation;
const absolute = typeof options.media !== 'string' && options.media.absoluteTimestamps;
// -seek_timestamp is an ffmpeg option; ffprobe intervals already use stream timestamps.
const probeInputArgs = normalizeMediaInput({
path: input.path,
...(typeof options.media !== 'string' ? { inputOptions: options.media.inputOptions } : {}),
}).inputArgs;
let offset = source?.key === key ? source.offset : undefined;
if (offset === undefined) {
const metadata = JSON.parse(
(
await execute('ffprobe', [
'-v',
'error',
...probeInputArgs,
'-show_entries',
'format=start_time',
'-of',
'json',
input.path,
])
).toString(),
) as { format?: { start_time?: string } };
const start = Number(metadata.format?.start_time ?? 0);
offset = absolute || !Number.isFinite(start) ? 0 : start;
if (generation === currentGeneration) source = { key, offset };
}
let times: number[];
if (
cached?.key === key &&
options.timestamp > cached.start + 0.5 &&
options.timestamp < cached.end - 0.5
) {
times = cached.times;
} else {
const start = Math.max(0, options.timestamp - PROBE_RADIUS_SECONDS);
const end = options.timestamp + PROBE_RADIUS_SECONDS;
const result = JSON.parse(
(
await execute('ffprobe', [
'-v',
'error',
...probeInputArgs,
'-read_intervals',
`${start + offset}%${end + offset}`,
'-select_streams',
'v:0',
'-show_entries',
'frame=best_effort_timestamp_time',
'-of',
'json',
input.path,
])
).toString(),
) as { frames?: { best_effort_timestamp_time?: string }[] };
times = [
...new Set(
(result.frames ?? [])
.map((frame) => Number(frame.best_effort_timestamp_time) - offset)
.filter((time) => Number.isFinite(time) && time >= 0 && time >= start && time <= end),
),
].sort((a, b) => a - b);
if (generation === currentGeneration) cached = { key, start, end, times };
}
const timestamp = selectMediaTimingFrame(times, options.timestamp, options.direction);
if (timestamp === undefined) throw new Error('No adjacent video frame is available here.');
// Round down by one microsecond so decimal timestamp rounding cannot skip the chosen frame.
const seekTime = Math.max(0, timestamp - 0.000001);
const image = await execute('ffmpeg', [
'-hide_banner',
'-nostdin',
'-loglevel',
'error',
'-ss',
String(seekTime),
...input.inputArgs,
'-i',
input.path,
'-map',
'0:v:0',
'-frames:v',
'1',
'-an',
'-sn',
'-vf',
'scale=w=640:h=360:force_original_aspect_ratio=decrease',
'-c:v',
'mjpeg',
'-q:v',
'3',
'-f',
'image2pipe',
'pipe:1',
]);
if (!image.length) throw new Error('No video frame is available here.');
return { dataUrl: `data:image/jpeg;base64,${image.toString('base64')}`, timestamp: seekTime };
}
return {
generate,
clear: () => {
generation += 1;
cached = null;
source = null;
},
};
}
+11
View File
@@ -475,6 +475,7 @@ import { MediaTimingPreviewSession } from './core/services/media-timing-preview'
import { getSharedRemoteMediaWindowCache } from './core/services/remote-media-window-cache';
import { resolveMediaGenerationInput } from './anki-integration/media-source';
import { generateSpeechWaveform } from './core/services/media-timing-waveform';
import { createMediaTimingFrameExtractor } from './core/services/media-timing-frame';
import {
collectMediaTimingContextLines,
createMediaTimingReviewRuntime,
@@ -2905,6 +2906,7 @@ function createOverlayHostedModalOpenDeps(): {
};
}
const mediaTimingFrameExtractor = createMediaTimingFrameExtractor();
const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({
getMpvClient: () => appState.mpvClient,
getCurrentMediaPath: () =>
@@ -2913,6 +2915,14 @@ const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({
configService.getConfig().mpv.executablePath || process.env.SUBMINER_MPV_PATH?.trim() || '',
createPreviewSession: () => new MediaTimingPreviewSession(),
generateWaveform: (options) => generateSpeechWaveform(options),
generateFrame: (options) => mediaTimingFrameExtractor.generate(options),
clearFrameCache: () => mediaTimingFrameExtractor.clear(),
resolveVideoSource: () =>
resolveMediaGenerationInput(appState.mpvClient, 'video', {
getCachedMediaPath: (currentVideoPath, kind) =>
getCachedYoutubeMediaPathForCurrentPlayback(currentVideoPath, kind),
remoteCacheMode: shouldRequireYoutubeMediaCacheForCurrentPlayback() ? 'required' : 'optional',
}),
resolveMediaSource: async () => {
const resolved = await resolveMediaGenerationInput(appState.mpvClient, 'audio', {
getCachedMediaPath: (currentVideoPath, kind) =>
@@ -5586,6 +5596,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
},
mainDeps: {
previewMediaTimingReview: (request) => mediaTimingReviewRuntime.previewRange(request),
getMediaTimingReviewFrame: (request) => mediaTimingReviewRuntime.getFrame(request),
getMediaTimingReviewWaveform: (request) => mediaTimingReviewRuntime.getWaveform(request),
stopMediaTimingReviewPreview: (reviewId) => mediaTimingReviewRuntime.stopPreview(reviewId),
resolveMediaTimingReview: (request) => mediaTimingReviewRuntime.resolveReview(request),
+2
View File
@@ -63,6 +63,7 @@ export interface MainIpcRuntimeServiceDepsParams {
handleOverlayNotificationAction?: IpcDepsRuntimeOptions['handleOverlayNotificationAction'];
onYoutubePickerResolve: IpcDepsRuntimeOptions['onYoutubePickerResolve'];
previewMediaTimingReview?: IpcDepsRuntimeOptions['previewMediaTimingReview'];
getMediaTimingReviewFrame?: IpcDepsRuntimeOptions['getMediaTimingReviewFrame'];
getMediaTimingReviewWaveform?: IpcDepsRuntimeOptions['getMediaTimingReviewWaveform'];
stopMediaTimingReviewPreview?: IpcDepsRuntimeOptions['stopMediaTimingReviewPreview'];
resolveMediaTimingReview?: IpcDepsRuntimeOptions['resolveMediaTimingReview'];
@@ -263,6 +264,7 @@ export function createMainIpcRuntimeServiceDeps(
handleOverlayNotificationAction: params.handleOverlayNotificationAction,
onYoutubePickerResolve: params.onYoutubePickerResolve,
previewMediaTimingReview: params.previewMediaTimingReview,
getMediaTimingReviewFrame: params.getMediaTimingReviewFrame,
getMediaTimingReviewWaveform: params.getMediaTimingReviewWaveform,
stopMediaTimingReviewPreview: params.stopMediaTimingReviewPreview,
resolveMediaTimingReview: params.resolveMediaTimingReview,
@@ -0,0 +1,161 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import type { MediaTimingReviewOpenPayload } from '../../types/anki';
import type { MediaTimingFrameOptions } from '../../core/services/media-timing-frame';
import {
createMediaTimingReviewRuntime,
type MediaTimingReviewRuntimeDeps,
} from './media-timing-review';
async function start(
options: Partial<MediaTimingReviewRuntimeDeps> = {},
screenshotEnabled = true,
) {
let opened!: (payload: MediaTimingReviewOpenPayload) => void;
const payloadPromise = new Promise<MediaTimingReviewOpenPayload>((resolve) => {
opened = resolve;
});
const frames: MediaTimingFrameOptions[] = [];
const runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
connected: true,
currentVideoPath: '/video.mkv',
send: () => {},
requestProperty: async (name) => (name === 'duration' ? 100 : true),
}),
getCurrentMediaPath: () => '/video.mkv',
getMpvExecutablePath: () => '',
createPreviewSession: () => ({
start: async () => {},
play: async () => {},
stop: async () => {},
onPlaybackEnded: () => {},
dispose: () => {},
}),
generateWaveform: async () => [0, 1],
resolveVideoSource: async () => ({ path: '/video.mkv' }),
generateFrame: async (options) => {
frames.push(options);
return { dataUrl: 'data:image/jpeg;base64,AA==', timestamp: options.timestamp };
},
openModal: async (payload) => {
opened(payload);
return true;
},
showStatus: () => {},
...options,
});
const decision = runtime.requestReview({
kind: 'word',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0,
maxMediaDuration: 30,
screenshotEnabled,
});
return { runtime, decision, payload: await payloadPromise, frames };
}
test('review accepts a screenshot outside the audio range and validates media bounds', async () => {
const { runtime, decision, payload, frames } = await start();
for (const timestamp of [NaN, Infinity, -1, 100]) {
assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp })).ok, false);
assert.equal(
runtime.resolveReview({
reviewId: payload.reviewId,
decision: { action: 'confirm', startTime: 10, endTime: 11, screenshotTime: timestamp },
}).ok,
false,
);
}
assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 13 })).ok, true);
assert.equal(frames[0]?.timestamp, 13);
assert.equal(
runtime.resolveReview({
reviewId: payload.reviewId,
decision: { action: 'confirm', startTime: 10, endTime: 11, screenshotTime: 13 },
}).ok,
true,
);
assert.deepEqual(await decision, {
action: 'confirm',
startTime: 10,
endTime: 11,
screenshotTime: 13,
});
});
test('screenshot preview reuses the remote audio window and its absolute timestamps', async () => {
let downloads = 0;
const remote = 'https://example.test/video';
const { runtime, payload, decision, frames } = await start({
resolveMediaSource: async () => ({ path: remote }),
resolveVideoSource: async () => ({ path: remote }),
acquireMediaWindow: async () => {
downloads += 1;
return {
path: '/window.mkv',
sourcePath: remote,
startTime: 7,
endTime: 15,
audioStreamIndex: null,
media: { path: '/window.mkv', absoluteTimestamps: true },
};
},
});
await runtime.getWaveform({ reviewId: payload.reviewId, startTime: 8, endTime: 14 });
assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 })).ok, true);
assert.equal(downloads, 1);
assert.deepEqual(frames[0]?.media, { path: '/window.mkv', absoluteTimestamps: true });
await runtime.dispose();
await decision;
});
test('split video streams never read an audio-only cache as a video source', async () => {
const video = {
path: 'https://example.test/video',
inputOptions: { headers: { 'X-Test': 'value' } },
};
const { runtime, payload, decision, frames } = await start({
resolveMediaSource: async () => ({ path: 'https://example.test/audio' }),
resolveVideoSource: async () => video,
});
await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 });
assert.deepEqual(frames[0]?.media, video);
await runtime.dispose();
await decision;
});
test('disabled screenshots and missing video inputs do not trigger extraction', async () => {
for (const [enabled, options] of [
[false, {}],
[true, { resolveVideoSource: async () => null }],
] as const) {
const { runtime, payload, decision, frames } = await start(options, enabled);
assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 })).ok, false);
assert.equal(frames.length, 0);
await runtime.dispose();
await decision;
}
});
test('closing a review invalidates in-flight frame results and clears the frame index', async () => {
let finish!: (value: { dataUrl: string; timestamp: number }) => void;
let clearCount = 0;
const { runtime, payload, decision } = await start({
generateFrame: () =>
new Promise((resolve) => {
finish = resolve;
}),
clearFrameCache: () => {
clearCount += 1;
},
});
const frame = runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 });
await runtime.dispose();
finish({ dataUrl: 'data:image/jpeg;base64,AA==', timestamp: 11 });
assert.equal((await frame).stale, true);
await decision;
assert.equal(clearCount, 1);
});
+89 -8
View File
@@ -7,10 +7,13 @@ import type {
MediaTimingReviewPreviewRequest,
MediaTimingReviewRequest,
MediaTimingReviewResolveRequest,
MediaTimingReviewFrameRequest,
MediaTimingReviewFrameResult,
MediaTimingReviewWaveformRequest,
MediaTimingReviewWaveformResult,
} from '../../types/anki';
import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform';
import type { MediaTimingFrameOptions } from '../../core/services/media-timing-frame';
import {
isRemoteMediaWindowSourcePath,
type RemoteMediaWindow,
@@ -59,6 +62,8 @@ interface ActiveReview {
mediaPath: string;
/** What the waveform reads when no cached window is available. */
waveformMedia: MediaInput;
videoSource: ReviewMediaSource | null;
frameInFlight: boolean;
audioStreamIndex?: number;
/** Remote source to download windows of; null for local media or without a cache. */
windowSource: RemoteMediaWindowSource | null;
@@ -81,6 +86,11 @@ export interface MediaTimingReviewRuntimeDeps {
generateWaveform: (options: SpeechWaveformOptions) => Promise<number[]>;
/** Resolves the FFmpeg-readable stream URL and headers behind the current media path. */
resolveMediaSource?: () => Promise<ReviewMediaSource | null>;
resolveVideoSource?: () => Promise<ReviewMediaSource | null>;
generateFrame?: (
options: MediaTimingFrameOptions,
) => Promise<{ dataUrl: string; timestamp: number }>;
clearFrameCache?: () => void;
/** Downloads (or reuses) a local window of a remote source covering the range. */
acquireMediaWindow?: (
source: RemoteMediaWindowSource,
@@ -227,6 +237,9 @@ export function buildMediaTimingReviewPayload(
timelineEndTime,
...(duration !== null && duration > 0 ? { mediaDuration: duration } : {}),
maxMediaDuration,
...(request.screenshotEnabled !== undefined
? { screenshotEnabled: request.screenshotEnabled }
: {}),
};
}
@@ -335,13 +348,15 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
return { action: 'use-original' };
}
const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw, resolvedSource] = await Promise.all([
mpvClient.requestProperty?.('pause').catch(() => null) ?? null,
mpvClient.requestProperty?.('duration').catch(() => null) ?? null,
mpvClient.requestProperty?.('aid').catch(() => null) ?? null,
mpvClient.requestProperty?.('volume').catch(() => null) ?? null,
deps.resolveMediaSource?.().catch(() => null) ?? null,
]);
const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw, resolvedSource, videoSource] =
await Promise.all([
mpvClient.requestProperty?.('pause').catch(() => null) ?? null,
mpvClient.requestProperty?.('duration').catch(() => null) ?? null,
mpvClient.requestProperty?.('aid').catch(() => null) ?? null,
mpvClient.requestProperty?.('volume').catch(() => null) ?? null,
deps.resolveMediaSource?.().catch(() => null) ?? null,
request.screenshotEnabled ? (deps.resolveVideoSource?.().catch(() => null) ?? null) : null,
]);
const pauseState = booleanProperty(pauseRaw);
mpvClient.send({ command: ['set_property', 'pause', 'yes'] });
pendingPauseRestore = pauseState === false ? mpvClient : null;
@@ -383,6 +398,8 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
payload,
mediaPath,
waveformMedia: inputOptions ? { path: sourcePath, inputOptions } : sourcePath,
videoSource,
frameInFlight: false,
...(audioStreamIndex !== undefined ? { audioStreamIndex } : {}),
windowSource,
window: null,
@@ -527,6 +544,58 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
}
}
async function getFrame(
request: MediaTimingReviewFrameRequest,
): Promise<MediaTimingReviewFrameResult> {
const current = active;
if (!current || request.reviewId !== current.payload.reviewId) return staleReviewResult();
if (!current.payload.screenshotEnabled || !deps.generateFrame || !current.videoSource) {
return { ok: false, message: 'Screenshot preview is unavailable for this media.' };
}
if (
!Number.isFinite(request.timestamp) ||
request.timestamp < 0 ||
(current.payload.mediaDuration !== undefined &&
request.timestamp >= current.payload.mediaDuration) ||
(request.direction !== undefined && request.direction !== -1 && request.direction !== 1)
) {
return { ok: false, message: 'The screenshot time is invalid.' };
}
if (current.frameInFlight)
return { ok: false, message: 'A screenshot preview is already loading.' };
current.frameInFlight = true;
try {
// Reuse the audio window only when it contains this same video source (not split streams).
const range = {
startTime: Math.max(0, Math.min(current.payload.timelineStartTime, request.timestamp - 2)),
endTime: Math.min(
current.payload.mediaDuration ?? Infinity,
Math.max(current.payload.timelineEndTime, request.timestamp + 2),
),
};
const window =
current.windowSource?.path === current.videoSource.path
? await ensureWindow(current, range)
: null;
if (active !== current) return staleReviewResult();
const frame = await deps.generateFrame({
media: window?.media ?? current.videoSource,
timestamp: request.timestamp,
...(request.direction !== undefined ? { direction: request.direction } : {}),
});
if (active !== current) return staleReviewResult();
return { ok: true, ...frame };
} catch {
if (active !== current) return staleReviewResult();
return {
ok: false,
message: 'Screenshot preview unavailable. Try another time or reset to the midpoint.',
};
} finally {
current.frameInFlight = false;
}
}
async function stopPreview(reviewId: string): Promise<MediaTimingReviewActionResult> {
const current = active;
if (!current || reviewId !== current.payload.reviewId) {
@@ -550,13 +619,23 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
return staleReviewResult();
}
if (request.decision.action === 'confirm') {
const { startTime, endTime, text } = request.decision;
const { startTime, endTime, text, screenshotTime } = request.decision;
if (!isValidMediaTimingRange(current.payload, startTime, endTime)) {
return { ok: false, message: 'The selected timing range is invalid.' };
}
if (text !== undefined && (typeof text !== 'string' || text.trim().length === 0)) {
return { ok: false, message: 'The combined sentence text is invalid.' };
}
if (
screenshotTime !== undefined &&
(!current.payload.screenshotEnabled ||
!Number.isFinite(screenshotTime) ||
screenshotTime < 0 ||
(current.payload.mediaDuration !== undefined &&
screenshotTime >= current.payload.mediaDuration))
) {
return { ok: false, message: 'The screenshot time is invalid.' };
}
}
current.resolve(request.decision);
return { ok: true };
@@ -566,6 +645,7 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
const current = active;
active = null;
if (!current) return;
deps.clearFrameCache?.();
void current.preview?.session.then((session) => session.dispose()).catch(() => {});
if (current.restorePlayback && current.mpvClient.connected) {
current.mpvClient.send({ command: ['set_property', 'pause', 'no'] });
@@ -582,6 +662,7 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
requestReview,
previewRange,
getWaveform,
getFrame,
stopPreview,
resolveReview,
dispose,
+6
View File
@@ -73,6 +73,8 @@ import type {
MediaTimingReviewOpenPayload,
MediaTimingReviewPreviewRequest,
MediaTimingReviewResolveRequest,
MediaTimingReviewFrameRequest,
MediaTimingReviewFrameResult,
MediaTimingReviewWaveformRequest,
} from './types';
import { IPC_CHANNELS } from './shared/ipc/contracts';
@@ -507,6 +509,10 @@ const electronAPI: ElectronAPI = {
request: MediaTimingReviewPreviewRequest,
): Promise<MediaTimingReviewActionResult> =>
ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewPreview, request),
getMediaTimingReviewFrame: (
request: MediaTimingReviewFrameRequest,
): Promise<MediaTimingReviewFrameResult> =>
ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewFrame, request),
getMediaTimingReviewWaveform: (request: MediaTimingReviewWaveformRequest) =>
ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewWaveform, request),
stopMediaTimingReviewPreview: (reviewId: string): Promise<MediaTimingReviewActionResult> =>
+51
View File
@@ -297,6 +297,57 @@
</div>
<blockquote id="mediaTimingReviewText" class="media-timing-review-text"></blockquote>
<div id="mediaTimingReviewFramePicker" class="media-timing-frame-picker hidden">
<div class="media-timing-frame-image-shell">
<img
id="mediaTimingReviewFrameImage"
class="hidden"
alt="Selected card screenshot"
/>
</div>
<div class="media-timing-frame-controls">
<div class="media-timing-frame-heading">
<label for="mediaTimingReviewFrameSlider">Card screenshot</label>
<output id="mediaTimingReviewFrameTime" for="mediaTimingReviewFrameSlider"
>—</output
>
</div>
<input
id="mediaTimingReviewFrameSlider"
type="range"
min="0"
max="1"
step="0.001"
value="0"
aria-label="Screenshot time"
/>
<div class="media-timing-frame-buttons">
<button
id="mediaTimingReviewFramePrevious"
type="button"
aria-label="Previous video frame"
>
← Frame
</button>
<button
id="mediaTimingReviewFrameNext"
type="button"
aria-label="Next video frame"
>
Frame →
</button>
<button
id="mediaTimingReviewFrameReset"
type="button"
aria-label="Reset screenshot"
>
Reset
</button>
</div>
<p id="mediaTimingReviewFrameStatus" role="status"></p>
</div>
</div>
<div class="media-timing-review-readout" aria-live="polite">
<div>
<span>Starts</span>
@@ -0,0 +1,116 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { setTimeout as tick } from 'node:timers/promises';
import { createMediaTimingFramePicker } from './media-timing-frame-picker';
import type { MediaTimingReviewFrameRequest, MediaTimingReviewFrameResult } from '../../types/anki';
function fixture() {
const requests: Array<{
request: MediaTimingReviewFrameRequest;
resolve: (result: MediaTimingReviewFrameResult) => void;
}> = [];
let stale = false;
const picker = createMediaTimingFramePicker({
debounceMs: 0,
load: (request) => new Promise((resolve) => requests.push({ request, resolve })),
onChange: () => {},
onStale: () => {
stale = true;
picker.close();
},
});
const complete = async (index: number, result: number | MediaTimingReviewFrameResult) => {
requests[index]!.resolve(
typeof result === 'number'
? { ok: true, timestamp: result, dataUrl: `data:image/jpeg;base64,${result}` }
: result,
);
await tick(0);
};
return { picker, requests, complete, isStale: () => stale };
}
test('a chosen frame stays fixed until Reset restores midpoint tracking', async () => {
const { picker, requests, complete } = fixture();
picker.open('r', true, 11);
await tick(5);
await complete(0, 11);
assert.equal(picker.getScreenshotTime(), undefined);
picker.choose(13);
await tick(5);
await complete(1, 13);
picker.updateMidpoint(12);
await tick(5);
assert.equal(picker.getScreenshotTime(), 13);
assert.equal(requests.length, 2);
picker.reset();
await tick(5);
assert.equal(requests[2]?.request.timestamp, 12);
await complete(2, 12);
assert.equal(picker.getScreenshotTime(), undefined);
picker.updateMidpoint(14);
await tick(5);
assert.equal(requests[3]?.request.timestamp, 14);
await complete(3, 14);
picker.close();
});
test('scrubbing keeps only the latest requested frame', async () => {
const { picker, requests, complete } = fixture();
picker.open('r', true, 11);
await tick(5);
picker.choose(12);
picker.choose(13);
picker.choose(14);
await tick(5);
assert.equal(requests.length, 1);
assert.equal(picker.getState().blockConfirm, true);
await complete(0, 11);
await tick(5);
assert.equal(picker.getState().timestamp, undefined);
assert.equal(requests[1]?.request.timestamp, 14);
await complete(1, 14);
assert.equal(picker.getScreenshotTime(), 14);
assert.equal(picker.getState().blockConfirm, false);
picker.close();
});
test('failed manual previews block confirmation; Reset allows the default fallback', async () => {
const { picker, complete } = fixture();
picker.open('r', true, 11);
await tick(5);
await complete(0, { ok: false });
assert.equal(picker.getState().blockConfirm, false);
picker.choose(12);
await tick(5);
await complete(1, { ok: false });
assert.equal(picker.getState().blockConfirm, true);
picker.reset();
assert.equal(picker.getState().blockConfirm, false);
picker.close();
});
test('replacing a review ignores old frames; stale responses close the current review', async () => {
const { picker, requests, complete, isStale } = fixture();
picker.open('old', true, 11);
await tick(5);
picker.close();
picker.open('new', true, 22);
await tick(5);
await complete(0, 11);
await tick(5);
assert.equal(picker.getState().timestamp, undefined);
assert.equal(requests[1]?.request.reviewId, 'new');
await complete(1, { ok: false, stale: true });
assert.equal(isStale(), true);
assert.equal(picker.getState().enabled, false);
});
test('disabled screenshots perform no extraction', async () => {
const { picker, requests } = fixture();
picker.open('r', false, 11);
picker.updateMidpoint(12);
await tick(5);
assert.equal(requests.length, 0);
picker.close();
});
@@ -0,0 +1,129 @@
import type { MediaTimingReviewFrameRequest, MediaTimingReviewFrameResult } from '../../types/anki';
export interface MediaTimingFramePickerState {
enabled: boolean;
manual: boolean;
loading: boolean;
timestamp?: number;
requestedTime?: number;
dataUrl?: string;
message: string;
blockConfirm: boolean;
}
/** Coalesces scrubbing into one active extraction and the latest requested frame. */
export function createMediaTimingFramePicker(options: {
load: (request: MediaTimingReviewFrameRequest) => Promise<MediaTimingReviewFrameResult>;
onChange: (state: MediaTimingFramePickerState) => void;
onStale: () => void;
debounceMs?: number;
}) {
let reviewId: string | null = null;
let midpoint = 0;
let sequence = 0;
let inFlight = false;
let pending: (MediaTimingReviewFrameRequest & { sequence: number }) | null = null;
let timer: ReturnType<typeof setTimeout> | null = null;
let state: MediaTimingFramePickerState = emptyState();
function emptyState(): MediaTimingFramePickerState {
return { enabled: false, manual: false, loading: false, message: '', blockConfirm: false };
}
function publish(): void {
options.onChange({ ...state });
}
async function drain(): Promise<void> {
if (inFlight || !pending) return;
const request = pending;
pending = null;
inFlight = true;
try {
const result = await options.load(request);
if (reviewId !== request.reviewId || sequence !== request.sequence) return;
if (result.stale) {
options.onStale();
return;
}
if (
!result.ok ||
!Number.isFinite(result.timestamp) ||
!result.dataUrl?.startsWith('data:image/jpeg;base64,')
) {
throw new Error(result.message ?? 'Screenshot preview is unavailable.');
}
state.timestamp = result.timestamp;
state.dataUrl = result.dataUrl;
state.blockConfirm = false;
state.message = state.manual
? 'Selected frame stays fixed when you trim audio.'
: 'Following the audio midpoint.';
} catch (error) {
if (reviewId !== request.reviewId || sequence !== request.sequence) return;
state.message = error instanceof Error ? error.message : 'Screenshot preview is unavailable.';
state.blockConfirm = state.manual;
} finally {
inFlight = false;
if (reviewId === request.reviewId && sequence === request.sequence) {
state.loading = false;
publish();
}
if (timer === null) void drain();
}
}
function request(timestamp: number, direction?: -1 | 1): void {
if (!reviewId || !state.enabled) return;
sequence += 1;
pending = { reviewId, timestamp, ...(direction ? { direction } : {}), sequence };
state.requestedTime = timestamp;
state.loading = true;
state.blockConfirm = state.manual;
state.message = 'Loading screenshot…';
publish();
if (timer !== null) clearTimeout(timer);
timer = setTimeout(() => {
timer = null;
void drain();
}, options.debounceMs ?? 120);
}
function close(): void {
sequence += 1;
reviewId = null;
pending = null;
if (timer !== null) clearTimeout(timer);
timer = null;
state = emptyState();
publish();
}
return {
open(id: string, enabled: boolean, time: number) {
close();
reviewId = id;
midpoint = time;
state.enabled = enabled;
publish();
if (enabled) request(time);
},
updateMidpoint(time: number) {
if (midpoint === time) return;
midpoint = time;
if (!state.manual) request(time);
},
choose(time: number, direction?: -1 | 1) {
if (!Number.isFinite(time) || time < 0) return;
state.manual = true;
request(time, direction);
},
reset() {
state.manual = false;
request(midpoint);
},
close,
getState: () => ({ ...state }),
getScreenshotTime: () => (state.manual && !state.blockConfirm ? state.timestamp : undefined),
};
}
+74 -1
View File
@@ -1,3 +1,4 @@
import { createMediaTimingFramePicker } from './media-timing-frame-picker';
import type {
MediaTimingReviewContextLine,
MediaTimingReviewDecision,
@@ -252,6 +253,45 @@ export function createMediaTimingReviewModal(
isModalLayer: ctx.platform.isModalLayer,
});
const previewRequest = createMediaTimingPreviewRequestGuard();
const framePicker = createMediaTimingFramePicker({
load: (request) => window.electronAPI.getMediaTimingReviewFrame(request),
onChange: () => renderFramePicker(),
onStale: () => closeResolvedReview(),
});
function renderFramePicker(): void {
const state = framePicker.getState();
const dom = ctx.dom;
dom.mediaTimingReviewFramePicker.classList.toggle('hidden', !state.enabled);
dom.mediaTimingReviewFramePicker.setAttribute('aria-busy', String(state.loading));
dom.mediaTimingReviewFrameSlider.min = String(timelineStart);
dom.mediaTimingReviewFrameSlider.max = String(Math.max(timelineStart, timelineEnd - 0.001));
const shownTime = state.loading ? state.requestedTime : state.timestamp;
if (shownTime !== undefined) dom.mediaTimingReviewFrameSlider.value = String(shownTime);
dom.mediaTimingReviewFrameTime.textContent =
shownTime === undefined ? '—' : formatMediaTimingTimestamp(shownTime);
dom.mediaTimingReviewFrameSlider.setAttribute(
'aria-valuetext',
dom.mediaTimingReviewFrameTime.textContent,
);
dom.mediaTimingReviewFrameStatus.textContent = state.message;
dom.mediaTimingReviewFrameImage.classList.toggle('hidden', !state.dataUrl);
if (state.dataUrl) dom.mediaTimingReviewFrameImage.src = state.dataUrl;
else dom.mediaTimingReviewFrameImage.removeAttribute('src');
dom.mediaTimingReviewFramePrevious.disabled =
resolveInFlight ||
state.loading ||
state.timestamp === undefined ||
state.timestamp <= timelineStart;
dom.mediaTimingReviewFrameNext.disabled =
resolveInFlight ||
state.loading ||
state.timestamp === undefined ||
state.timestamp >= timelineEnd - 0.001;
dom.mediaTimingReviewFrameSlider.disabled = resolveInFlight;
dom.mediaTimingReviewFrameReset.disabled = resolveInFlight || !state.manual;
dom.mediaTimingReviewConfirm.disabled = resolveInFlight || state.blockConfirm;
}
function setStatus(message: string, isError = false): void {
ctx.dom.mediaTimingReviewStatus.textContent = message;
@@ -308,6 +348,8 @@ export function createMediaTimingReviewModal(
}
function renderSelection(): void {
framePicker.updateMidpoint(selectionStart + (selectionEnd - selectionStart) / 2);
renderFramePicker();
const span = Math.max(MINIMUM_CLIP_SECONDS, timelineEnd - timelineStart);
const startPercent = ((selectionStart - timelineStart) / span) * 100;
const endPercent = ((selectionEnd - timelineStart) / span) * 100;
@@ -690,6 +732,7 @@ export function createMediaTimingReviewModal(
window.electronAPI.notifyOverlayModalClosed('media-timing-review');
options.syncSettingsModalSubtitleSuppression();
payload = null;
framePicker.close();
if (!options.modalStateReader.isAnyModalOpen()) {
ctx.dom.overlay.classList.remove('interactive');
if (ctx.platform.shouldToggleMouseIgnore) {
@@ -701,6 +744,7 @@ export function createMediaTimingReviewModal(
async function resolveReview(decision: MediaTimingReviewDecision): Promise<void> {
if (!payload || resolveInFlight) return;
resolveInFlight = true;
renderFramePicker();
stopPreview();
const controls = ctx.dom.mediaTimingReviewModal.querySelectorAll<HTMLButtonElement>('button');
controls.forEach((button) => {
@@ -733,12 +777,14 @@ export function createMediaTimingReviewModal(
}
function confirmSelection(): void {
if (!payload) return;
if (!payload || framePicker.getState().blockConfirm) return;
const screenshotTime = framePicker.getScreenshotTime();
const includesAdjacentLines = previousCount > 0 || nextCount > 0;
void resolveReview({
action: 'confirm',
startTime: selectionStart,
endTime: selectionEnd,
...(screenshotTime !== undefined ? { screenshotTime } : {}),
...(includesAdjacentLines ? { text: currentLineSelection().sentence } : {}),
});
}
@@ -818,6 +864,11 @@ export function createMediaTimingReviewModal(
nextPayload.noteId !== undefined ? 'Delete card' : "Don't create card";
setStatus('');
showEditor();
framePicker.open(
nextPayload.reviewId,
nextPayload.screenshotEnabled === true,
selectionStart + (selectionEnd - selectionStart) / 2,
);
renderSelection();
renderSentence();
ctx.state.mediaTimingReviewModalOpen = true;
@@ -892,6 +943,28 @@ export function createMediaTimingReviewModal(
}
function wireDomEvents(): void {
ctx.dom.mediaTimingReviewFrameSlider.addEventListener('input', () => {
if (!resolveInFlight) framePicker.choose(Number(ctx.dom.mediaTimingReviewFrameSlider.value));
});
const stepFrame = (direction: -1 | 1) => {
const state = framePicker.getState();
if (!resolveInFlight && !state.loading && state.timestamp !== undefined)
framePicker.choose(state.timestamp, direction);
};
ctx.dom.mediaTimingReviewFrameSlider.addEventListener('keydown', (event) => {
if (event.key === 'ArrowLeft' || event.key === 'ArrowDown') {
event.preventDefault();
stepFrame(-1);
} else if (event.key === 'ArrowRight' || event.key === 'ArrowUp') {
event.preventDefault();
stepFrame(1);
}
});
ctx.dom.mediaTimingReviewFramePrevious.addEventListener('click', () => stepFrame(-1));
ctx.dom.mediaTimingReviewFrameNext.addEventListener('click', () => stepFrame(1));
ctx.dom.mediaTimingReviewFrameReset.addEventListener('click', () => {
if (!resolveInFlight) framePicker.reset();
});
const track = ctx.dom.mediaTimingReviewSelectionTrack;
track.addEventListener('pointerdown', beginDrag);
track.addEventListener('pointermove', (event) => {
+81 -1
View File
@@ -1347,7 +1347,7 @@ body:focus-visible,
.media-timing-review-content {
width: min(720px, calc(100vw - 32px));
max-height: min(700px, calc(100vh - 32px));
max-height: min(900px, calc(100vh - 32px));
overflow: auto;
gap: 0;
padding: 0;
@@ -1425,6 +1425,86 @@ body:focus-visible,
padding: 14px 20px 18px;
}
.media-timing-frame-picker {
display: grid;
grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
gap: 16px;
align-items: center;
margin: 0 0 16px;
padding: 12px;
border: 1px solid var(--ctp-surface1);
border-radius: 10px;
background: var(--ctp-mantle);
}
.media-timing-frame-picker.hidden {
display: none;
}
.media-timing-frame-image-shell {
aspect-ratio: 16 / 9;
display: flex;
align-items: center;
justify-content: center;
overflow: hidden;
border-radius: 6px;
background: var(--ctp-crust);
}
.media-timing-frame-image-shell img {
width: 100%;
height: 100%;
object-fit: contain;
}
.media-timing-frame-picker[aria-busy='true'] img {
opacity: 0.55;
}
.media-timing-frame-heading {
display: flex;
justify-content: space-between;
gap: 8px;
font-size: 12px;
}
.media-timing-frame-heading output {
color: var(--ctp-teal);
font-variant-numeric: tabular-nums;
}
.media-timing-frame-controls input {
width: 100%;
margin: 12px 0;
accent-color: var(--ctp-teal);
}
.media-timing-frame-buttons {
display: flex;
flex-wrap: wrap;
gap: 6px;
}
.media-timing-frame-buttons button {
padding: 5px 8px;
border: 1px solid var(--ctp-surface2);
border-radius: 6px;
background: var(--ctp-surface0);
color: var(--ctp-text);
font-size: 11px;
cursor: pointer;
}
.media-timing-frame-buttons button:disabled {
opacity: 0.45;
cursor: default;
}
.media-timing-frame-controls p {
margin: 8px 0 0;
min-height: 2.6em;
color: var(--ctp-subtext0);
font-size: 11px;
}
@media (max-width: 520px) {
.media-timing-frame-picker {
grid-template-columns: 1fr;
}
.media-timing-frame-image-shell {
max-height: 180px;
}
}
.media-timing-review-sentence-header {
display: flex;
align-items: center;
+26
View File
@@ -46,6 +46,14 @@ export type RendererDom = {
youtubePickerStatus: HTMLDivElement;
youtubePickerTracks: HTMLUListElement;
mediaTimingReviewFramePicker: HTMLDivElement;
mediaTimingReviewFrameImage: HTMLImageElement;
mediaTimingReviewFrameSlider: HTMLInputElement;
mediaTimingReviewFrameTime: HTMLElement;
mediaTimingReviewFrameStatus: HTMLElement;
mediaTimingReviewFramePrevious: HTMLButtonElement;
mediaTimingReviewFrameNext: HTMLButtonElement;
mediaTimingReviewFrameReset: HTMLButtonElement;
mediaTimingReviewModal: HTMLDivElement;
mediaTimingReviewKind: HTMLDivElement;
mediaTimingReviewText: HTMLElement;
@@ -249,6 +257,24 @@ export function resolveRendererDom(): RendererDom {
youtubePickerStatus: getRequiredElement<HTMLDivElement>('youtubePickerStatus'),
youtubePickerTracks: getRequiredElement<HTMLUListElement>('youtubePickerTracks'),
mediaTimingReviewFramePicker: getRequiredElement<HTMLDivElement>(
'mediaTimingReviewFramePicker',
),
mediaTimingReviewFrameImage: getRequiredElement<HTMLImageElement>(
'mediaTimingReviewFrameImage',
),
mediaTimingReviewFrameSlider: getRequiredElement<HTMLInputElement>(
'mediaTimingReviewFrameSlider',
),
mediaTimingReviewFrameTime: getRequiredElement<HTMLElement>('mediaTimingReviewFrameTime'),
mediaTimingReviewFrameStatus: getRequiredElement<HTMLElement>('mediaTimingReviewFrameStatus'),
mediaTimingReviewFramePrevious: getRequiredElement<HTMLButtonElement>(
'mediaTimingReviewFramePrevious',
),
mediaTimingReviewFrameNext: getRequiredElement<HTMLButtonElement>('mediaTimingReviewFrameNext'),
mediaTimingReviewFrameReset: getRequiredElement<HTMLButtonElement>(
'mediaTimingReviewFrameReset',
),
mediaTimingReviewModal: getRequiredElement<HTMLDivElement>('mediaTimingReviewModal'),
mediaTimingReviewKind: getRequiredElement<HTMLDivElement>('mediaTimingReviewKind'),
mediaTimingReviewText: getRequiredElement<HTMLElement>('mediaTimingReviewText'),
+1
View File
@@ -138,6 +138,7 @@ export const IPC_CHANNELS = {
getChangelogSnapshot: 'changelog:get-snapshot',
mediaTimingReviewPreview: 'media-timing-review:preview',
mediaTimingReviewWaveform: 'media-timing-review:waveform',
mediaTimingReviewFrame: 'media-timing-review:frame',
mediaTimingReviewStopPreview: 'media-timing-review:stop-preview',
mediaTimingReviewResolve: 'media-timing-review:resolve',
},
+22 -1
View File
@@ -21,6 +21,8 @@ export interface MediaTimingReviewRequest {
noteId?: number;
audioPadding: number;
maxMediaDuration: number;
/** Still screenshots only; animated images continue to follow the audio range. */
screenshotEnabled?: boolean;
}
/** A subtitle line adjacent to the mined one that the review can pull onto the card. */
@@ -32,7 +34,13 @@ export interface MediaTimingReviewContextLine {
export type MediaTimingReviewDecision =
/** `text` is set when the review combined adjacent lines into the card sentence. */
| { action: 'confirm'; startTime: number; endTime: number; text?: string }
| {
action: 'confirm';
startTime: number;
endTime: number;
text?: string;
screenshotTime?: number;
}
| { action: 'use-original' }
| { action: 'skip-media' }
| { action: 'discard' };
@@ -53,6 +61,19 @@ export interface MediaTimingReviewOpenPayload {
timelineEndTime: number;
mediaDuration?: number;
maxMediaDuration: number;
screenshotEnabled?: boolean;
}
export interface MediaTimingReviewFrameRequest {
reviewId: string;
timestamp: number;
/** Step to the adjacent decoded frame instead of seeking to a time. */
direction?: -1 | 1;
}
export interface MediaTimingReviewFrameResult extends MediaTimingReviewActionResult {
dataUrl?: string;
timestamp?: number;
}
export interface MediaTimingReviewPreviewRequest {
+5
View File
@@ -8,6 +8,8 @@ import type {
MediaTimingReviewOpenPayload,
MediaTimingReviewPreviewRequest,
MediaTimingReviewResolveRequest,
MediaTimingReviewFrameRequest,
MediaTimingReviewFrameResult,
MediaTimingReviewWaveformRequest,
MediaTimingReviewWaveformResult,
} from './anki';
@@ -550,6 +552,9 @@ export interface ElectronAPI {
previewMediaTimingReview: (
request: MediaTimingReviewPreviewRequest,
) => Promise<MediaTimingReviewActionResult>;
getMediaTimingReviewFrame: (
request: MediaTimingReviewFrameRequest,
) => Promise<MediaTimingReviewFrameResult>;
getMediaTimingReviewWaveform: (
request: MediaTimingReviewWaveformRequest,
) => Promise<MediaTimingReviewWaveformResult>;
+2
View File
@@ -246,6 +246,8 @@ export interface SubtitleMiningContext {
capturedAtMs?: number;
/** Explicit generator padding. Confirmed timing-review ranges set this to zero. */
mediaPaddingSeconds?: number;
/** Independent still screenshot selected during media timing review. */
screenshotTime?: number;
}
export interface SubtitleHoverTokenPayload {