mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-09-21 17:16:20 -07:00
feat(mining): add a screenshot frame picker to media review (#254)
This commit is contained in:
Abdulrazzaq Alhendi
@@ -1118,9 +1118,11 @@ export class AnkiIntegration {
|
||||
return null;
|
||||
}
|
||||
const mediaRange = this.getSubtitleMediaRange(context);
|
||||
const timestamp = context
|
||||
? mediaRange.startTime + (mediaRange.endTime - mediaRange.startTime) / 2
|
||||
: this.mpvClient.currentTimePos || 0;
|
||||
const timestamp =
|
||||
context?.screenshotTime ??
|
||||
(context
|
||||
? mediaRange.startTime + (mediaRange.endTime - mediaRange.startTime) / 2
|
||||
: this.mpvClient.currentTimePos || 0);
|
||||
|
||||
if (this.config.media?.imageType === 'avif') {
|
||||
return this.mediaGenerator.generateAnimatedImage(
|
||||
@@ -1797,6 +1799,8 @@ export class AnkiIntegration {
|
||||
...request,
|
||||
audioPadding: Math.max(0, this.config.media.audioPadding ?? 0),
|
||||
maxMediaDuration: Math.max(0, this.config.media.maxMediaDuration ?? 30),
|
||||
screenshotEnabled:
|
||||
this.config.media.generateImage !== false && this.config.media.imageType !== 'avif',
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -290,6 +290,9 @@ test('CardCreationService keeps updating after recordCardsMinedCallback throws',
|
||||
});
|
||||
|
||||
test('CardCreationService uses stream-open-filename for remote media generation', async () => {
|
||||
let reviewing = false;
|
||||
const audioRanges: number[][] = [];
|
||||
const imageTimes: number[] = [];
|
||||
const audioPaths: string[] = [];
|
||||
const imagePaths: string[] = [];
|
||||
const recordMediaPath = (mediaInput: MediaInput): string =>
|
||||
@@ -319,6 +322,10 @@ test('CardCreationService uses stream-open-filename for remote media generation'
|
||||
behavior: {},
|
||||
ai: false,
|
||||
}) as AnkiConnectConfig,
|
||||
reviewMediaTiming: async () =>
|
||||
reviewing
|
||||
? { action: 'confirm', startTime: 0.2, endTime: 0.8, screenshotTime: 3.125 }
|
||||
: { action: 'use-original' },
|
||||
getAiConfig: () => ({}),
|
||||
getTimingTracker: () => ({}) as never,
|
||||
getMpvClient: () =>
|
||||
@@ -349,16 +356,18 @@ test('CardCreationService uses stream-open-filename for remote media generation'
|
||||
],
|
||||
updateNoteFields: async () => undefined,
|
||||
storeMediaFile: async () => undefined,
|
||||
findNotes: async () => [],
|
||||
findNotes: async () => [42],
|
||||
retrieveMediaFile: async () => '',
|
||||
deleteNotes: async () => undefined,
|
||||
},
|
||||
mediaGenerator: {
|
||||
generateAudio: async (path) => {
|
||||
generateAudio: async (path, start, end, padding) => {
|
||||
audioRanges.push([start, end, padding ?? -1]);
|
||||
audioPaths.push(recordMediaPath(path));
|
||||
return Buffer.from('audio');
|
||||
},
|
||||
generateScreenshot: async (path) => {
|
||||
generateScreenshot: async (path, timestamp) => {
|
||||
imageTimes.push(timestamp);
|
||||
imagePaths.push(recordMediaPath(path));
|
||||
return Buffer.from('image');
|
||||
},
|
||||
@@ -406,6 +415,14 @@ test('CardCreationService uses stream-open-filename for remote media generation'
|
||||
assert.equal(created, true);
|
||||
assert.deepEqual(audioPaths, [audioUrl]);
|
||||
assert.deepEqual(imagePaths, [videoUrl]);
|
||||
reviewing = true;
|
||||
assert.equal(await service.createSentenceCard('テスト', 0, 1), true);
|
||||
assert.deepEqual(audioRanges.at(-1), [0.2, 0.8, 0]);
|
||||
assert.equal(imageTimes.at(-1), 3.125);
|
||||
await service.markLastCardAsAudioCard();
|
||||
assert.equal(imageTimes.length, 3);
|
||||
assert.deepEqual(audioRanges.at(-1), [0.2, 0.8, 0]);
|
||||
assert.equal(imageTimes.at(-1), 3.125);
|
||||
});
|
||||
|
||||
test('CardCreationService does not use mpv stream indexes for ready cached YouTube media', async () => {
|
||||
|
||||
@@ -536,6 +536,7 @@ export class CardCreationService {
|
||||
endTime,
|
||||
animatedLeadInSeconds,
|
||||
exactReviewedRange,
|
||||
timingDecision.action === 'confirm' ? timingDecision.screenshotTime : undefined,
|
||||
);
|
||||
|
||||
const imageField = this.deps.getConfig().fields?.image;
|
||||
@@ -796,6 +797,9 @@ export class CardCreationService {
|
||||
generateImage,
|
||||
volumeScale,
|
||||
...(exactReviewedRange ? { mediaPaddingSeconds: 0 } : {}),
|
||||
...(timingDecision.action === 'confirm' && timingDecision.screenshotTime !== undefined
|
||||
? { screenshotTime: timingDecision.screenshotTime }
|
||||
: {}),
|
||||
});
|
||||
await this.deps.showNotification(noteId, label, 'media queued');
|
||||
return true;
|
||||
@@ -840,6 +844,7 @@ export class CardCreationService {
|
||||
endTime,
|
||||
0,
|
||||
exactReviewedRange,
|
||||
timingDecision.action === 'confirm' ? timingDecision.screenshotTime : undefined,
|
||||
);
|
||||
|
||||
const imageField = config.fields?.image;
|
||||
@@ -922,15 +927,16 @@ export class CardCreationService {
|
||||
endTime: number,
|
||||
animatedLeadInSeconds = 0,
|
||||
exactReviewedRange = false,
|
||||
screenshotTime?: number,
|
||||
): Promise<Buffer | null> {
|
||||
const mpvClient = this.deps.getMpvClient();
|
||||
if (!mpvClient) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const timestamp = exactReviewedRange
|
||||
? startTime + (endTime - startTime) / 2
|
||||
: mpvClient.currentTimePos || 0;
|
||||
const timestamp =
|
||||
screenshotTime ??
|
||||
(exactReviewedRange ? startTime + (endTime - startTime) / 2 : mpvClient.currentTimePos || 0);
|
||||
|
||||
if (this.deps.getConfig().media?.imageType === 'avif') {
|
||||
let imageStart = startTime;
|
||||
|
||||
@@ -729,6 +729,7 @@ test('NoteUpdateWorkflow uses the combined review sentence for the card and medi
|
||||
startTime: 2,
|
||||
endTime: 7,
|
||||
text: 'previous-line current-line next-line',
|
||||
screenshotTime: 8,
|
||||
});
|
||||
harness.deps.generateAudio = async (context) => {
|
||||
audioContexts.push(context);
|
||||
@@ -745,6 +746,7 @@ test('NoteUpdateWorkflow uses the combined review sentence for the card and medi
|
||||
assert.equal(audioContexts[0]?.startTime, 2);
|
||||
assert.equal(audioContexts[0]?.endTime, 7);
|
||||
assert.equal(audioContexts[0]?.mediaPaddingSeconds, 0);
|
||||
assert.equal(audioContexts[0]?.screenshotTime, 8);
|
||||
});
|
||||
|
||||
test('NoteUpdateWorkflow keeps cache unchanged and reports when deletion fails', async () => {
|
||||
|
||||
@@ -253,6 +253,9 @@ export class NoteUpdateWorkflow {
|
||||
startTime: timingDecision.startTime,
|
||||
endTime: timingDecision.endTime,
|
||||
mediaPaddingSeconds: 0,
|
||||
...(timingDecision.screenshotTime !== undefined
|
||||
? { screenshotTime: timingDecision.screenshotTime }
|
||||
: {}),
|
||||
};
|
||||
} else if (timingDecision.action === 'skip-media') {
|
||||
skipMedia = true;
|
||||
|
||||
@@ -51,6 +51,41 @@ function createDeps(
|
||||
return deps;
|
||||
}
|
||||
|
||||
test('queued media keeps a chosen screenshot separate from the reviewed audio range', async () => {
|
||||
const screenshots: number[] = [];
|
||||
const audioRanges: number[][] = [];
|
||||
const deps = createDeps();
|
||||
deps.client.notesInfo = async () => [{ noteId: 42, fields: { Picture: { value: '' } } }];
|
||||
deps.mediaGenerator.generateScreenshot = async (_media, time) => {
|
||||
screenshots.push(time);
|
||||
return Buffer.from('image');
|
||||
};
|
||||
deps.mediaGenerator.generateAudio = async (_media, start, end, padding) => {
|
||||
audioRanges.push([start, end, padding ?? -1]);
|
||||
return Buffer.from('audio');
|
||||
};
|
||||
const queue = new PendingYoutubeMediaQueue(deps);
|
||||
assert.equal(
|
||||
await queue.queueFromNote({
|
||||
noteId: 42,
|
||||
noteInfo: { noteId: 42, fields: {} },
|
||||
label: 'test',
|
||||
context: {
|
||||
source: 'overlay',
|
||||
text: '字幕',
|
||||
startTime: 1,
|
||||
endTime: 2,
|
||||
mediaPaddingSeconds: 0,
|
||||
screenshotTime: 3.125,
|
||||
},
|
||||
}),
|
||||
true,
|
||||
);
|
||||
await queue.handleReady('https://youtu.be/abc123', '/cache/video.mkv');
|
||||
assert.deepEqual(screenshots, [3.125]);
|
||||
assert.deepEqual(audioRanges, [[1, 2, 0]]);
|
||||
});
|
||||
|
||||
test('PendingYoutubeMediaQueue treats cache lookup failures as an immediate generation fallback', async () => {
|
||||
const deps = createDeps({
|
||||
getCachedMediaPath: async () => {
|
||||
|
||||
@@ -147,6 +147,9 @@ export class PendingYoutubeMediaQueue {
|
||||
generateAudio: shouldGenerateAudio(config),
|
||||
generateImage: shouldGenerateImage(config),
|
||||
volumeScale,
|
||||
...(job.context?.screenshotTime !== undefined
|
||||
? { screenshotTime: job.context.screenshotTime }
|
||||
: {}),
|
||||
...(job.context?.mediaPaddingSeconds !== undefined
|
||||
? { mediaPaddingSeconds: job.context.mediaPaddingSeconds }
|
||||
: {}),
|
||||
@@ -320,6 +323,7 @@ export class PendingYoutubeMediaQueue {
|
||||
job.endTime,
|
||||
animatedLeadInSeconds,
|
||||
job.mediaPaddingSeconds,
|
||||
job.screenshotTime,
|
||||
);
|
||||
if (imageBuffer) {
|
||||
await this.deps.client.storeMediaFile(imageFilename, imageBuffer);
|
||||
@@ -381,6 +385,7 @@ export class PendingYoutubeMediaQueue {
|
||||
endTime: number,
|
||||
animatedLeadInSeconds = 0,
|
||||
mediaPaddingSeconds?: number,
|
||||
screenshotTime?: number,
|
||||
): Promise<Buffer | null> {
|
||||
const config = this.deps.getConfig();
|
||||
if (config.media?.imageType === 'avif') {
|
||||
@@ -399,7 +404,7 @@ export class PendingYoutubeMediaQueue {
|
||||
);
|
||||
}
|
||||
|
||||
const timestamp = startTime + (endTime - startTime) / 2;
|
||||
const timestamp = screenshotTime ?? startTime + (endTime - startTime) / 2;
|
||||
return this.deps.mediaGenerator.generateScreenshot(videoPath, timestamp, {
|
||||
format: config.media?.imageFormat as 'jpg' | 'png' | 'webp',
|
||||
quality: config.media?.imageQuality,
|
||||
|
||||
@@ -11,6 +11,7 @@ export interface PendingYoutubeMediaUpdate {
|
||||
generateImage: boolean;
|
||||
volumeScale?: number;
|
||||
mediaPaddingSeconds?: number;
|
||||
screenshotTime?: number;
|
||||
}
|
||||
|
||||
function trimToNonEmptyString(value: unknown): string | null {
|
||||
|
||||
@@ -671,7 +671,34 @@ test('registerIpcHandlers accepts the keep-without-media timing decision', async
|
||||
assert.deepEqual(requests, [{ reviewId: 'review-1', decision: { action: 'skip-media' } }]);
|
||||
});
|
||||
|
||||
test('registerIpcHandlers validates and forwards combined timing review text', async () => {
|
||||
test('frame IPC validates timestamps and directions', async () => {
|
||||
const { registrar, handlers } = createFakeIpcRegistrar();
|
||||
const requests: unknown[] = [];
|
||||
registerIpcHandlers(
|
||||
createRegisterIpcDeps({
|
||||
getMediaTimingReviewFrame: async (request) => {
|
||||
requests.push(request);
|
||||
return { ok: true };
|
||||
},
|
||||
}),
|
||||
registrar,
|
||||
);
|
||||
const frame = handlers.handle.get(IPC_CHANNELS.request.mediaTimingReviewFrame)!;
|
||||
const valid = { reviewId: 'r', timestamp: 13, direction: 1 };
|
||||
assert.deepEqual(await frame({}, valid), { ok: true });
|
||||
for (const invalid of [
|
||||
null,
|
||||
{},
|
||||
{ ...valid, timestamp: NaN },
|
||||
{ ...valid, timestamp: '13' },
|
||||
{ ...valid, direction: 2 },
|
||||
]) {
|
||||
assert.equal(((await frame({}, invalid)) as { ok: boolean }).ok, false);
|
||||
}
|
||||
assert.deepEqual(requests, [valid]);
|
||||
});
|
||||
|
||||
test('registerIpcHandlers validates and forwards timing review text and screenshot selection', async () => {
|
||||
const { registrar, handlers } = createFakeIpcRegistrar();
|
||||
const requests: unknown[] = [];
|
||||
registerIpcHandlers(
|
||||
@@ -696,6 +723,7 @@ test('registerIpcHandlers validates and forwards combined timing review text', a
|
||||
startTime: 10,
|
||||
endTime: 12,
|
||||
text: '前の行 対象の行',
|
||||
screenshotTime: 13,
|
||||
},
|
||||
},
|
||||
),
|
||||
@@ -709,20 +737,23 @@ test('registerIpcHandlers validates and forwards combined timing review text', a
|
||||
startTime: 10,
|
||||
endTime: 12,
|
||||
text: '前の行 対象の行',
|
||||
screenshotTime: 13,
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
assert.deepEqual(
|
||||
await handler!(
|
||||
{},
|
||||
{
|
||||
reviewId: 'review-1',
|
||||
decision: { action: 'confirm', startTime: 10, endTime: 12, text: ' ' },
|
||||
},
|
||||
),
|
||||
{ ok: false, message: 'Timing review is unavailable.' },
|
||||
);
|
||||
for (const invalid of [{ text: ' ' }, { screenshotTime: Infinity }]) {
|
||||
assert.deepEqual(
|
||||
await handler!(
|
||||
{},
|
||||
{
|
||||
reviewId: 'review-1',
|
||||
decision: { action: 'confirm', startTime: 10, endTime: 12, ...invalid },
|
||||
},
|
||||
),
|
||||
{ ok: false, message: 'Timing review is unavailable.' },
|
||||
);
|
||||
}
|
||||
assert.equal(requests.length, 1);
|
||||
});
|
||||
|
||||
|
||||
@@ -24,6 +24,8 @@ import type {
|
||||
MediaTimingReviewActionResult,
|
||||
MediaTimingReviewPreviewRequest,
|
||||
MediaTimingReviewResolveRequest,
|
||||
MediaTimingReviewFrameRequest,
|
||||
MediaTimingReviewFrameResult,
|
||||
MediaTimingReviewWaveformRequest,
|
||||
MediaTimingReviewWaveformResult,
|
||||
} from '../../types/anki';
|
||||
@@ -111,6 +113,9 @@ export interface IpcServiceDeps {
|
||||
previewMediaTimingReview?: (
|
||||
request: MediaTimingReviewPreviewRequest,
|
||||
) => Promise<MediaTimingReviewActionResult>;
|
||||
getMediaTimingReviewFrame?: (
|
||||
request: MediaTimingReviewFrameRequest,
|
||||
) => Promise<MediaTimingReviewFrameResult>;
|
||||
getMediaTimingReviewWaveform?: (
|
||||
request: MediaTimingReviewWaveformRequest,
|
||||
) => Promise<MediaTimingReviewWaveformResult>;
|
||||
@@ -269,6 +274,26 @@ function parseMediaTimingReviewWaveformRequest(
|
||||
return parseMediaTimingReviewPreviewRequest(payload);
|
||||
}
|
||||
|
||||
function parseMediaTimingReviewFrameRequest(
|
||||
payload: unknown,
|
||||
): MediaTimingReviewFrameRequest | null {
|
||||
if (!payload || typeof payload !== 'object') return null;
|
||||
const record = payload as Record<string, unknown>;
|
||||
if (
|
||||
typeof record.reviewId !== 'string' ||
|
||||
!record.reviewId ||
|
||||
typeof record.timestamp !== 'number' ||
|
||||
!Number.isFinite(record.timestamp) ||
|
||||
(record.direction !== undefined && record.direction !== -1 && record.direction !== 1)
|
||||
)
|
||||
return null;
|
||||
return {
|
||||
reviewId: record.reviewId,
|
||||
timestamp: record.timestamp,
|
||||
...(record.direction !== undefined ? { direction: record.direction as -1 | 1 } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function parseMediaTimingReviewResolveRequest(
|
||||
payload: unknown,
|
||||
): MediaTimingReviewResolveRequest | null {
|
||||
@@ -291,6 +316,9 @@ function parseMediaTimingReviewResolveRequest(
|
||||
Number.isFinite(decisionRecord.startTime) &&
|
||||
typeof decisionRecord.endTime === 'number' &&
|
||||
Number.isFinite(decisionRecord.endTime) &&
|
||||
(decisionRecord.screenshotTime === undefined ||
|
||||
(typeof decisionRecord.screenshotTime === 'number' &&
|
||||
Number.isFinite(decisionRecord.screenshotTime))) &&
|
||||
(decisionRecord.text === undefined ||
|
||||
(typeof decisionRecord.text === 'string' && decisionRecord.text.trim().length > 0))
|
||||
) {
|
||||
@@ -300,6 +328,9 @@ function parseMediaTimingReviewResolveRequest(
|
||||
action: 'confirm',
|
||||
startTime: decisionRecord.startTime,
|
||||
endTime: decisionRecord.endTime,
|
||||
...(decisionRecord.screenshotTime === undefined
|
||||
? {}
|
||||
: { screenshotTime: decisionRecord.screenshotTime as number }),
|
||||
...(decisionRecord.text === undefined ? {} : { text: decisionRecord.text }),
|
||||
},
|
||||
};
|
||||
@@ -365,6 +396,7 @@ export interface IpcDepsRuntimeOptions {
|
||||
request: YoutubePickerResolveRequest,
|
||||
) => Promise<YoutubePickerResolveResult>;
|
||||
previewMediaTimingReview?: IpcServiceDeps['previewMediaTimingReview'];
|
||||
getMediaTimingReviewFrame?: IpcServiceDeps['getMediaTimingReviewFrame'];
|
||||
getMediaTimingReviewWaveform?: IpcServiceDeps['getMediaTimingReviewWaveform'];
|
||||
stopMediaTimingReviewPreview?: IpcServiceDeps['stopMediaTimingReviewPreview'];
|
||||
resolveMediaTimingReview?: IpcServiceDeps['resolveMediaTimingReview'];
|
||||
@@ -463,6 +495,7 @@ export function createIpcDepsRuntime(options: IpcDepsRuntimeOptions): IpcService
|
||||
runSubsyncManual: options.runSubsyncManual,
|
||||
onYoutubePickerResolve: options.onYoutubePickerResolve,
|
||||
previewMediaTimingReview: options.previewMediaTimingReview,
|
||||
getMediaTimingReviewFrame: options.getMediaTimingReviewFrame,
|
||||
getMediaTimingReviewWaveform: options.getMediaTimingReviewWaveform,
|
||||
stopMediaTimingReviewPreview: options.stopMediaTimingReviewPreview,
|
||||
resolveMediaTimingReview: options.resolveMediaTimingReview,
|
||||
@@ -613,6 +646,16 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar
|
||||
return await deps.getMediaTimingReviewWaveform(request);
|
||||
},
|
||||
);
|
||||
ipc.handle(
|
||||
IPC_CHANNELS.request.mediaTimingReviewFrame,
|
||||
async (_event: unknown, payload: unknown) => {
|
||||
const request = parseMediaTimingReviewFrameRequest(payload);
|
||||
if (!request || !deps.getMediaTimingReviewFrame) {
|
||||
return { ok: false, message: 'Screenshot preview is unavailable.' };
|
||||
}
|
||||
return await deps.getMediaTimingReviewFrame(request);
|
||||
},
|
||||
);
|
||||
ipc.handle(
|
||||
IPC_CHANNELS.request.mediaTimingReviewStopPreview,
|
||||
async (_event: unknown, reviewId: unknown) => {
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { createMediaTimingFrameExtractor, selectMediaTimingFrame } from './media-timing-frame';
|
||||
|
||||
test('frame stepping follows decoded timestamps with variable frame durations', () => {
|
||||
const times = [10, 10.041667, 10.125, 10.166667];
|
||||
assert.equal(selectMediaTimingFrame(times, 10.05), 10.125);
|
||||
assert.equal(selectMediaTimingFrame(times, 10.125 - 0.000001, 1), 10.166667);
|
||||
assert.equal(selectMediaTimingFrame(times, 10.125 - 0.000001, -1), 10.041667);
|
||||
assert.equal(selectMediaTimingFrame(times, 10, -1), undefined);
|
||||
assert.equal(selectMediaTimingFrame(times, 10.166667, 1), undefined);
|
||||
assert.equal(selectMediaTimingFrame([], 10), undefined);
|
||||
});
|
||||
|
||||
function fixture(startTime = 0) {
|
||||
const calls: Array<{ file: string; args: string[] }> = [];
|
||||
const extractor = createMediaTimingFrameExtractor(async (file, args) => {
|
||||
calls.push({ file, args });
|
||||
if (file === 'ffmpeg') return Buffer.from('image');
|
||||
if (args.includes('format=start_time'))
|
||||
return Buffer.from(JSON.stringify({ format: { start_time: String(startTime) } }));
|
||||
return Buffer.from(
|
||||
JSON.stringify({
|
||||
frames: [10, 10.04, 10.12, 10.16].map((time) => ({
|
||||
best_effort_timestamp_time: String(time + startTime),
|
||||
})),
|
||||
}),
|
||||
);
|
||||
});
|
||||
return { calls, extractor };
|
||||
}
|
||||
|
||||
test('frame extraction normalizes nonzero source start times and reuses the frame index', async () => {
|
||||
const { calls, extractor } = fixture(5);
|
||||
const first = await extractor.generate({ media: '/movie.mkv', timestamp: 10.05 });
|
||||
assert.ok(Math.abs(first.timestamp - 10.119999) < 0.000001);
|
||||
assert.equal(first.dataUrl, 'data:image/jpeg;base64,aW1hZ2U=');
|
||||
await extractor.generate({ media: '/movie.mkv', timestamp: first.timestamp, direction: 1 });
|
||||
assert.equal(calls.filter((call) => call.file === 'ffprobe').length, 2);
|
||||
assert.ok(calls[1]!.args.includes('13.05%17.05'));
|
||||
extractor.clear();
|
||||
await extractor.generate({ media: '/movie.mkv', timestamp: 10.05 });
|
||||
assert.equal(calls.filter((call) => call.file === 'ffprobe').length, 4);
|
||||
});
|
||||
|
||||
test('cached windows keep source timestamps for ffmpeg and use absolute ffprobe intervals', async () => {
|
||||
const { calls, extractor } = fixture();
|
||||
await extractor.generate({
|
||||
media: { path: '/window.mkv', absoluteTimestamps: true },
|
||||
timestamp: 10.05,
|
||||
});
|
||||
assert.ok(calls[1]!.args.includes('8.05%12.05'));
|
||||
assert.equal(calls[1]!.args.includes('-seek_timestamp'), false);
|
||||
assert.ok(calls[2]!.args.includes('-seek_timestamp'));
|
||||
});
|
||||
|
||||
test('remote frame reads carry source headers and do not silently reuse a different input', async () => {
|
||||
const { calls, extractor } = fixture();
|
||||
const media = {
|
||||
path: 'https://example.test/video',
|
||||
inputOptions: { headers: { 'X-Emby-Token': 'test-token' } },
|
||||
};
|
||||
await extractor.generate({ media, timestamp: 10.05 });
|
||||
assert.ok(calls.every((call) => call.args.includes('X-Emby-Token: test-token\r\n')));
|
||||
await extractor.generate({
|
||||
media: { ...media, path: 'https://example.test/other' },
|
||||
timestamp: 10.05,
|
||||
});
|
||||
assert.equal(calls.filter((call) => call.file === 'ffprobe').length, 4);
|
||||
});
|
||||
@@ -0,0 +1,159 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { normalizeMediaInput, type MediaInput } from '../../media-input';
|
||||
|
||||
export interface MediaTimingFrameOptions {
|
||||
media: MediaInput;
|
||||
timestamp: number;
|
||||
direction?: -1 | 1;
|
||||
}
|
||||
|
||||
const FRAME_EPSILON = 0.00001;
|
||||
const PROBE_RADIUS_SECONDS = 2;
|
||||
|
||||
function run(file: string, args: string[]): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile(
|
||||
file,
|
||||
args,
|
||||
{ encoding: 'buffer', timeout: 30_000, maxBuffer: 8 * 1024 * 1024, windowsHide: true },
|
||||
(error, stdout) => {
|
||||
// Child-process errors contain the input URL, which may contain authentication tokens.
|
||||
if (error)
|
||||
reject(
|
||||
new Error('Screenshot preview is unavailable. Check FFmpeg and the video source.'),
|
||||
);
|
||||
else resolve(stdout);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** Uses decoded timestamps, rather than an assumed FPS, including for variable-rate video. */
|
||||
export function selectMediaTimingFrame(
|
||||
times: readonly number[],
|
||||
timestamp: number,
|
||||
direction?: -1 | 1,
|
||||
): number | undefined {
|
||||
if (direction === -1)
|
||||
return [...times].reverse().find((time) => time < timestamp - FRAME_EPSILON);
|
||||
if (direction === 1) return times.find((time) => time > timestamp + FRAME_EPSILON);
|
||||
return times.find((time) => time >= timestamp - FRAME_EPSILON) ?? times.at(-1);
|
||||
}
|
||||
|
||||
export function createMediaTimingFrameExtractor(execute: typeof run = run) {
|
||||
let cached: { key: string; start: number; end: number; times: number[] } | null = null;
|
||||
let source: { key: string; offset: number } | null = null;
|
||||
let generation = 0;
|
||||
|
||||
async function generate(
|
||||
options: MediaTimingFrameOptions,
|
||||
): Promise<{ dataUrl: string; timestamp: number }> {
|
||||
const input = normalizeMediaInput(options.media);
|
||||
const key = JSON.stringify(options.media);
|
||||
const currentGeneration = generation;
|
||||
const absolute = typeof options.media !== 'string' && options.media.absoluteTimestamps;
|
||||
// -seek_timestamp is an ffmpeg option; ffprobe intervals already use stream timestamps.
|
||||
const probeInputArgs = normalizeMediaInput({
|
||||
path: input.path,
|
||||
...(typeof options.media !== 'string' ? { inputOptions: options.media.inputOptions } : {}),
|
||||
}).inputArgs;
|
||||
let offset = source?.key === key ? source.offset : undefined;
|
||||
if (offset === undefined) {
|
||||
const metadata = JSON.parse(
|
||||
(
|
||||
await execute('ffprobe', [
|
||||
'-v',
|
||||
'error',
|
||||
...probeInputArgs,
|
||||
'-show_entries',
|
||||
'format=start_time',
|
||||
'-of',
|
||||
'json',
|
||||
input.path,
|
||||
])
|
||||
).toString(),
|
||||
) as { format?: { start_time?: string } };
|
||||
const start = Number(metadata.format?.start_time ?? 0);
|
||||
offset = absolute || !Number.isFinite(start) ? 0 : start;
|
||||
if (generation === currentGeneration) source = { key, offset };
|
||||
}
|
||||
let times: number[];
|
||||
if (
|
||||
cached?.key === key &&
|
||||
options.timestamp > cached.start + 0.5 &&
|
||||
options.timestamp < cached.end - 0.5
|
||||
) {
|
||||
times = cached.times;
|
||||
} else {
|
||||
const start = Math.max(0, options.timestamp - PROBE_RADIUS_SECONDS);
|
||||
const end = options.timestamp + PROBE_RADIUS_SECONDS;
|
||||
const result = JSON.parse(
|
||||
(
|
||||
await execute('ffprobe', [
|
||||
'-v',
|
||||
'error',
|
||||
...probeInputArgs,
|
||||
'-read_intervals',
|
||||
`${start + offset}%${end + offset}`,
|
||||
'-select_streams',
|
||||
'v:0',
|
||||
'-show_entries',
|
||||
'frame=best_effort_timestamp_time',
|
||||
'-of',
|
||||
'json',
|
||||
input.path,
|
||||
])
|
||||
).toString(),
|
||||
) as { frames?: { best_effort_timestamp_time?: string }[] };
|
||||
times = [
|
||||
...new Set(
|
||||
(result.frames ?? [])
|
||||
.map((frame) => Number(frame.best_effort_timestamp_time) - offset)
|
||||
.filter((time) => Number.isFinite(time) && time >= 0 && time >= start && time <= end),
|
||||
),
|
||||
].sort((a, b) => a - b);
|
||||
if (generation === currentGeneration) cached = { key, start, end, times };
|
||||
}
|
||||
const timestamp = selectMediaTimingFrame(times, options.timestamp, options.direction);
|
||||
if (timestamp === undefined) throw new Error('No adjacent video frame is available here.');
|
||||
// Round down by one microsecond so decimal timestamp rounding cannot skip the chosen frame.
|
||||
const seekTime = Math.max(0, timestamp - 0.000001);
|
||||
const image = await execute('ffmpeg', [
|
||||
'-hide_banner',
|
||||
'-nostdin',
|
||||
'-loglevel',
|
||||
'error',
|
||||
'-ss',
|
||||
String(seekTime),
|
||||
...input.inputArgs,
|
||||
'-i',
|
||||
input.path,
|
||||
'-map',
|
||||
'0:v:0',
|
||||
'-frames:v',
|
||||
'1',
|
||||
'-an',
|
||||
'-sn',
|
||||
'-vf',
|
||||
'scale=w=640:h=360:force_original_aspect_ratio=decrease',
|
||||
'-c:v',
|
||||
'mjpeg',
|
||||
'-q:v',
|
||||
'3',
|
||||
'-f',
|
||||
'image2pipe',
|
||||
'pipe:1',
|
||||
]);
|
||||
if (!image.length) throw new Error('No video frame is available here.');
|
||||
return { dataUrl: `data:image/jpeg;base64,${image.toString('base64')}`, timestamp: seekTime };
|
||||
}
|
||||
|
||||
return {
|
||||
generate,
|
||||
clear: () => {
|
||||
generation += 1;
|
||||
cached = null;
|
||||
source = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
+11
@@ -475,6 +475,7 @@ import { MediaTimingPreviewSession } from './core/services/media-timing-preview'
|
||||
import { getSharedRemoteMediaWindowCache } from './core/services/remote-media-window-cache';
|
||||
import { resolveMediaGenerationInput } from './anki-integration/media-source';
|
||||
import { generateSpeechWaveform } from './core/services/media-timing-waveform';
|
||||
import { createMediaTimingFrameExtractor } from './core/services/media-timing-frame';
|
||||
import {
|
||||
collectMediaTimingContextLines,
|
||||
createMediaTimingReviewRuntime,
|
||||
@@ -2905,6 +2906,7 @@ function createOverlayHostedModalOpenDeps(): {
|
||||
};
|
||||
}
|
||||
|
||||
const mediaTimingFrameExtractor = createMediaTimingFrameExtractor();
|
||||
const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({
|
||||
getMpvClient: () => appState.mpvClient,
|
||||
getCurrentMediaPath: () =>
|
||||
@@ -2913,6 +2915,14 @@ const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({
|
||||
configService.getConfig().mpv.executablePath || process.env.SUBMINER_MPV_PATH?.trim() || '',
|
||||
createPreviewSession: () => new MediaTimingPreviewSession(),
|
||||
generateWaveform: (options) => generateSpeechWaveform(options),
|
||||
generateFrame: (options) => mediaTimingFrameExtractor.generate(options),
|
||||
clearFrameCache: () => mediaTimingFrameExtractor.clear(),
|
||||
resolveVideoSource: () =>
|
||||
resolveMediaGenerationInput(appState.mpvClient, 'video', {
|
||||
getCachedMediaPath: (currentVideoPath, kind) =>
|
||||
getCachedYoutubeMediaPathForCurrentPlayback(currentVideoPath, kind),
|
||||
remoteCacheMode: shouldRequireYoutubeMediaCacheForCurrentPlayback() ? 'required' : 'optional',
|
||||
}),
|
||||
resolveMediaSource: async () => {
|
||||
const resolved = await resolveMediaGenerationInput(appState.mpvClient, 'audio', {
|
||||
getCachedMediaPath: (currentVideoPath, kind) =>
|
||||
@@ -5586,6 +5596,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
|
||||
},
|
||||
mainDeps: {
|
||||
previewMediaTimingReview: (request) => mediaTimingReviewRuntime.previewRange(request),
|
||||
getMediaTimingReviewFrame: (request) => mediaTimingReviewRuntime.getFrame(request),
|
||||
getMediaTimingReviewWaveform: (request) => mediaTimingReviewRuntime.getWaveform(request),
|
||||
stopMediaTimingReviewPreview: (reviewId) => mediaTimingReviewRuntime.stopPreview(reviewId),
|
||||
resolveMediaTimingReview: (request) => mediaTimingReviewRuntime.resolveReview(request),
|
||||
|
||||
@@ -63,6 +63,7 @@ export interface MainIpcRuntimeServiceDepsParams {
|
||||
handleOverlayNotificationAction?: IpcDepsRuntimeOptions['handleOverlayNotificationAction'];
|
||||
onYoutubePickerResolve: IpcDepsRuntimeOptions['onYoutubePickerResolve'];
|
||||
previewMediaTimingReview?: IpcDepsRuntimeOptions['previewMediaTimingReview'];
|
||||
getMediaTimingReviewFrame?: IpcDepsRuntimeOptions['getMediaTimingReviewFrame'];
|
||||
getMediaTimingReviewWaveform?: IpcDepsRuntimeOptions['getMediaTimingReviewWaveform'];
|
||||
stopMediaTimingReviewPreview?: IpcDepsRuntimeOptions['stopMediaTimingReviewPreview'];
|
||||
resolveMediaTimingReview?: IpcDepsRuntimeOptions['resolveMediaTimingReview'];
|
||||
@@ -263,6 +264,7 @@ export function createMainIpcRuntimeServiceDeps(
|
||||
handleOverlayNotificationAction: params.handleOverlayNotificationAction,
|
||||
onYoutubePickerResolve: params.onYoutubePickerResolve,
|
||||
previewMediaTimingReview: params.previewMediaTimingReview,
|
||||
getMediaTimingReviewFrame: params.getMediaTimingReviewFrame,
|
||||
getMediaTimingReviewWaveform: params.getMediaTimingReviewWaveform,
|
||||
stopMediaTimingReviewPreview: params.stopMediaTimingReviewPreview,
|
||||
resolveMediaTimingReview: params.resolveMediaTimingReview,
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import type { MediaTimingReviewOpenPayload } from '../../types/anki';
|
||||
import type { MediaTimingFrameOptions } from '../../core/services/media-timing-frame';
|
||||
import {
|
||||
createMediaTimingReviewRuntime,
|
||||
type MediaTimingReviewRuntimeDeps,
|
||||
} from './media-timing-review';
|
||||
|
||||
async function start(
|
||||
options: Partial<MediaTimingReviewRuntimeDeps> = {},
|
||||
screenshotEnabled = true,
|
||||
) {
|
||||
let opened!: (payload: MediaTimingReviewOpenPayload) => void;
|
||||
const payloadPromise = new Promise<MediaTimingReviewOpenPayload>((resolve) => {
|
||||
opened = resolve;
|
||||
});
|
||||
const frames: MediaTimingFrameOptions[] = [];
|
||||
const runtime = createMediaTimingReviewRuntime({
|
||||
getMpvClient: () => ({
|
||||
connected: true,
|
||||
currentVideoPath: '/video.mkv',
|
||||
send: () => {},
|
||||
requestProperty: async (name) => (name === 'duration' ? 100 : true),
|
||||
}),
|
||||
getCurrentMediaPath: () => '/video.mkv',
|
||||
getMpvExecutablePath: () => '',
|
||||
createPreviewSession: () => ({
|
||||
start: async () => {},
|
||||
play: async () => {},
|
||||
stop: async () => {},
|
||||
onPlaybackEnded: () => {},
|
||||
dispose: () => {},
|
||||
}),
|
||||
generateWaveform: async () => [0, 1],
|
||||
resolveVideoSource: async () => ({ path: '/video.mkv' }),
|
||||
generateFrame: async (options) => {
|
||||
frames.push(options);
|
||||
return { dataUrl: 'data:image/jpeg;base64,AA==', timestamp: options.timestamp };
|
||||
},
|
||||
openModal: async (payload) => {
|
||||
opened(payload);
|
||||
return true;
|
||||
},
|
||||
showStatus: () => {},
|
||||
...options,
|
||||
});
|
||||
const decision = runtime.requestReview({
|
||||
kind: 'word',
|
||||
text: '字幕',
|
||||
startTime: 10,
|
||||
endTime: 12,
|
||||
audioPadding: 0,
|
||||
maxMediaDuration: 30,
|
||||
screenshotEnabled,
|
||||
});
|
||||
return { runtime, decision, payload: await payloadPromise, frames };
|
||||
}
|
||||
|
||||
test('review accepts a screenshot outside the audio range and validates media bounds', async () => {
|
||||
const { runtime, decision, payload, frames } = await start();
|
||||
for (const timestamp of [NaN, Infinity, -1, 100]) {
|
||||
assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp })).ok, false);
|
||||
assert.equal(
|
||||
runtime.resolveReview({
|
||||
reviewId: payload.reviewId,
|
||||
decision: { action: 'confirm', startTime: 10, endTime: 11, screenshotTime: timestamp },
|
||||
}).ok,
|
||||
false,
|
||||
);
|
||||
}
|
||||
assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 13 })).ok, true);
|
||||
assert.equal(frames[0]?.timestamp, 13);
|
||||
assert.equal(
|
||||
runtime.resolveReview({
|
||||
reviewId: payload.reviewId,
|
||||
decision: { action: 'confirm', startTime: 10, endTime: 11, screenshotTime: 13 },
|
||||
}).ok,
|
||||
true,
|
||||
);
|
||||
assert.deepEqual(await decision, {
|
||||
action: 'confirm',
|
||||
startTime: 10,
|
||||
endTime: 11,
|
||||
screenshotTime: 13,
|
||||
});
|
||||
});
|
||||
|
||||
test('screenshot preview reuses the remote audio window and its absolute timestamps', async () => {
|
||||
let downloads = 0;
|
||||
const remote = 'https://example.test/video';
|
||||
const { runtime, payload, decision, frames } = await start({
|
||||
resolveMediaSource: async () => ({ path: remote }),
|
||||
resolveVideoSource: async () => ({ path: remote }),
|
||||
acquireMediaWindow: async () => {
|
||||
downloads += 1;
|
||||
return {
|
||||
path: '/window.mkv',
|
||||
sourcePath: remote,
|
||||
startTime: 7,
|
||||
endTime: 15,
|
||||
audioStreamIndex: null,
|
||||
media: { path: '/window.mkv', absoluteTimestamps: true },
|
||||
};
|
||||
},
|
||||
});
|
||||
await runtime.getWaveform({ reviewId: payload.reviewId, startTime: 8, endTime: 14 });
|
||||
assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 })).ok, true);
|
||||
assert.equal(downloads, 1);
|
||||
assert.deepEqual(frames[0]?.media, { path: '/window.mkv', absoluteTimestamps: true });
|
||||
await runtime.dispose();
|
||||
await decision;
|
||||
});
|
||||
|
||||
test('split video streams never read an audio-only cache as a video source', async () => {
|
||||
const video = {
|
||||
path: 'https://example.test/video',
|
||||
inputOptions: { headers: { 'X-Test': 'value' } },
|
||||
};
|
||||
const { runtime, payload, decision, frames } = await start({
|
||||
resolveMediaSource: async () => ({ path: 'https://example.test/audio' }),
|
||||
resolveVideoSource: async () => video,
|
||||
});
|
||||
await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 });
|
||||
assert.deepEqual(frames[0]?.media, video);
|
||||
await runtime.dispose();
|
||||
await decision;
|
||||
});
|
||||
|
||||
test('disabled screenshots and missing video inputs do not trigger extraction', async () => {
|
||||
for (const [enabled, options] of [
|
||||
[false, {}],
|
||||
[true, { resolveVideoSource: async () => null }],
|
||||
] as const) {
|
||||
const { runtime, payload, decision, frames } = await start(options, enabled);
|
||||
assert.equal((await runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 })).ok, false);
|
||||
assert.equal(frames.length, 0);
|
||||
await runtime.dispose();
|
||||
await decision;
|
||||
}
|
||||
});
|
||||
|
||||
test('closing a review invalidates in-flight frame results and clears the frame index', async () => {
|
||||
let finish!: (value: { dataUrl: string; timestamp: number }) => void;
|
||||
let clearCount = 0;
|
||||
const { runtime, payload, decision } = await start({
|
||||
generateFrame: () =>
|
||||
new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
clearFrameCache: () => {
|
||||
clearCount += 1;
|
||||
},
|
||||
});
|
||||
const frame = runtime.getFrame({ reviewId: payload.reviewId, timestamp: 11 });
|
||||
await runtime.dispose();
|
||||
finish({ dataUrl: 'data:image/jpeg;base64,AA==', timestamp: 11 });
|
||||
assert.equal((await frame).stale, true);
|
||||
await decision;
|
||||
assert.equal(clearCount, 1);
|
||||
});
|
||||
@@ -7,10 +7,13 @@ import type {
|
||||
MediaTimingReviewPreviewRequest,
|
||||
MediaTimingReviewRequest,
|
||||
MediaTimingReviewResolveRequest,
|
||||
MediaTimingReviewFrameRequest,
|
||||
MediaTimingReviewFrameResult,
|
||||
MediaTimingReviewWaveformRequest,
|
||||
MediaTimingReviewWaveformResult,
|
||||
} from '../../types/anki';
|
||||
import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform';
|
||||
import type { MediaTimingFrameOptions } from '../../core/services/media-timing-frame';
|
||||
import {
|
||||
isRemoteMediaWindowSourcePath,
|
||||
type RemoteMediaWindow,
|
||||
@@ -59,6 +62,8 @@ interface ActiveReview {
|
||||
mediaPath: string;
|
||||
/** What the waveform reads when no cached window is available. */
|
||||
waveformMedia: MediaInput;
|
||||
videoSource: ReviewMediaSource | null;
|
||||
frameInFlight: boolean;
|
||||
audioStreamIndex?: number;
|
||||
/** Remote source to download windows of; null for local media or without a cache. */
|
||||
windowSource: RemoteMediaWindowSource | null;
|
||||
@@ -81,6 +86,11 @@ export interface MediaTimingReviewRuntimeDeps {
|
||||
generateWaveform: (options: SpeechWaveformOptions) => Promise<number[]>;
|
||||
/** Resolves the FFmpeg-readable stream URL and headers behind the current media path. */
|
||||
resolveMediaSource?: () => Promise<ReviewMediaSource | null>;
|
||||
resolveVideoSource?: () => Promise<ReviewMediaSource | null>;
|
||||
generateFrame?: (
|
||||
options: MediaTimingFrameOptions,
|
||||
) => Promise<{ dataUrl: string; timestamp: number }>;
|
||||
clearFrameCache?: () => void;
|
||||
/** Downloads (or reuses) a local window of a remote source covering the range. */
|
||||
acquireMediaWindow?: (
|
||||
source: RemoteMediaWindowSource,
|
||||
@@ -227,6 +237,9 @@ export function buildMediaTimingReviewPayload(
|
||||
timelineEndTime,
|
||||
...(duration !== null && duration > 0 ? { mediaDuration: duration } : {}),
|
||||
maxMediaDuration,
|
||||
...(request.screenshotEnabled !== undefined
|
||||
? { screenshotEnabled: request.screenshotEnabled }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -335,13 +348,15 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
return { action: 'use-original' };
|
||||
}
|
||||
|
||||
const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw, resolvedSource] = await Promise.all([
|
||||
mpvClient.requestProperty?.('pause').catch(() => null) ?? null,
|
||||
mpvClient.requestProperty?.('duration').catch(() => null) ?? null,
|
||||
mpvClient.requestProperty?.('aid').catch(() => null) ?? null,
|
||||
mpvClient.requestProperty?.('volume').catch(() => null) ?? null,
|
||||
deps.resolveMediaSource?.().catch(() => null) ?? null,
|
||||
]);
|
||||
const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw, resolvedSource, videoSource] =
|
||||
await Promise.all([
|
||||
mpvClient.requestProperty?.('pause').catch(() => null) ?? null,
|
||||
mpvClient.requestProperty?.('duration').catch(() => null) ?? null,
|
||||
mpvClient.requestProperty?.('aid').catch(() => null) ?? null,
|
||||
mpvClient.requestProperty?.('volume').catch(() => null) ?? null,
|
||||
deps.resolveMediaSource?.().catch(() => null) ?? null,
|
||||
request.screenshotEnabled ? (deps.resolveVideoSource?.().catch(() => null) ?? null) : null,
|
||||
]);
|
||||
const pauseState = booleanProperty(pauseRaw);
|
||||
mpvClient.send({ command: ['set_property', 'pause', 'yes'] });
|
||||
pendingPauseRestore = pauseState === false ? mpvClient : null;
|
||||
@@ -383,6 +398,8 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
payload,
|
||||
mediaPath,
|
||||
waveformMedia: inputOptions ? { path: sourcePath, inputOptions } : sourcePath,
|
||||
videoSource,
|
||||
frameInFlight: false,
|
||||
...(audioStreamIndex !== undefined ? { audioStreamIndex } : {}),
|
||||
windowSource,
|
||||
window: null,
|
||||
@@ -527,6 +544,58 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
}
|
||||
}
|
||||
|
||||
async function getFrame(
|
||||
request: MediaTimingReviewFrameRequest,
|
||||
): Promise<MediaTimingReviewFrameResult> {
|
||||
const current = active;
|
||||
if (!current || request.reviewId !== current.payload.reviewId) return staleReviewResult();
|
||||
if (!current.payload.screenshotEnabled || !deps.generateFrame || !current.videoSource) {
|
||||
return { ok: false, message: 'Screenshot preview is unavailable for this media.' };
|
||||
}
|
||||
if (
|
||||
!Number.isFinite(request.timestamp) ||
|
||||
request.timestamp < 0 ||
|
||||
(current.payload.mediaDuration !== undefined &&
|
||||
request.timestamp >= current.payload.mediaDuration) ||
|
||||
(request.direction !== undefined && request.direction !== -1 && request.direction !== 1)
|
||||
) {
|
||||
return { ok: false, message: 'The screenshot time is invalid.' };
|
||||
}
|
||||
if (current.frameInFlight)
|
||||
return { ok: false, message: 'A screenshot preview is already loading.' };
|
||||
current.frameInFlight = true;
|
||||
try {
|
||||
// Reuse the audio window only when it contains this same video source (not split streams).
|
||||
const range = {
|
||||
startTime: Math.max(0, Math.min(current.payload.timelineStartTime, request.timestamp - 2)),
|
||||
endTime: Math.min(
|
||||
current.payload.mediaDuration ?? Infinity,
|
||||
Math.max(current.payload.timelineEndTime, request.timestamp + 2),
|
||||
),
|
||||
};
|
||||
const window =
|
||||
current.windowSource?.path === current.videoSource.path
|
||||
? await ensureWindow(current, range)
|
||||
: null;
|
||||
if (active !== current) return staleReviewResult();
|
||||
const frame = await deps.generateFrame({
|
||||
media: window?.media ?? current.videoSource,
|
||||
timestamp: request.timestamp,
|
||||
...(request.direction !== undefined ? { direction: request.direction } : {}),
|
||||
});
|
||||
if (active !== current) return staleReviewResult();
|
||||
return { ok: true, ...frame };
|
||||
} catch {
|
||||
if (active !== current) return staleReviewResult();
|
||||
return {
|
||||
ok: false,
|
||||
message: 'Screenshot preview unavailable. Try another time or reset to the midpoint.',
|
||||
};
|
||||
} finally {
|
||||
current.frameInFlight = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function stopPreview(reviewId: string): Promise<MediaTimingReviewActionResult> {
|
||||
const current = active;
|
||||
if (!current || reviewId !== current.payload.reviewId) {
|
||||
@@ -550,13 +619,23 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
return staleReviewResult();
|
||||
}
|
||||
if (request.decision.action === 'confirm') {
|
||||
const { startTime, endTime, text } = request.decision;
|
||||
const { startTime, endTime, text, screenshotTime } = request.decision;
|
||||
if (!isValidMediaTimingRange(current.payload, startTime, endTime)) {
|
||||
return { ok: false, message: 'The selected timing range is invalid.' };
|
||||
}
|
||||
if (text !== undefined && (typeof text !== 'string' || text.trim().length === 0)) {
|
||||
return { ok: false, message: 'The combined sentence text is invalid.' };
|
||||
}
|
||||
if (
|
||||
screenshotTime !== undefined &&
|
||||
(!current.payload.screenshotEnabled ||
|
||||
!Number.isFinite(screenshotTime) ||
|
||||
screenshotTime < 0 ||
|
||||
(current.payload.mediaDuration !== undefined &&
|
||||
screenshotTime >= current.payload.mediaDuration))
|
||||
) {
|
||||
return { ok: false, message: 'The screenshot time is invalid.' };
|
||||
}
|
||||
}
|
||||
current.resolve(request.decision);
|
||||
return { ok: true };
|
||||
@@ -566,6 +645,7 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
const current = active;
|
||||
active = null;
|
||||
if (!current) return;
|
||||
deps.clearFrameCache?.();
|
||||
void current.preview?.session.then((session) => session.dispose()).catch(() => {});
|
||||
if (current.restorePlayback && current.mpvClient.connected) {
|
||||
current.mpvClient.send({ command: ['set_property', 'pause', 'no'] });
|
||||
@@ -582,6 +662,7 @@ export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDep
|
||||
requestReview,
|
||||
previewRange,
|
||||
getWaveform,
|
||||
getFrame,
|
||||
stopPreview,
|
||||
resolveReview,
|
||||
dispose,
|
||||
|
||||
@@ -73,6 +73,8 @@ import type {
|
||||
MediaTimingReviewOpenPayload,
|
||||
MediaTimingReviewPreviewRequest,
|
||||
MediaTimingReviewResolveRequest,
|
||||
MediaTimingReviewFrameRequest,
|
||||
MediaTimingReviewFrameResult,
|
||||
MediaTimingReviewWaveformRequest,
|
||||
} from './types';
|
||||
import { IPC_CHANNELS } from './shared/ipc/contracts';
|
||||
@@ -507,6 +509,10 @@ const electronAPI: ElectronAPI = {
|
||||
request: MediaTimingReviewPreviewRequest,
|
||||
): Promise<MediaTimingReviewActionResult> =>
|
||||
ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewPreview, request),
|
||||
getMediaTimingReviewFrame: (
|
||||
request: MediaTimingReviewFrameRequest,
|
||||
): Promise<MediaTimingReviewFrameResult> =>
|
||||
ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewFrame, request),
|
||||
getMediaTimingReviewWaveform: (request: MediaTimingReviewWaveformRequest) =>
|
||||
ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewWaveform, request),
|
||||
stopMediaTimingReviewPreview: (reviewId: string): Promise<MediaTimingReviewActionResult> =>
|
||||
|
||||
@@ -297,6 +297,57 @@
|
||||
</div>
|
||||
<blockquote id="mediaTimingReviewText" class="media-timing-review-text"></blockquote>
|
||||
|
||||
<div id="mediaTimingReviewFramePicker" class="media-timing-frame-picker hidden">
|
||||
<div class="media-timing-frame-image-shell">
|
||||
<img
|
||||
id="mediaTimingReviewFrameImage"
|
||||
class="hidden"
|
||||
alt="Selected card screenshot"
|
||||
/>
|
||||
</div>
|
||||
<div class="media-timing-frame-controls">
|
||||
<div class="media-timing-frame-heading">
|
||||
<label for="mediaTimingReviewFrameSlider">Card screenshot</label>
|
||||
<output id="mediaTimingReviewFrameTime" for="mediaTimingReviewFrameSlider"
|
||||
>—</output
|
||||
>
|
||||
</div>
|
||||
<input
|
||||
id="mediaTimingReviewFrameSlider"
|
||||
type="range"
|
||||
min="0"
|
||||
max="1"
|
||||
step="0.001"
|
||||
value="0"
|
||||
aria-label="Screenshot time"
|
||||
/>
|
||||
<div class="media-timing-frame-buttons">
|
||||
<button
|
||||
id="mediaTimingReviewFramePrevious"
|
||||
type="button"
|
||||
aria-label="Previous video frame"
|
||||
>
|
||||
← Frame
|
||||
</button>
|
||||
<button
|
||||
id="mediaTimingReviewFrameNext"
|
||||
type="button"
|
||||
aria-label="Next video frame"
|
||||
>
|
||||
Frame →
|
||||
</button>
|
||||
<button
|
||||
id="mediaTimingReviewFrameReset"
|
||||
type="button"
|
||||
aria-label="Reset screenshot"
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</div>
|
||||
<p id="mediaTimingReviewFrameStatus" role="status"></p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="media-timing-review-readout" aria-live="polite">
|
||||
<div>
|
||||
<span>Starts</span>
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { setTimeout as tick } from 'node:timers/promises';
|
||||
import { createMediaTimingFramePicker } from './media-timing-frame-picker';
|
||||
import type { MediaTimingReviewFrameRequest, MediaTimingReviewFrameResult } from '../../types/anki';
|
||||
|
||||
function fixture() {
|
||||
const requests: Array<{
|
||||
request: MediaTimingReviewFrameRequest;
|
||||
resolve: (result: MediaTimingReviewFrameResult) => void;
|
||||
}> = [];
|
||||
let stale = false;
|
||||
const picker = createMediaTimingFramePicker({
|
||||
debounceMs: 0,
|
||||
load: (request) => new Promise((resolve) => requests.push({ request, resolve })),
|
||||
onChange: () => {},
|
||||
onStale: () => {
|
||||
stale = true;
|
||||
picker.close();
|
||||
},
|
||||
});
|
||||
const complete = async (index: number, result: number | MediaTimingReviewFrameResult) => {
|
||||
requests[index]!.resolve(
|
||||
typeof result === 'number'
|
||||
? { ok: true, timestamp: result, dataUrl: `data:image/jpeg;base64,${result}` }
|
||||
: result,
|
||||
);
|
||||
await tick(0);
|
||||
};
|
||||
return { picker, requests, complete, isStale: () => stale };
|
||||
}
|
||||
|
||||
test('a chosen frame stays fixed until Reset restores midpoint tracking', async () => {
|
||||
const { picker, requests, complete } = fixture();
|
||||
picker.open('r', true, 11);
|
||||
await tick(5);
|
||||
await complete(0, 11);
|
||||
assert.equal(picker.getScreenshotTime(), undefined);
|
||||
picker.choose(13);
|
||||
await tick(5);
|
||||
await complete(1, 13);
|
||||
picker.updateMidpoint(12);
|
||||
await tick(5);
|
||||
assert.equal(picker.getScreenshotTime(), 13);
|
||||
assert.equal(requests.length, 2);
|
||||
picker.reset();
|
||||
await tick(5);
|
||||
assert.equal(requests[2]?.request.timestamp, 12);
|
||||
await complete(2, 12);
|
||||
assert.equal(picker.getScreenshotTime(), undefined);
|
||||
picker.updateMidpoint(14);
|
||||
await tick(5);
|
||||
assert.equal(requests[3]?.request.timestamp, 14);
|
||||
await complete(3, 14);
|
||||
picker.close();
|
||||
});
|
||||
|
||||
test('scrubbing keeps only the latest requested frame', async () => {
|
||||
const { picker, requests, complete } = fixture();
|
||||
picker.open('r', true, 11);
|
||||
await tick(5);
|
||||
picker.choose(12);
|
||||
picker.choose(13);
|
||||
picker.choose(14);
|
||||
await tick(5);
|
||||
assert.equal(requests.length, 1);
|
||||
assert.equal(picker.getState().blockConfirm, true);
|
||||
await complete(0, 11);
|
||||
await tick(5);
|
||||
assert.equal(picker.getState().timestamp, undefined);
|
||||
assert.equal(requests[1]?.request.timestamp, 14);
|
||||
await complete(1, 14);
|
||||
assert.equal(picker.getScreenshotTime(), 14);
|
||||
assert.equal(picker.getState().blockConfirm, false);
|
||||
picker.close();
|
||||
});
|
||||
|
||||
test('failed manual previews block confirmation; Reset allows the default fallback', async () => {
|
||||
const { picker, complete } = fixture();
|
||||
picker.open('r', true, 11);
|
||||
await tick(5);
|
||||
await complete(0, { ok: false });
|
||||
assert.equal(picker.getState().blockConfirm, false);
|
||||
picker.choose(12);
|
||||
await tick(5);
|
||||
await complete(1, { ok: false });
|
||||
assert.equal(picker.getState().blockConfirm, true);
|
||||
picker.reset();
|
||||
assert.equal(picker.getState().blockConfirm, false);
|
||||
picker.close();
|
||||
});
|
||||
|
||||
test('replacing a review ignores old frames; stale responses close the current review', async () => {
|
||||
const { picker, requests, complete, isStale } = fixture();
|
||||
picker.open('old', true, 11);
|
||||
await tick(5);
|
||||
picker.close();
|
||||
picker.open('new', true, 22);
|
||||
await tick(5);
|
||||
await complete(0, 11);
|
||||
await tick(5);
|
||||
assert.equal(picker.getState().timestamp, undefined);
|
||||
assert.equal(requests[1]?.request.reviewId, 'new');
|
||||
await complete(1, { ok: false, stale: true });
|
||||
assert.equal(isStale(), true);
|
||||
assert.equal(picker.getState().enabled, false);
|
||||
});
|
||||
|
||||
test('disabled screenshots perform no extraction', async () => {
|
||||
const { picker, requests } = fixture();
|
||||
picker.open('r', false, 11);
|
||||
picker.updateMidpoint(12);
|
||||
await tick(5);
|
||||
assert.equal(requests.length, 0);
|
||||
picker.close();
|
||||
});
|
||||
@@ -0,0 +1,129 @@
|
||||
import type { MediaTimingReviewFrameRequest, MediaTimingReviewFrameResult } from '../../types/anki';
|
||||
|
||||
export interface MediaTimingFramePickerState {
|
||||
enabled: boolean;
|
||||
manual: boolean;
|
||||
loading: boolean;
|
||||
timestamp?: number;
|
||||
requestedTime?: number;
|
||||
dataUrl?: string;
|
||||
message: string;
|
||||
blockConfirm: boolean;
|
||||
}
|
||||
|
||||
/** Coalesces scrubbing into one active extraction and the latest requested frame. */
|
||||
export function createMediaTimingFramePicker(options: {
|
||||
load: (request: MediaTimingReviewFrameRequest) => Promise<MediaTimingReviewFrameResult>;
|
||||
onChange: (state: MediaTimingFramePickerState) => void;
|
||||
onStale: () => void;
|
||||
debounceMs?: number;
|
||||
}) {
|
||||
let reviewId: string | null = null;
|
||||
let midpoint = 0;
|
||||
let sequence = 0;
|
||||
let inFlight = false;
|
||||
let pending: (MediaTimingReviewFrameRequest & { sequence: number }) | null = null;
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
let state: MediaTimingFramePickerState = emptyState();
|
||||
|
||||
function emptyState(): MediaTimingFramePickerState {
|
||||
return { enabled: false, manual: false, loading: false, message: '', blockConfirm: false };
|
||||
}
|
||||
|
||||
function publish(): void {
|
||||
options.onChange({ ...state });
|
||||
}
|
||||
|
||||
async function drain(): Promise<void> {
|
||||
if (inFlight || !pending) return;
|
||||
const request = pending;
|
||||
pending = null;
|
||||
inFlight = true;
|
||||
try {
|
||||
const result = await options.load(request);
|
||||
if (reviewId !== request.reviewId || sequence !== request.sequence) return;
|
||||
if (result.stale) {
|
||||
options.onStale();
|
||||
return;
|
||||
}
|
||||
if (
|
||||
!result.ok ||
|
||||
!Number.isFinite(result.timestamp) ||
|
||||
!result.dataUrl?.startsWith('data:image/jpeg;base64,')
|
||||
) {
|
||||
throw new Error(result.message ?? 'Screenshot preview is unavailable.');
|
||||
}
|
||||
state.timestamp = result.timestamp;
|
||||
state.dataUrl = result.dataUrl;
|
||||
state.blockConfirm = false;
|
||||
state.message = state.manual
|
||||
? 'Selected frame stays fixed when you trim audio.'
|
||||
: 'Following the audio midpoint.';
|
||||
} catch (error) {
|
||||
if (reviewId !== request.reviewId || sequence !== request.sequence) return;
|
||||
state.message = error instanceof Error ? error.message : 'Screenshot preview is unavailable.';
|
||||
state.blockConfirm = state.manual;
|
||||
} finally {
|
||||
inFlight = false;
|
||||
if (reviewId === request.reviewId && sequence === request.sequence) {
|
||||
state.loading = false;
|
||||
publish();
|
||||
}
|
||||
if (timer === null) void drain();
|
||||
}
|
||||
}
|
||||
|
||||
function request(timestamp: number, direction?: -1 | 1): void {
|
||||
if (!reviewId || !state.enabled) return;
|
||||
sequence += 1;
|
||||
pending = { reviewId, timestamp, ...(direction ? { direction } : {}), sequence };
|
||||
state.requestedTime = timestamp;
|
||||
state.loading = true;
|
||||
state.blockConfirm = state.manual;
|
||||
state.message = 'Loading screenshot…';
|
||||
publish();
|
||||
if (timer !== null) clearTimeout(timer);
|
||||
timer = setTimeout(() => {
|
||||
timer = null;
|
||||
void drain();
|
||||
}, options.debounceMs ?? 120);
|
||||
}
|
||||
|
||||
function close(): void {
|
||||
sequence += 1;
|
||||
reviewId = null;
|
||||
pending = null;
|
||||
if (timer !== null) clearTimeout(timer);
|
||||
timer = null;
|
||||
state = emptyState();
|
||||
publish();
|
||||
}
|
||||
|
||||
return {
|
||||
open(id: string, enabled: boolean, time: number) {
|
||||
close();
|
||||
reviewId = id;
|
||||
midpoint = time;
|
||||
state.enabled = enabled;
|
||||
publish();
|
||||
if (enabled) request(time);
|
||||
},
|
||||
updateMidpoint(time: number) {
|
||||
if (midpoint === time) return;
|
||||
midpoint = time;
|
||||
if (!state.manual) request(time);
|
||||
},
|
||||
choose(time: number, direction?: -1 | 1) {
|
||||
if (!Number.isFinite(time) || time < 0) return;
|
||||
state.manual = true;
|
||||
request(time, direction);
|
||||
},
|
||||
reset() {
|
||||
state.manual = false;
|
||||
request(midpoint);
|
||||
},
|
||||
close,
|
||||
getState: () => ({ ...state }),
|
||||
getScreenshotTime: () => (state.manual && !state.blockConfirm ? state.timestamp : undefined),
|
||||
};
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createMediaTimingFramePicker } from './media-timing-frame-picker';
|
||||
import type {
|
||||
MediaTimingReviewContextLine,
|
||||
MediaTimingReviewDecision,
|
||||
@@ -252,6 +253,45 @@ export function createMediaTimingReviewModal(
|
||||
isModalLayer: ctx.platform.isModalLayer,
|
||||
});
|
||||
const previewRequest = createMediaTimingPreviewRequestGuard();
|
||||
const framePicker = createMediaTimingFramePicker({
|
||||
load: (request) => window.electronAPI.getMediaTimingReviewFrame(request),
|
||||
onChange: () => renderFramePicker(),
|
||||
onStale: () => closeResolvedReview(),
|
||||
});
|
||||
|
||||
function renderFramePicker(): void {
|
||||
const state = framePicker.getState();
|
||||
const dom = ctx.dom;
|
||||
dom.mediaTimingReviewFramePicker.classList.toggle('hidden', !state.enabled);
|
||||
dom.mediaTimingReviewFramePicker.setAttribute('aria-busy', String(state.loading));
|
||||
dom.mediaTimingReviewFrameSlider.min = String(timelineStart);
|
||||
dom.mediaTimingReviewFrameSlider.max = String(Math.max(timelineStart, timelineEnd - 0.001));
|
||||
const shownTime = state.loading ? state.requestedTime : state.timestamp;
|
||||
if (shownTime !== undefined) dom.mediaTimingReviewFrameSlider.value = String(shownTime);
|
||||
dom.mediaTimingReviewFrameTime.textContent =
|
||||
shownTime === undefined ? '—' : formatMediaTimingTimestamp(shownTime);
|
||||
dom.mediaTimingReviewFrameSlider.setAttribute(
|
||||
'aria-valuetext',
|
||||
dom.mediaTimingReviewFrameTime.textContent,
|
||||
);
|
||||
dom.mediaTimingReviewFrameStatus.textContent = state.message;
|
||||
dom.mediaTimingReviewFrameImage.classList.toggle('hidden', !state.dataUrl);
|
||||
if (state.dataUrl) dom.mediaTimingReviewFrameImage.src = state.dataUrl;
|
||||
else dom.mediaTimingReviewFrameImage.removeAttribute('src');
|
||||
dom.mediaTimingReviewFramePrevious.disabled =
|
||||
resolveInFlight ||
|
||||
state.loading ||
|
||||
state.timestamp === undefined ||
|
||||
state.timestamp <= timelineStart;
|
||||
dom.mediaTimingReviewFrameNext.disabled =
|
||||
resolveInFlight ||
|
||||
state.loading ||
|
||||
state.timestamp === undefined ||
|
||||
state.timestamp >= timelineEnd - 0.001;
|
||||
dom.mediaTimingReviewFrameSlider.disabled = resolveInFlight;
|
||||
dom.mediaTimingReviewFrameReset.disabled = resolveInFlight || !state.manual;
|
||||
dom.mediaTimingReviewConfirm.disabled = resolveInFlight || state.blockConfirm;
|
||||
}
|
||||
|
||||
function setStatus(message: string, isError = false): void {
|
||||
ctx.dom.mediaTimingReviewStatus.textContent = message;
|
||||
@@ -308,6 +348,8 @@ export function createMediaTimingReviewModal(
|
||||
}
|
||||
|
||||
function renderSelection(): void {
|
||||
framePicker.updateMidpoint(selectionStart + (selectionEnd - selectionStart) / 2);
|
||||
renderFramePicker();
|
||||
const span = Math.max(MINIMUM_CLIP_SECONDS, timelineEnd - timelineStart);
|
||||
const startPercent = ((selectionStart - timelineStart) / span) * 100;
|
||||
const endPercent = ((selectionEnd - timelineStart) / span) * 100;
|
||||
@@ -690,6 +732,7 @@ export function createMediaTimingReviewModal(
|
||||
window.electronAPI.notifyOverlayModalClosed('media-timing-review');
|
||||
options.syncSettingsModalSubtitleSuppression();
|
||||
payload = null;
|
||||
framePicker.close();
|
||||
if (!options.modalStateReader.isAnyModalOpen()) {
|
||||
ctx.dom.overlay.classList.remove('interactive');
|
||||
if (ctx.platform.shouldToggleMouseIgnore) {
|
||||
@@ -701,6 +744,7 @@ export function createMediaTimingReviewModal(
|
||||
async function resolveReview(decision: MediaTimingReviewDecision): Promise<void> {
|
||||
if (!payload || resolveInFlight) return;
|
||||
resolveInFlight = true;
|
||||
renderFramePicker();
|
||||
stopPreview();
|
||||
const controls = ctx.dom.mediaTimingReviewModal.querySelectorAll<HTMLButtonElement>('button');
|
||||
controls.forEach((button) => {
|
||||
@@ -733,12 +777,14 @@ export function createMediaTimingReviewModal(
|
||||
}
|
||||
|
||||
function confirmSelection(): void {
|
||||
if (!payload) return;
|
||||
if (!payload || framePicker.getState().blockConfirm) return;
|
||||
const screenshotTime = framePicker.getScreenshotTime();
|
||||
const includesAdjacentLines = previousCount > 0 || nextCount > 0;
|
||||
void resolveReview({
|
||||
action: 'confirm',
|
||||
startTime: selectionStart,
|
||||
endTime: selectionEnd,
|
||||
...(screenshotTime !== undefined ? { screenshotTime } : {}),
|
||||
...(includesAdjacentLines ? { text: currentLineSelection().sentence } : {}),
|
||||
});
|
||||
}
|
||||
@@ -818,6 +864,11 @@ export function createMediaTimingReviewModal(
|
||||
nextPayload.noteId !== undefined ? 'Delete card' : "Don't create card";
|
||||
setStatus('');
|
||||
showEditor();
|
||||
framePicker.open(
|
||||
nextPayload.reviewId,
|
||||
nextPayload.screenshotEnabled === true,
|
||||
selectionStart + (selectionEnd - selectionStart) / 2,
|
||||
);
|
||||
renderSelection();
|
||||
renderSentence();
|
||||
ctx.state.mediaTimingReviewModalOpen = true;
|
||||
@@ -892,6 +943,28 @@ export function createMediaTimingReviewModal(
|
||||
}
|
||||
|
||||
function wireDomEvents(): void {
|
||||
ctx.dom.mediaTimingReviewFrameSlider.addEventListener('input', () => {
|
||||
if (!resolveInFlight) framePicker.choose(Number(ctx.dom.mediaTimingReviewFrameSlider.value));
|
||||
});
|
||||
const stepFrame = (direction: -1 | 1) => {
|
||||
const state = framePicker.getState();
|
||||
if (!resolveInFlight && !state.loading && state.timestamp !== undefined)
|
||||
framePicker.choose(state.timestamp, direction);
|
||||
};
|
||||
ctx.dom.mediaTimingReviewFrameSlider.addEventListener('keydown', (event) => {
|
||||
if (event.key === 'ArrowLeft' || event.key === 'ArrowDown') {
|
||||
event.preventDefault();
|
||||
stepFrame(-1);
|
||||
} else if (event.key === 'ArrowRight' || event.key === 'ArrowUp') {
|
||||
event.preventDefault();
|
||||
stepFrame(1);
|
||||
}
|
||||
});
|
||||
ctx.dom.mediaTimingReviewFramePrevious.addEventListener('click', () => stepFrame(-1));
|
||||
ctx.dom.mediaTimingReviewFrameNext.addEventListener('click', () => stepFrame(1));
|
||||
ctx.dom.mediaTimingReviewFrameReset.addEventListener('click', () => {
|
||||
if (!resolveInFlight) framePicker.reset();
|
||||
});
|
||||
const track = ctx.dom.mediaTimingReviewSelectionTrack;
|
||||
track.addEventListener('pointerdown', beginDrag);
|
||||
track.addEventListener('pointermove', (event) => {
|
||||
|
||||
+81
-1
@@ -1347,7 +1347,7 @@ body:focus-visible,
|
||||
|
||||
.media-timing-review-content {
|
||||
width: min(720px, calc(100vw - 32px));
|
||||
max-height: min(700px, calc(100vh - 32px));
|
||||
max-height: min(900px, calc(100vh - 32px));
|
||||
overflow: auto;
|
||||
gap: 0;
|
||||
padding: 0;
|
||||
@@ -1425,6 +1425,86 @@ body:focus-visible,
|
||||
padding: 14px 20px 18px;
|
||||
}
|
||||
|
||||
.media-timing-frame-picker {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
|
||||
gap: 16px;
|
||||
align-items: center;
|
||||
margin: 0 0 16px;
|
||||
padding: 12px;
|
||||
border: 1px solid var(--ctp-surface1);
|
||||
border-radius: 10px;
|
||||
background: var(--ctp-mantle);
|
||||
}
|
||||
|
||||
.media-timing-frame-picker.hidden {
|
||||
display: none;
|
||||
}
|
||||
.media-timing-frame-image-shell {
|
||||
aspect-ratio: 16 / 9;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
overflow: hidden;
|
||||
border-radius: 6px;
|
||||
background: var(--ctp-crust);
|
||||
}
|
||||
.media-timing-frame-image-shell img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: contain;
|
||||
}
|
||||
.media-timing-frame-picker[aria-busy='true'] img {
|
||||
opacity: 0.55;
|
||||
}
|
||||
.media-timing-frame-heading {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
font-size: 12px;
|
||||
}
|
||||
.media-timing-frame-heading output {
|
||||
color: var(--ctp-teal);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.media-timing-frame-controls input {
|
||||
width: 100%;
|
||||
margin: 12px 0;
|
||||
accent-color: var(--ctp-teal);
|
||||
}
|
||||
.media-timing-frame-buttons {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
}
|
||||
.media-timing-frame-buttons button {
|
||||
padding: 5px 8px;
|
||||
border: 1px solid var(--ctp-surface2);
|
||||
border-radius: 6px;
|
||||
background: var(--ctp-surface0);
|
||||
color: var(--ctp-text);
|
||||
font-size: 11px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.media-timing-frame-buttons button:disabled {
|
||||
opacity: 0.45;
|
||||
cursor: default;
|
||||
}
|
||||
.media-timing-frame-controls p {
|
||||
margin: 8px 0 0;
|
||||
min-height: 2.6em;
|
||||
color: var(--ctp-subtext0);
|
||||
font-size: 11px;
|
||||
}
|
||||
@media (max-width: 520px) {
|
||||
.media-timing-frame-picker {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
.media-timing-frame-image-shell {
|
||||
max-height: 180px;
|
||||
}
|
||||
}
|
||||
|
||||
.media-timing-review-sentence-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
||||
@@ -46,6 +46,14 @@ export type RendererDom = {
|
||||
youtubePickerStatus: HTMLDivElement;
|
||||
youtubePickerTracks: HTMLUListElement;
|
||||
|
||||
mediaTimingReviewFramePicker: HTMLDivElement;
|
||||
mediaTimingReviewFrameImage: HTMLImageElement;
|
||||
mediaTimingReviewFrameSlider: HTMLInputElement;
|
||||
mediaTimingReviewFrameTime: HTMLElement;
|
||||
mediaTimingReviewFrameStatus: HTMLElement;
|
||||
mediaTimingReviewFramePrevious: HTMLButtonElement;
|
||||
mediaTimingReviewFrameNext: HTMLButtonElement;
|
||||
mediaTimingReviewFrameReset: HTMLButtonElement;
|
||||
mediaTimingReviewModal: HTMLDivElement;
|
||||
mediaTimingReviewKind: HTMLDivElement;
|
||||
mediaTimingReviewText: HTMLElement;
|
||||
@@ -249,6 +257,24 @@ export function resolveRendererDom(): RendererDom {
|
||||
youtubePickerStatus: getRequiredElement<HTMLDivElement>('youtubePickerStatus'),
|
||||
youtubePickerTracks: getRequiredElement<HTMLUListElement>('youtubePickerTracks'),
|
||||
|
||||
mediaTimingReviewFramePicker: getRequiredElement<HTMLDivElement>(
|
||||
'mediaTimingReviewFramePicker',
|
||||
),
|
||||
mediaTimingReviewFrameImage: getRequiredElement<HTMLImageElement>(
|
||||
'mediaTimingReviewFrameImage',
|
||||
),
|
||||
mediaTimingReviewFrameSlider: getRequiredElement<HTMLInputElement>(
|
||||
'mediaTimingReviewFrameSlider',
|
||||
),
|
||||
mediaTimingReviewFrameTime: getRequiredElement<HTMLElement>('mediaTimingReviewFrameTime'),
|
||||
mediaTimingReviewFrameStatus: getRequiredElement<HTMLElement>('mediaTimingReviewFrameStatus'),
|
||||
mediaTimingReviewFramePrevious: getRequiredElement<HTMLButtonElement>(
|
||||
'mediaTimingReviewFramePrevious',
|
||||
),
|
||||
mediaTimingReviewFrameNext: getRequiredElement<HTMLButtonElement>('mediaTimingReviewFrameNext'),
|
||||
mediaTimingReviewFrameReset: getRequiredElement<HTMLButtonElement>(
|
||||
'mediaTimingReviewFrameReset',
|
||||
),
|
||||
mediaTimingReviewModal: getRequiredElement<HTMLDivElement>('mediaTimingReviewModal'),
|
||||
mediaTimingReviewKind: getRequiredElement<HTMLDivElement>('mediaTimingReviewKind'),
|
||||
mediaTimingReviewText: getRequiredElement<HTMLElement>('mediaTimingReviewText'),
|
||||
|
||||
@@ -138,6 +138,7 @@ export const IPC_CHANNELS = {
|
||||
getChangelogSnapshot: 'changelog:get-snapshot',
|
||||
mediaTimingReviewPreview: 'media-timing-review:preview',
|
||||
mediaTimingReviewWaveform: 'media-timing-review:waveform',
|
||||
mediaTimingReviewFrame: 'media-timing-review:frame',
|
||||
mediaTimingReviewStopPreview: 'media-timing-review:stop-preview',
|
||||
mediaTimingReviewResolve: 'media-timing-review:resolve',
|
||||
},
|
||||
|
||||
+22
-1
@@ -21,6 +21,8 @@ export interface MediaTimingReviewRequest {
|
||||
noteId?: number;
|
||||
audioPadding: number;
|
||||
maxMediaDuration: number;
|
||||
/** Still screenshots only; animated images continue to follow the audio range. */
|
||||
screenshotEnabled?: boolean;
|
||||
}
|
||||
|
||||
/** A subtitle line adjacent to the mined one that the review can pull onto the card. */
|
||||
@@ -32,7 +34,13 @@ export interface MediaTimingReviewContextLine {
|
||||
|
||||
export type MediaTimingReviewDecision =
|
||||
/** `text` is set when the review combined adjacent lines into the card sentence. */
|
||||
| { action: 'confirm'; startTime: number; endTime: number; text?: string }
|
||||
| {
|
||||
action: 'confirm';
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
text?: string;
|
||||
screenshotTime?: number;
|
||||
}
|
||||
| { action: 'use-original' }
|
||||
| { action: 'skip-media' }
|
||||
| { action: 'discard' };
|
||||
@@ -53,6 +61,19 @@ export interface MediaTimingReviewOpenPayload {
|
||||
timelineEndTime: number;
|
||||
mediaDuration?: number;
|
||||
maxMediaDuration: number;
|
||||
screenshotEnabled?: boolean;
|
||||
}
|
||||
|
||||
export interface MediaTimingReviewFrameRequest {
|
||||
reviewId: string;
|
||||
timestamp: number;
|
||||
/** Step to the adjacent decoded frame instead of seeking to a time. */
|
||||
direction?: -1 | 1;
|
||||
}
|
||||
|
||||
export interface MediaTimingReviewFrameResult extends MediaTimingReviewActionResult {
|
||||
dataUrl?: string;
|
||||
timestamp?: number;
|
||||
}
|
||||
|
||||
export interface MediaTimingReviewPreviewRequest {
|
||||
|
||||
@@ -8,6 +8,8 @@ import type {
|
||||
MediaTimingReviewOpenPayload,
|
||||
MediaTimingReviewPreviewRequest,
|
||||
MediaTimingReviewResolveRequest,
|
||||
MediaTimingReviewFrameRequest,
|
||||
MediaTimingReviewFrameResult,
|
||||
MediaTimingReviewWaveformRequest,
|
||||
MediaTimingReviewWaveformResult,
|
||||
} from './anki';
|
||||
@@ -550,6 +552,9 @@ export interface ElectronAPI {
|
||||
previewMediaTimingReview: (
|
||||
request: MediaTimingReviewPreviewRequest,
|
||||
) => Promise<MediaTimingReviewActionResult>;
|
||||
getMediaTimingReviewFrame: (
|
||||
request: MediaTimingReviewFrameRequest,
|
||||
) => Promise<MediaTimingReviewFrameResult>;
|
||||
getMediaTimingReviewWaveform: (
|
||||
request: MediaTimingReviewWaveformRequest,
|
||||
) => Promise<MediaTimingReviewWaveformResult>;
|
||||
|
||||
@@ -246,6 +246,8 @@ export interface SubtitleMiningContext {
|
||||
capturedAtMs?: number;
|
||||
/** Explicit generator padding. Confirmed timing-review ranges set this to zero. */
|
||||
mediaPaddingSeconds?: number;
|
||||
/** Independent still screenshot selected during media timing review. */
|
||||
screenshotTime?: number;
|
||||
}
|
||||
|
||||
export interface SubtitleHoverTokenPayload {
|
||||
|
||||
Reference in New Issue
Block a user