feat(anki): add media timing review before card creation (#203)

This commit is contained in:
2026-09-04 01:40:56 -07:00
committed by GitHub
parent 99266294b8
commit 84f718043a
78 changed files with 7022 additions and 135 deletions
+35
View File
@@ -1263,6 +1263,41 @@ test('AnkiIntegration dismisses persistent overlay update progress when no termi
assert.deepEqual(dismissedIds, ['anki-update-progress']);
});
test('AnkiIntegration dismisses overlay update progress after notifications switch to OSD', () => {
const behavior: NonNullable<AnkiConnectConfig['behavior']> = {
notificationType: 'overlay',
};
const dismissedIds: string[] = [];
const integration = new AnkiIntegration(
{ behavior },
{} as never,
{} as never,
undefined,
undefined,
undefined,
undefined,
{},
undefined,
() => {},
undefined,
undefined,
undefined,
(id) => {
dismissedIds.push(id);
},
);
const updateNotifications = integration as unknown as {
beginUpdateProgress: (message: string) => void;
endUpdateProgress: () => void;
};
updateNotifications.beginUpdateProgress('Updating card');
behavior.notificationType = 'osd';
updateNotifications.endUpdateProgress();
assert.deepEqual(dismissedIds, ['anki-update-progress']);
});
test('AnkiIntegration keeps overlay notification image when temp icon write fails', async () => {
const desktopNotifications: Array<{ title: string; body?: string; icon?: string }> = [];
const overlayNotifications: TestOverlayNotificationPayload[] = [];
+42 -6
View File
@@ -28,6 +28,8 @@ import {
KikuMergePreviewResponse,
NotificationOptions,
type WordCardKind,
type MediaTimingReviewDecision,
type MediaTimingReviewRequest,
} from './types/anki';
import { AiConfig } from './types/integrations';
import type { KnownWordMaturityTier } from './types/subtitle';
@@ -240,6 +242,9 @@ export class AnkiIntegration {
private recordCardsMinedCallback: ((count: number, noteIds?: number[]) => void) | null = null;
private knownWordCacheUpdatedCallback: (() => void) | null = null;
private consumeSubtitleMiningContextCallback: (() => SubtitleMiningContext | null) | null = null;
private mediaTimingReviewCallback:
| ((request: MediaTimingReviewRequest) => Promise<MediaTimingReviewDecision>)
| null = null;
private noteIdRedirects = new Map<number, number>();
private trackedDuplicateNoteIds = new Map<number, number[]>();
private getCachedMediaPath: MediaGenerationInputResolverOptions['getCachedMediaPath'] | null =
@@ -511,6 +516,7 @@ export class AnkiIntegration {
findNotes: async (query, options) =>
(await this.client.findNotes(query, options)) as number[],
retrieveMediaFile: (filename) => this.client.retrieveMediaFile(filename),
deleteNotes: (noteIds) => this.client.deleteNotes(noteIds),
},
mediaGenerator: {
generateAudio: (
@@ -568,6 +574,7 @@ export class AnkiIntegration {
getEffectiveSentenceCardConfig: () => this.getEffectiveSentenceCardConfig(),
getFallbackDurationSeconds: () => this.getFallbackDurationSeconds(),
appendKnownWordsFromNoteInfo: (noteInfo) => this.appendKnownWordsFromNoteInfo(noteInfo),
removeKnownWordNote: (noteId) => this.removeKnownWordNote(noteId),
isUpdateInProgress: () => this.updateInProgress,
setUpdateInProgress: (value) => {
this.updateInProgress = value;
@@ -583,6 +590,7 @@ export class AnkiIntegration {
recordCardsMinedCallback: (count, noteIds) => {
this.recordCardsMinedSafely(count, noteIds, 'card creation');
},
reviewMediaTiming: (request) => this.reviewMediaTiming(request),
});
}
@@ -639,12 +647,14 @@ export class AnkiIntegration {
notesInfo: async (noteIds) => (await this.client.notesInfo(noteIds)) as unknown,
updateNoteFields: (noteId, fields) => this.client.updateNoteFields(noteId, fields),
storeMediaFile: (filename, data) => this.client.storeMediaFile(filename, data),
deleteNotes: (noteIds) => this.client.deleteNotes(noteIds),
},
getConfig: () => this.config,
getCurrentSubtitleText: () => this.mpvClient.currentSubText,
getCurrentSubtitleStart: () => this.mpvClient.currentSubStart,
getEffectiveSentenceCardConfig: () => this.getEffectiveSentenceCardConfig(),
appendKnownWordsFromNoteInfo: (noteInfo) => this.appendKnownWordsFromNoteInfo(noteInfo),
removeKnownWordNote: (noteId) => this.removeKnownWordNote(noteId),
extractFields: (fields) => this.extractFields(fields),
findDuplicateNote: (expression, excludeNoteId, noteInfo) =>
this.findDuplicateNote(expression, excludeNoteId, noteInfo),
@@ -680,6 +690,7 @@ export class AnkiIntegration {
logWarn: (...args) => log.warn(args[0] as string, ...args.slice(1)),
logInfo: (...args) => log.info(args[0] as string, ...args.slice(1)),
logError: (...args) => log.error(args[0] as string, ...args.slice(1)),
reviewMediaTiming: (request) => this.reviewMediaTiming(request),
});
}
@@ -799,6 +810,12 @@ export class AnkiIntegration {
}
}
private removeKnownWordNote(noteId: number): void {
if (this.knownWordCache.removeNote(noteId)) {
this.notifyKnownWordCacheUpdated();
}
}
private notifyKnownWordCacheUpdated(): void {
if (!this.knownWordCacheUpdatedCallback) {
return;
@@ -1076,7 +1093,7 @@ export class AnkiIntegration {
videoPath,
startTime,
endTime,
this.config.media?.audioPadding,
context?.mediaPaddingSeconds ?? this.config.media?.audioPadding,
resolveAudioStreamIndexForMediaGeneration(videoPath, this.mpvClient.currentAudioStreamIndex),
this.config.media?.normalizeAudio !== false,
await this.getMpvVolumeScale(),
@@ -1109,7 +1126,7 @@ export class AnkiIntegration {
videoPath,
mediaRange.startTime,
mediaRange.endTime,
this.config.media?.audioPadding,
context?.mediaPaddingSeconds ?? this.config.media?.audioPadding,
{
fps: this.config.media?.animatedFps,
maxWidth: this.config.media?.animatedMaxWidth,
@@ -1257,11 +1274,11 @@ export class AnkiIntegration {
}
private endUpdateProgress(): void {
if (this.overlayUpdateProgressActive) {
this.overlayUpdateProgressActive = false;
this.overlayNotificationDismissCallback?.('anki-update-progress');
}
if (!this.shouldUseOsdNotifications()) {
if (this.overlayUpdateProgressActive) {
this.overlayUpdateProgressActive = false;
this.overlayNotificationDismissCallback?.('anki-update-progress');
}
return;
}
endUpdateProgress(this.uiFeedbackState, (timer) => {
@@ -1761,6 +1778,25 @@ export class AnkiIntegration {
this.consumeSubtitleMiningContextCallback = callback;
}
setMediaTimingReviewCallback(
callback: ((request: MediaTimingReviewRequest) => Promise<MediaTimingReviewDecision>) | null,
): void {
this.mediaTimingReviewCallback = callback;
}
private async reviewMediaTiming(
request: Omit<MediaTimingReviewRequest, 'audioPadding' | 'maxMediaDuration'>,
): Promise<MediaTimingReviewDecision> {
if (this.config.media?.reviewTiming !== true || !this.mediaTimingReviewCallback) {
return { action: 'use-original' };
}
return await this.mediaTimingReviewCallback({
...request,
audioPadding: Math.max(0, this.config.media.audioPadding ?? 0),
maxMediaDuration: Math.max(0, this.config.media.maxMediaDuration ?? 30),
});
}
resolveCurrentNoteId(noteId: number): number {
let resolved = noteId;
const seen = new Set<number>();
@@ -85,6 +85,7 @@ function createManualUpdateService(overrides: Partial<CardCreationDeps> = {}): {
},
findNotes: async () => [42],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
mediaGenerator: {
generateAudio: async () => Buffer.from('audio'),
@@ -128,6 +129,7 @@ function createManualUpdateService(overrides: Partial<CardCreationDeps> = {}): {
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
removeKnownWordNote: () => undefined,
isUpdateInProgress: () => false,
setUpdateInProgress: () => undefined,
trackLastAddedNoteId: () => undefined,
@@ -200,6 +202,7 @@ test('manual clipboard word-card update uses configured fields with Lapis and Ki
storeMediaFile: async () => undefined,
findNotes: async () => [42],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
getEffectiveSentenceCardConfig: () => ({
model: 'Sentence',
@@ -266,6 +269,7 @@ test('audio-card action keeps Lapis and Kiku sentence fields', async () => {
storeMediaFile: async () => undefined,
findNotes: async () => [42],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
getEffectiveSentenceCardConfig: () => ({
model: 'Sentence',
@@ -328,6 +332,7 @@ test('manual clipboard subtitle update marks Kiku word cards as word-and-sentenc
storeMediaFile: async () => undefined,
findNotes: async () => [42],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
getEffectiveSentenceCardConfig: () => ({
model: 'Sentence',
@@ -374,6 +379,7 @@ test('manual clipboard subtitle update uses configured audio when SentenceAudio
},
findNotes: async () => [42],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
});
@@ -462,6 +468,7 @@ test('manual clipboard subtitle update uses resolved mpv stream URLs for remote
},
findNotes: async () => [42],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
mediaGenerator: {
generateAudio: async (path) => {
@@ -510,3 +517,98 @@ test('createSentenceCard relies on Anki progress notification without standalone
assert.deepEqual(progressMessages, ['Creating sentence card']);
assert.deepEqual(statusMessages, []);
});
test('discarding an audio-card timing review deletes the note before evicting its cache entry', async () => {
const events: string[] = [];
const statusMessages: string[] = [];
const { service } = createManualUpdateService({
getMpvClient: () =>
({
currentVideoPath: '/video.mp4',
currentSubText: '字幕',
currentSubStart: 4,
currentSubEnd: 6,
currentTimePos: 5,
}) as never,
client: {
addNote: async () => 0,
addTags: async () => undefined,
notesInfo: async () => [
{
noteId: 42,
fields: { Expression: { value: '単語' } },
},
],
updateNoteFields: async () => undefined,
storeMediaFile: async () => undefined,
findNotes: async () => [42],
retrieveMediaFile: async () => '',
deleteNotes: async (noteIds) => {
events.push(`delete:${noteIds.join(',')}`);
},
},
reviewMediaTiming: async () => ({ action: 'discard' }),
removeKnownWordNote: (noteId) => {
events.push(`cache:${noteId}`);
},
showStatusNotification: (message) => {
statusMessages.push(message);
},
});
await service.markLastCardAsAudioCard();
assert.deepEqual(events, ['delete:42', 'cache:42']);
assert.deepEqual(statusMessages, ['Card deleted.']);
});
test('keeping an audio card without media skips generation and preserves the note', async () => {
let generatedAudio = false;
let deleted = false;
const updates: Array<{ noteId: number; fields: Record<string, string> }> = [];
const { service, storedMedia } = createManualUpdateService({
getMpvClient: () =>
({
currentVideoPath: '/video.mp4',
currentSubText: '字幕',
currentSubStart: 4,
currentSubEnd: 6,
currentTimePos: 5,
}) as never,
client: {
addNote: async () => 0,
addTags: async () => undefined,
notesInfo: async () => [
{
noteId: 42,
fields: { Expression: { value: '単語' }, Sentence: { value: '' } },
},
],
updateNoteFields: async (noteId, fields) => {
updates.push({ noteId, fields });
},
storeMediaFile: async () => undefined,
findNotes: async () => [42],
retrieveMediaFile: async () => '',
deleteNotes: async () => {
deleted = true;
},
},
mediaGenerator: {
generateAudio: async () => {
generatedAudio = true;
return Buffer.from('audio');
},
generateScreenshot: async () => null,
generateAnimatedImage: async () => null,
},
reviewMediaTiming: async () => ({ action: 'skip-media' }),
});
await service.markLastCardAsAudioCard();
assert.equal(generatedAudio, false);
assert.equal(deleted, false);
assert.deepEqual(storedMedia, []);
assert.deepEqual(updates, [{ noteId: 42, fields: { Sentence: '字幕' } }]);
});
@@ -12,6 +12,7 @@ test('sentence card writes generated audio only to sentence audio field', async
const storedMedia: string[] = [];
const requestedProperties: string[] = [];
const audioVolumeScales: Array<number | undefined> = [];
const audioRanges: Array<{ start: number; end: number; padding: number | undefined }> = [];
const deps: CardCreationDeps = {
getConfig: () =>
@@ -73,17 +74,19 @@ test('sentence card writes generated audio only to sentence audio field', async
},
findNotes: async () => [],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
mediaGenerator: {
generateAudio: async (
_path,
_startTime,
_endTime,
_audioPadding,
startTime,
endTime,
audioPadding,
_audioStreamIndex,
_normalizeAudio,
volumeScale,
) => {
audioRanges.push({ start: startTime, end: endTime, padding: audioPadding });
audioVolumeScales.push(volumeScale);
return Buffer.from('audio');
},
@@ -121,17 +124,15 @@ test('sentence card writes generated audio only to sentence audio field', async
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
removeKnownWordNote: () => undefined,
isUpdateInProgress: () => false,
setUpdateInProgress: () => undefined,
trackLastAddedNoteId: () => undefined,
reviewMediaTiming: async () => ({ action: 'confirm', startTime: 11.4, endTime: 14.2 }),
};
const created = await new CardCreationService(deps).createSentenceCard(
'字幕',
12,
14,
'Subtitle',
);
const service = new CardCreationService(deps);
const created = await service.createSentenceCard('字幕', 12, 14, 'Subtitle');
assert.equal(created, true);
assert.deepEqual(addedFields[0], {
@@ -143,7 +144,19 @@ test('sentence card writes generated audio only to sentence audio field', async
assert.equal(storedMedia.length, 1);
assert.deepEqual(requestedProperties, ['volume']);
assert.deepEqual(audioVolumeScales, [0.4 ** 3]);
assert.deepEqual(audioRanges, [{ start: 11.4, end: 14.2, padding: 0 }]);
const mediaUpdate = updatedFields.find((fields) => 'SentenceAudio' in fields);
assert.equal(mediaUpdate?.SentenceAudio, `[sound:${storedMedia[0]}]`);
assert.equal('ExpressionAudio' in mediaUpdate!, false);
deps.reviewMediaTiming = async () => ({ action: 'discard' });
assert.equal(await service.createSentenceCard('作らない', 20, 22), false);
assert.equal(addedFields.length, 1);
deps.reviewMediaTiming = async () => ({ action: 'skip-media' });
assert.equal(await service.createSentenceCard('メディアなし', 30, 32), true);
assert.equal(addedFields.length, 2);
assert.equal(storedMedia.length, 1);
assert.deepEqual(audioRanges, [{ start: 11.4, end: 14.2, padding: 0 }]);
assert.deepEqual(requestedProperties, ['volume']);
});
@@ -42,6 +42,7 @@ test('CardCreationService counts locally created sentence cards', async () => {
storeMediaFile: async () => undefined,
findNotes: async () => [],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
mediaGenerator: {
generateAudio: async () => null,
@@ -73,6 +74,7 @@ test('CardCreationService counts locally created sentence cards', async () => {
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
removeKnownWordNote: () => undefined,
isUpdateInProgress: () => false,
setUpdateInProgress: () => undefined,
trackLastAddedNoteId: () => undefined,
@@ -138,6 +140,7 @@ test('CardCreationService keeps updating after trackLastAddedNoteId throws', asy
storeMediaFile: async () => undefined,
findNotes: async () => [],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
mediaGenerator: {
generateAudio: async () => null,
@@ -171,6 +174,7 @@ test('CardCreationService keeps updating after trackLastAddedNoteId throws', asy
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
removeKnownWordNote: () => undefined,
isUpdateInProgress: () => false,
setUpdateInProgress: () => undefined,
trackLastAddedNoteId: () => {
@@ -236,6 +240,7 @@ test('CardCreationService keeps updating after recordCardsMinedCallback throws',
storeMediaFile: async () => undefined,
findNotes: async () => [],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
mediaGenerator: {
generateAudio: async () => null,
@@ -269,6 +274,7 @@ test('CardCreationService keeps updating after recordCardsMinedCallback throws',
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
removeKnownWordNote: () => undefined,
isUpdateInProgress: () => false,
setUpdateInProgress: () => undefined,
recordCardsMinedCallback: () => {
@@ -345,6 +351,7 @@ test('CardCreationService uses stream-open-filename for remote media generation'
storeMediaFile: async () => undefined,
findNotes: async () => [],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
mediaGenerator: {
generateAudio: async (path) => {
@@ -388,6 +395,7 @@ test('CardCreationService uses stream-open-filename for remote media generation'
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
removeKnownWordNote: () => undefined,
isUpdateInProgress: () => false,
setUpdateInProgress: () => undefined,
trackLastAddedNoteId: () => undefined,
@@ -450,6 +458,7 @@ test('CardCreationService does not use mpv stream indexes for ready cached YouTu
storeMediaFile: async () => undefined,
findNotes: async () => [],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
mediaGenerator: {
generateAudio: async (path, _startTime, _endTime, _padding, audioStreamIndex) => {
@@ -490,6 +499,7 @@ test('CardCreationService does not use mpv stream indexes for ready cached YouTu
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
removeKnownWordNote: () => undefined,
isUpdateInProgress: () => false,
setUpdateInProgress: () => undefined,
trackLastAddedNoteId: () => undefined,
@@ -585,6 +595,7 @@ test('CardCreationService queues YouTube media when required cache is not ready'
storeMediaFile: async () => undefined,
findNotes: async () => [],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
mediaGenerator: {
generateAudio: async () => {
@@ -628,6 +639,7 @@ test('CardCreationService queues YouTube media when required cache is not ready'
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
removeKnownWordNote: () => undefined,
isUpdateInProgress: () => false,
setUpdateInProgress: () => undefined,
trackLastAddedNoteId: () => undefined,
@@ -695,6 +707,7 @@ test('CardCreationService tracks pre-add duplicate note ids for kiku sentence ca
storeMediaFile: async () => undefined,
findNotes: async () => [],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
mediaGenerator: {
generateAudio: async () => null,
@@ -726,6 +739,7 @@ test('CardCreationService tracks pre-add duplicate note ids for kiku sentence ca
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
removeKnownWordNote: () => undefined,
isUpdateInProgress: () => false,
setUpdateInProgress: () => undefined,
trackLastAddedNoteId: () => undefined,
@@ -783,6 +797,7 @@ test('CardCreationService does not track duplicate ids when pre-add lookup retur
storeMediaFile: async () => undefined,
findNotes: async () => [],
retrieveMediaFile: async () => '',
deleteNotes: async () => undefined,
},
mediaGenerator: {
generateAudio: async () => null,
@@ -814,6 +829,7 @@ test('CardCreationService does not track duplicate ids when pre-add lookup retur
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
removeKnownWordNote: () => undefined,
isUpdateInProgress: () => false,
setUpdateInProgress: () => undefined,
trackLastAddedNoteId: () => undefined,
+99 -24
View File
@@ -3,7 +3,13 @@ import {
getConfiguredWordFieldName,
getPreferredWordValueFromExtractedFields,
} from '../anki-field-config';
import { AnkiConnectConfig, type CardKind, type WordCardKind } from '../types/anki';
import {
AnkiConnectConfig,
type CardKind,
type MediaTimingReviewDecision,
type MediaTimingReviewRequest,
type WordCardKind,
} from '../types/anki';
import { createLogger } from '../logger';
import type { MediaInput } from '../media-input';
import { SubtitleTimingTracker } from '../subtitle-timing-tracker';
@@ -55,6 +61,7 @@ interface CardCreationClient {
storeMediaFile(filename: string, data: Buffer): Promise<void>;
findNotes(query: string, options?: { maxRetries?: number }): Promise<number[]>;
retrieveMediaFile(filename: string): Promise<string>;
deleteNotes(noteIds: number[]): Promise<void>;
}
interface CardCreationMediaGenerator {
@@ -137,12 +144,16 @@ interface CardCreationDeps {
};
getFallbackDurationSeconds: () => number;
appendKnownWordsFromNoteInfo: (noteInfo: CardCreationNoteInfo) => void;
removeKnownWordNote: (noteId: number) => void;
isUpdateInProgress: () => boolean;
setUpdateInProgress: (value: boolean) => void;
trackLastAddedNoteId?: (noteId: number) => void;
trackLastAddedDuplicateNoteIds?: (noteId: number, duplicateNoteIds: number[]) => void;
findDuplicateNoteIds?: (expression: string, noteInfo: CardCreationNoteInfo) => Promise<number[]>;
recordCardsMinedCallback?: (count: number, noteIds?: number[]) => void;
reviewMediaTiming?: (
request: Omit<MediaTimingReviewRequest, 'audioPadding' | 'maxMediaDuration'>,
) => Promise<MediaTimingReviewDecision>;
}
export class CardCreationService {
@@ -456,6 +467,30 @@ export class CardCreationService {
this.deps.getConfig(),
);
const timingDecision = this.deps.reviewMediaTiming
? await this.deps.reviewMediaTiming({
kind: 'audio',
text: mpvClient.currentSubText,
startTime,
endTime,
noteId,
})
: ({ action: 'use-original' } as const);
if (timingDecision.action === 'discard') {
await this.deps.client.deleteNotes([noteId]);
this.deps.removeKnownWordNote(noteId);
this.deps.showStatusNotification('Card deleted.');
return;
}
const skipMedia = timingDecision.action === 'skip-media';
const exactReviewedRange = timingDecision.action === 'confirm';
let sentenceText = mpvClient.currentSubText;
if (timingDecision.action === 'confirm') {
startTime = timingDecision.startTime;
endTime = timingDecision.endTime;
sentenceText = timingDecision.text?.trim() || sentenceText;
}
const updatedFields: Record<string, string> = {};
const errors: string[] = [];
let miscInfoFilename: string | null = null;
@@ -465,30 +500,33 @@ export class CardCreationService {
const sentenceCardConfig = this.deps.getEffectiveSentenceCardConfig();
const sentenceField = sentenceCardConfig.sentenceField;
if (sentenceField) {
const processedSentence = this.deps.processSentence(mpvClient.currentSubText, fields);
const processedSentence = this.deps.processSentence(sentenceText, fields);
updatedFields[sentenceField] = processedSentence;
}
const audioFieldName = sentenceCardConfig.audioField;
try {
const audioFilename = this.generateAudioFilename();
const audioBuffer = await this.mediaGenerateAudio(
mpvClient.currentVideoPath,
startTime,
endTime,
);
if (!skipMedia) {
try {
const audioFilename = this.generateAudioFilename();
const audioBuffer = await this.mediaGenerateAudio(
mpvClient.currentVideoPath,
startTime,
endTime,
exactReviewedRange ? 0 : undefined,
);
if (audioBuffer) {
await this.deps.client.storeMediaFile(audioFilename, audioBuffer);
updatedFields[audioFieldName] = `[sound:${audioFilename}]`;
miscInfoFilename = audioFilename;
if (audioBuffer) {
await this.deps.client.storeMediaFile(audioFilename, audioBuffer);
updatedFields[audioFieldName] = `[sound:${audioFilename}]`;
miscInfoFilename = audioFilename;
}
} catch (error) {
log.error('Failed to generate audio for audio card:', (error as Error).message);
errors.push('audio');
}
} catch (error) {
log.error('Failed to generate audio for audio card:', (error as Error).message);
errors.push('audio');
}
if (shouldGenerateImage(this.deps.getConfig())) {
if (!skipMedia && shouldGenerateImage(this.deps.getConfig())) {
try {
const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo);
const imageFilename = this.generateImageFilename();
@@ -497,6 +535,7 @@ export class CardCreationService {
startTime,
endTime,
animatedLeadInSeconds,
exactReviewedRange,
);
const imageField = this.deps.getConfig().fields?.image;
@@ -569,9 +608,29 @@ export class CardCreationService {
try {
return await this.deps.withUpdateProgress('Creating sentence card', async () => {
const timingDecision = this.deps.reviewMediaTiming
? await this.deps.reviewMediaTiming({
kind: 'sentence',
text: sentence,
startTime,
endTime,
})
: ({ action: 'use-original' } as const);
if (timingDecision.action === 'discard') {
this.deps.showStatusNotification('Card creation cancelled.');
return false;
}
const skipMedia = timingDecision.action === 'skip-media';
const exactReviewedRange = timingDecision.action === 'confirm';
if (timingDecision.action === 'confirm') {
startTime = timingDecision.startTime;
endTime = timingDecision.endTime;
sentence = timingDecision.text?.trim() || sentence;
}
const config = this.deps.getConfig();
const generateAudio = shouldGenerateAudio(config);
const generateImage = shouldGenerateImage(config);
const generateAudio = !skipMedia && shouldGenerateAudio(config);
const generateImage = !skipMedia && shouldGenerateImage(config);
const mediaResolverOptions = this.getMediaResolverOptions();
const videoPath = generateImage
? await resolveMediaGenerationInput(mpvClient, 'video', mediaResolverOptions)
@@ -736,6 +795,7 @@ export class CardCreationService {
generateAudio,
generateImage,
volumeScale,
...(exactReviewedRange ? { mediaPaddingSeconds: 0 } : {}),
});
await this.deps.showNotification(noteId, label, 'media queued');
return true;
@@ -751,7 +811,12 @@ export class CardCreationService {
try {
const audioFilename = this.generateAudioFilename();
const audioBuffer = audioSourcePath
? await this.mediaGenerateAudio(audioSourcePath, startTime, endTime)
? await this.mediaGenerateAudio(
audioSourcePath,
startTime,
endTime,
exactReviewedRange ? 0 : undefined,
)
: null;
if (audioBuffer) {
@@ -769,7 +834,13 @@ export class CardCreationService {
if (generateImage) {
try {
const imageFilename = this.generateImageFilename();
const imageBuffer = await this.generateImageBuffer(videoPath!, startTime, endTime);
const imageBuffer = await this.generateImageBuffer(
videoPath!,
startTime,
endTime,
0,
exactReviewedRange,
);
const imageField = config.fields?.image;
if (imageBuffer && imageField) {
@@ -821,6 +892,7 @@ export class CardCreationService {
videoPath: MediaInput,
startTime: number,
endTime: number,
audioPaddingOverride?: number,
): Promise<Buffer | null> {
const mpvClient = this.deps.getMpvClient();
if (!mpvClient) {
@@ -831,7 +903,7 @@ export class CardCreationService {
videoPath,
startTime,
endTime,
this.deps.getConfig().media?.audioPadding,
audioPaddingOverride ?? this.deps.getConfig().media?.audioPadding,
resolveAudioStreamIndexForMediaGeneration(
videoPath,
mpvClient.currentAudioStreamIndex ?? undefined,
@@ -849,13 +921,16 @@ export class CardCreationService {
startTime: number,
endTime: number,
animatedLeadInSeconds = 0,
exactReviewedRange = false,
): Promise<Buffer | null> {
const mpvClient = this.deps.getMpvClient();
if (!mpvClient) {
return null;
}
const timestamp = mpvClient.currentTimePos || 0;
const timestamp = exactReviewedRange
? startTime + (endTime - startTime) / 2
: mpvClient.currentTimePos || 0;
if (this.deps.getConfig().media?.imageType === 'avif') {
let imageStart = startTime;
@@ -871,7 +946,7 @@ export class CardCreationService {
videoPath,
imageStart,
imageEnd,
this.deps.getConfig().media?.audioPadding,
exactReviewedRange ? 0 : this.deps.getConfig().media?.audioPadding,
{
fps: this.deps.getConfig().media?.animatedFps,
maxWidth: this.deps.getConfig().media?.animatedMaxWidth,
@@ -261,6 +261,32 @@ test('KnownWordCacheManager invalidates persisted cache when fields.word changes
}
});
test('KnownWordCacheManager removes a deleted note from memory and persisted state', () => {
const config: AnkiConnectConfig = {
fields: { word: 'Word' },
knownWords: { highlightEnabled: true },
};
const { manager, statePath, cleanup } = createKnownWordCacheHarness(config);
try {
manager.appendFromNoteInfo({
noteId: 42,
fields: { Word: { value: '猫' } },
});
assert.equal(manager.removeNote(42), true);
assert.equal(manager.removeNote(42), false);
assert.equal(manager.isKnownWord('猫'), false);
const persisted = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as {
notes?: Record<string, unknown>;
};
assert.deepEqual(persisted.notes, {});
} finally {
cleanup();
}
});
test('KnownWordCacheManager refresh incrementally reconciles deleted and edited note words', async () => {
const config: AnkiConnectConfig = {
fields: {
+11
View File
@@ -350,6 +350,17 @@ export class KnownWordCacheManager {
return true;
}
removeNote(noteId: number): boolean {
if (!this.noteEntriesById.has(noteId)) {
return false;
}
this.removeNoteSnapshot(noteId);
this.persistKnownWordCacheState();
log.info('Known-word cache removed deleted note', `noteId=${noteId}`);
return true;
}
clearKnownWordCacheState(): void {
this.clearInMemoryState();
this.knownWordsStateKey = this.getKnownWordCacheStateKey();
@@ -44,6 +44,7 @@ function createWorkflowHarness() {
updates.push({ noteId, fields });
},
storeMediaFile: async () => undefined,
deleteNotes: async () => undefined,
},
getConfig: () => ({
fields: {
@@ -61,6 +62,7 @@ function createWorkflowHarness() {
fieldGroupingMode: 'disabled' as const,
}),
appendKnownWordsFromNoteInfo: (_noteInfo: NoteUpdateWorkflowNoteInfo) => undefined,
removeKnownWordNote: (_noteId: number) => undefined,
extractFields: (fields: Record<string, { value: string }>) => {
const out: Record<string, string> = {};
for (const [key, value] of Object.entries(fields)) {
@@ -634,3 +636,141 @@ test('NoteUpdateWorkflow queues media updates when YouTube cache is pending', as
assert.equal(queuedUpdates[0]?.context, undefined);
assert.deepEqual(harness.updates, [{ noteId: 42, fields: { Sentence: 'subtitle-text' } }]);
});
test('NoteUpdateWorkflow deletes an existing word card when timing review discards it', async () => {
const harness = createWorkflowHarness();
const deletedNoteIds: number[][] = [];
const removedKnownWordNoteIds: number[] = [];
let appendedKnownWords = false;
harness.deps.captureSubtitleMediaContext = () => ({
source: 'overlay',
text: 'subtitle-text',
startTime: 4,
endTime: 6,
});
harness.deps.client.deleteNotes = async (noteIds) => {
deletedNoteIds.push(noteIds);
};
harness.deps.appendKnownWordsFromNoteInfo = () => {
appendedKnownWords = true;
};
harness.deps.removeKnownWordNote = (noteId) => {
removedKnownWordNoteIds.push(noteId);
};
harness.deps.reviewMediaTiming = async () => ({ action: 'discard' });
await harness.workflow.execute(42);
assert.deepEqual(deletedNoteIds, [[42]]);
assert.deepEqual(removedKnownWordNoteIds, [42]);
assert.equal(appendedKnownWords, false);
assert.deepEqual(harness.updates, []);
assert.deepEqual(harness.notifications, []);
});
test('NoteUpdateWorkflow keeps the word card but skips media after timing review', async () => {
const harness = createWorkflowHarness();
const mediaCalls: string[] = [];
const deletedNoteIds: number[][] = [];
const queuedUpdates: unknown[] = [];
harness.deps.captureSubtitleMediaContext = () => ({
source: 'overlay',
text: 'subtitle-text',
startTime: 4,
endTime: 6,
});
harness.deps.getConfig = () => ({
fields: { sentence: 'Sentence', image: 'Picture' },
media: { generateAudio: true, generateImage: true },
behavior: {},
});
harness.deps.reviewMediaTiming = async () => ({ action: 'skip-media' });
harness.deps.generateAudio = async () => {
mediaCalls.push('audio');
return Buffer.from('audio');
};
harness.deps.generateImage = async () => {
mediaCalls.push('image');
return Buffer.from('image');
};
harness.deps.queuePendingYoutubeMediaUpdate = async (update) => {
queuedUpdates.push(update);
return true;
};
harness.deps.client.deleteNotes = async (noteIds) => {
deletedNoteIds.push(noteIds);
};
await harness.workflow.execute(42);
assert.deepEqual(mediaCalls, []);
assert.deepEqual(queuedUpdates, []);
assert.deepEqual(deletedNoteIds, []);
assert.deepEqual(harness.updates, [{ noteId: 42, fields: { Sentence: 'subtitle-text' } }]);
assert.deepEqual(harness.notifications, [{ noteId: 42, label: 'taberu' }]);
});
test('NoteUpdateWorkflow uses the combined review sentence for the card and media range', async () => {
const harness = createWorkflowHarness();
const audioContexts: Array<SubtitleMiningContext | undefined> = [];
harness.deps.captureSubtitleMediaContext = () => ({
source: 'overlay',
text: 'current-line',
startTime: 4,
endTime: 6,
});
harness.deps.getConfig = () => ({
fields: { sentence: 'Sentence' },
media: { generateAudio: true, generateImage: false },
behavior: {},
});
harness.deps.reviewMediaTiming = async () => ({
action: 'confirm',
startTime: 2,
endTime: 7,
text: 'previous-line current-line next-line',
});
harness.deps.generateAudio = async (context) => {
audioContexts.push(context);
return null;
};
await harness.workflow.execute(42);
assert.deepEqual(harness.updates, [
{ noteId: 42, fields: { Sentence: 'previous-line current-line next-line' } },
]);
assert.equal(audioContexts.length, 1);
assert.equal(audioContexts[0]?.text, 'previous-line current-line next-line');
assert.equal(audioContexts[0]?.startTime, 2);
assert.equal(audioContexts[0]?.endTime, 7);
assert.equal(audioContexts[0]?.mediaPaddingSeconds, 0);
});
test('NoteUpdateWorkflow keeps cache unchanged and reports when deletion fails', async () => {
const harness = createWorkflowHarness();
const statusMessages: string[] = [];
let removedKnownWord = false;
harness.deps.captureSubtitleMediaContext = () => ({
source: 'overlay',
text: 'subtitle-text',
startTime: 4,
endTime: 6,
});
harness.deps.client.deleteNotes = async () => {
throw new Error('delete failed');
};
harness.deps.removeKnownWordNote = () => {
removedKnownWord = true;
};
harness.deps.showOsdNotification = (message) => {
statusMessages.push(message);
};
harness.deps.reviewMediaTiming = async () => ({ action: 'discard' });
await harness.workflow.execute(42);
assert.equal(removedKnownWord, false);
assert.deepEqual(statusMessages, ['Card deletion failed: delete failed']);
assert.ok(harness.warnings.length === 0);
});
+57 -6
View File
@@ -1,7 +1,12 @@
import { DEFAULT_ANKI_CONNECT_CONFIG } from '../config';
import { getPreferredWordValueFromExtractedFields } from '../anki-field-config';
import type { SubtitleMiningContext } from '../types/subtitle';
import type { CardKind, WordCardKind } from '../types/anki';
import type {
CardKind,
MediaTimingReviewDecision,
MediaTimingReviewRequest,
WordCardKind,
} from '../types/anki';
import { resolveWordCardKind } from './note-field-utils';
export interface NoteUpdateWorkflowNoteInfo {
@@ -14,6 +19,7 @@ export interface NoteUpdateWorkflowDeps {
notesInfo(noteIds: number[]): Promise<unknown>;
updateNoteFields(noteId: number, fields: Record<string, string>): Promise<void>;
storeMediaFile(filename: string, data: Buffer): Promise<void>;
deleteNotes(noteIds: number[]): Promise<void>;
};
getConfig: () => {
fields?: {
@@ -44,6 +50,7 @@ export interface NoteUpdateWorkflowDeps {
wordCardKind?: WordCardKind;
};
appendKnownWordsFromNoteInfo: (noteInfo: NoteUpdateWorkflowNoteInfo) => void;
removeKnownWordNote: (noteId: number) => void;
extractFields: (fields: Record<string, { value: string }>) => Record<string, string>;
findDuplicateNote: (
expression: string,
@@ -102,6 +109,9 @@ export interface NoteUpdateWorkflowDeps {
logWarn: (message: string, ...args: unknown[]) => void;
logInfo: (message: string, ...args: unknown[]) => void;
logError: (message: string, ...args: unknown[]) => void;
reviewMediaTiming?: (
request: Omit<MediaTimingReviewRequest, 'audioPadding' | 'maxMediaDuration'>,
) => Promise<MediaTimingReviewDecision>;
}
function normalizeSubtitleContextText(text: string): string {
@@ -171,7 +181,6 @@ export class NoteUpdateWorkflow {
}
const noteInfo = notesInfo[0]!;
this.deps.appendKnownWordsFromNoteInfo(noteInfo);
const fields = this.deps.extractFields(noteInfo.fields);
const config = this.deps.getConfig();
@@ -207,11 +216,53 @@ export class NoteUpdateWorkflow {
// Audio and image generation run sequentially and audio extraction can take tens of
// seconds, so resolve the clip range exactly once up front; reading live mpv sub
// timings per generator clips whichever line is on screen when each one starts.
const mediaTimingContext =
let mediaTimingContext =
subtitleMiningContext ?? this.deps.captureSubtitleMediaContext?.() ?? null;
let skipMedia = false;
let reviewedSentenceText: string | undefined;
const noteLabel = hasExpressionText ? expressionText : noteId;
const currentSubtitleText = subtitleMiningContext?.text ?? this.deps.getCurrentSubtitleText();
if (mediaTimingContext) {
const timingDecision = this.deps.reviewMediaTiming
? await this.deps.reviewMediaTiming({
kind: 'word',
text: mediaTimingContext.text,
startTime: mediaTimingContext.startTime,
endTime: mediaTimingContext.endTime,
noteId,
})
: ({ action: 'use-original' } as const);
if (timingDecision.action === 'discard') {
try {
await this.deps.client.deleteNotes([noteId]);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
this.deps.logError('Failed to delete discarded card:', message);
this.deps.showOsdNotification(`Card deletion failed: ${message}`);
return;
}
this.deps.removeKnownWordNote(noteId);
this.deps.showOsdNotification('Card deleted.');
return;
}
if (timingDecision.action === 'confirm') {
reviewedSentenceText = timingDecision.text?.trim() || undefined;
mediaTimingContext = {
...mediaTimingContext,
...(reviewedSentenceText !== undefined ? { text: reviewedSentenceText } : {}),
startTime: timingDecision.startTime,
endTime: timingDecision.endTime,
mediaPaddingSeconds: 0,
};
} else if (timingDecision.action === 'skip-media') {
skipMedia = true;
}
}
this.deps.appendKnownWordsFromNoteInfo(noteInfo);
const currentSubtitleText =
reviewedSentenceText ?? subtitleMiningContext?.text ?? this.deps.getCurrentSubtitleText();
if (sentenceField && currentSubtitleText) {
const processedSentence = this.deps.processSentence(currentSubtitleText, fields);
updatedFields[sentenceField] = processedSentence;
@@ -239,8 +290,8 @@ export class NoteUpdateWorkflow {
}
}
const generateAudio = config.media?.generateAudio !== false;
const generateImage = config.media?.generateImage !== false;
const generateAudio = !skipMedia && config.media?.generateAudio !== false;
const generateImage = !skipMedia && config.media?.generateImage !== false;
const mediaCacheQueued =
(generateAudio || generateImage) && this.deps.queuePendingYoutubeMediaUpdate
? await this.deps.queuePendingYoutubeMediaUpdate({
@@ -147,6 +147,9 @@ export class PendingYoutubeMediaQueue {
generateAudio: shouldGenerateAudio(config),
generateImage: shouldGenerateImage(config),
volumeScale,
...(job.context?.mediaPaddingSeconds !== undefined
? { mediaPaddingSeconds: job.context.mediaPaddingSeconds }
: {}),
});
return true;
}
@@ -282,7 +285,7 @@ export class PendingYoutubeMediaQueue {
cachedMediaInput,
job.startTime,
job.endTime,
config.media?.audioPadding,
job.mediaPaddingSeconds ?? config.media?.audioPadding,
undefined,
config.media?.normalizeAudio !== false,
job.volumeScale,
@@ -316,6 +319,7 @@ export class PendingYoutubeMediaQueue {
job.startTime,
job.endTime,
animatedLeadInSeconds,
job.mediaPaddingSeconds,
);
if (imageBuffer) {
await this.deps.client.storeMediaFile(imageFilename, imageBuffer);
@@ -376,6 +380,7 @@ export class PendingYoutubeMediaQueue {
startTime: number,
endTime: number,
animatedLeadInSeconds = 0,
mediaPaddingSeconds?: number,
): Promise<Buffer | null> {
const config = this.deps.getConfig();
if (config.media?.imageType === 'avif') {
@@ -383,7 +388,7 @@ export class PendingYoutubeMediaQueue {
videoPath,
startTime,
endTime,
config.media?.audioPadding,
mediaPaddingSeconds ?? config.media?.audioPadding,
{
fps: config.media?.animatedFps,
maxWidth: config.media?.animatedMaxWidth,
@@ -10,6 +10,7 @@ export interface PendingYoutubeMediaUpdate {
generateAudio: boolean;
generateImage: boolean;
volumeScale?: number;
mediaPaddingSeconds?: number;
}
function trimToNonEmptyString(value: unknown): string | null {
+1
View File
@@ -2181,6 +2181,7 @@ test('runtime options registry is centralized', () => {
const ids = RUNTIME_OPTION_REGISTRY.map((entry) => entry.id);
assert.deepEqual(ids, [
'anki.autoUpdateNewCards',
'anki.mediaReviewTiming',
'subtitle.annotation.knownWords.highlightEnabled',
'subtitle.annotation.knownWords.maturityEnabled',
'subtitle.annotation.nPlusOne',
@@ -54,6 +54,7 @@ export const INTEGRATIONS_DEFAULT_CONFIG: Pick<
syncAnimatedImageToWordAudio: true,
normalizeAudio: true,
mirrorMpvVolume: true,
reviewTiming: false,
audioPadding: 0,
fallbackDuration: 3.0,
maxMediaDuration: 30,
@@ -196,6 +196,14 @@ export function buildIntegrationConfigOptionRegistry(
description:
"Apply mpv's current software volume curve to generated sentence audio. Changes apply live.",
},
{
path: 'ankiConnect.media.reviewTiming',
kind: 'boolean',
defaultValue: defaultConfig.ankiConnect.media.reviewTiming,
description:
'Review and preview subtitle media timing before SubMiner creates or enriches a mined card.',
runtime: runtimeOptionById.get('anki.mediaReviewTiming'),
},
{
path: 'ankiConnect.media.generateImage',
kind: 'boolean',
+14
View File
@@ -19,6 +19,20 @@ export function buildRuntimeOptionRegistry(
behavior: { autoUpdateNewCards: value === true },
}),
},
{
id: 'anki.mediaReviewTiming',
path: 'ankiConnect.media.reviewTiming',
label: 'Review Media Timing',
scope: 'ankiConnect',
valueType: 'boolean',
allowedValues: [true, false],
defaultValue: defaultConfig.ankiConnect.media.reviewTiming,
requiresRestart: false,
formatValueForOsd: (value) => (value === true ? 'On' : 'Off'),
toAnkiPatch: (value) => ({
media: { reviewTiming: value === true },
}),
},
{
id: 'subtitle.annotation.knownWords.highlightEnabled',
path: 'ankiConnect.knownWords.highlightEnabled',
+1 -1
View File
@@ -135,7 +135,7 @@ const INTEGRATION_TEMPLATE_SECTIONS: ConfigTemplateSection[] = [
title: 'AnkiConnect Integration',
description: ['Automatic Anki updates and media generation options.'],
notes: [
'Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, isSenren.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running.',
'Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume/reviewTiming, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, isSenren.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running.',
'Shared AI provider transport settings are read from top-level ai and typically require restart.',
'Most other AnkiConnect settings still require restart.',
],
+28
View File
@@ -21,6 +21,34 @@ function makeContext(ankiConnect: unknown): {
return { context, warnings };
}
test('media timing review is disabled by default and accepts a boolean override', () => {
const defaultContext = makeContext({});
applyAnkiConnectResolution(defaultContext.context);
assert.equal(defaultContext.context.resolved.ankiConnect.media.reviewTiming, false);
const enabledContext = makeContext({ media: { reviewTiming: true } });
applyAnkiConnectResolution(enabledContext.context);
assert.equal(enabledContext.context.resolved.ankiConnect.media.reviewTiming, true);
assert.deepEqual(enabledContext.warnings, []);
});
test('modern media duration accepts zero as the disabled cap sentinel', () => {
const disabledCap = makeContext({ media: { maxMediaDuration: 0 } });
applyAnkiConnectResolution(disabledCap.context);
assert.equal(disabledCap.context.resolved.ankiConnect.media.maxMediaDuration, 0);
assert.deepEqual(disabledCap.warnings, []);
const invalidCap = makeContext({ media: { maxMediaDuration: -1 } });
applyAnkiConnectResolution(invalidCap.context);
assert.equal(
invalidCap.context.resolved.ankiConnect.media.maxMediaDuration,
DEFAULT_CONFIG.ankiConnect.media.maxMediaDuration,
);
assert.ok(
invalidCap.warnings.some((warning) => warning.path === 'ankiConnect.media.maxMediaDuration'),
);
});
test('modern invalid knownWords.highlightEnabled warns modern key and does not fallback to legacy', () => {
const { context, warnings } = makeContext({
nPlusOne: { highlightEnabled: true },
+25 -14
View File
@@ -19,6 +19,7 @@ export function applyModernMediaResolution(
'syncAnimatedImageToWordAudio',
'normalizeAudio',
'mirrorMpvVolume',
'reviewTiming',
] as const) {
applyModernValue(
context,
@@ -128,18 +129,28 @@ export function applyModernMediaResolution(
'Expected non-negative number.',
);
for (const key of ['fallbackDuration', 'maxMediaDuration'] as const) {
applyModernValue(
context,
media,
key,
`ankiConnect.media.${key}`,
asPositiveNumber,
DEFAULT_CONFIG.ankiConnect.media[key],
(value) => {
context.resolved.ankiConnect.media[key] = value;
},
'Expected positive number.',
);
}
applyModernValue(
context,
media,
'fallbackDuration',
'ankiConnect.media.fallbackDuration',
asPositiveNumber,
DEFAULT_CONFIG.ankiConnect.media.fallbackDuration,
(value) => {
context.resolved.ankiConnect.media.fallbackDuration = value;
},
'Expected positive number.',
);
applyModernValue(
context,
media,
'maxMediaDuration',
'ankiConnect.media.maxMediaDuration',
asNonNegativeNumber,
DEFAULT_CONFIG.ankiConnect.media.maxMediaDuration,
(value) => {
context.resolved.ankiConnect.media.maxMediaDuration = value;
},
'Expected non-negative number.',
);
}
+1
View File
@@ -354,6 +354,7 @@ test('settings registry marks safe live config paths as hot-reloadable', () => {
'ankiConnect.deck',
'ankiConnect.media.normalizeAudio',
'ankiConnect.media.mirrorMpvVolume',
'ankiConnect.media.reviewTiming',
'ankiConnect.knownWords.highlightEnabled',
'ankiConnect.knownWords.refreshMinutes',
'ankiConnect.knownWords.addMinedWordsImmediately',
+2
View File
@@ -246,6 +246,7 @@ const LABEL_OVERRIDES: Record<string, string> = {
'mpv.aniskipEnabled': 'Enable AniSkip',
'mpv.aniskipButtonKey': 'AniSkip Button Key',
'ankiConnect.media.mirrorMpvVolume': 'Mirror mpv Volume',
'ankiConnect.media.reviewTiming': 'Review Media Timing',
'discordPresence.updateIntervalMs': 'Update Interval (ms)',
};
@@ -699,6 +700,7 @@ function restartBehaviorForPath(path: string): ConfigSettingsRestartBehavior {
path === 'ankiConnect.ai.enabled' ||
path === 'ankiConnect.media.normalizeAudio' ||
path === 'ankiConnect.media.mirrorMpvVolume' ||
path === 'ankiConnect.media.reviewTiming' ||
path === 'ankiConnect.behavior.autoUpdateNewCards' ||
path === 'ankiConnect.knownWords.highlightEnabled' ||
path === 'ankiConnect.knownWords.refreshMinutes' ||
@@ -33,6 +33,7 @@ test('classifyConfigHotReloadDiff treats safe nested config paths as hot-reloada
next.ankiConnect.deck = 'Mining';
next.ankiConnect.media.normalizeAudio = !prev.ankiConnect.media.normalizeAudio;
next.ankiConnect.media.mirrorMpvVolume = !prev.ankiConnect.media.mirrorMpvVolume;
next.ankiConnect.media.reviewTiming = !prev.ankiConnect.media.reviewTiming;
next.ankiConnect.behavior.autoUpdateNewCards = !prev.ankiConnect.behavior.autoUpdateNewCards;
next.ankiConnect.knownWords.highlightEnabled = !prev.ankiConnect.knownWords.highlightEnabled;
next.ankiConnect.knownWords.refreshMinutes = prev.ankiConnect.knownWords.refreshMinutes + 5;
@@ -69,6 +70,7 @@ test('classifyConfigHotReloadDiff treats safe nested config paths as hot-reloada
'ankiConnect.deck',
'ankiConnect.media.normalizeAudio',
'ankiConnect.media.mirrorMpvVolume',
'ankiConnect.media.reviewTiming',
'ankiConnect.behavior.autoUpdateNewCards',
'ankiConnect.knownWords.highlightEnabled',
'ankiConnect.knownWords.refreshMinutes',
+1
View File
@@ -70,6 +70,7 @@ const HOT_RELOAD_EXACT_OR_PREFIX_PATHS = [
'ankiConnect.deck',
'ankiConnect.media.normalizeAudio',
'ankiConnect.media.mirrorMpvVolume',
'ankiConnect.media.reviewTiming',
'ankiConnect.behavior.autoUpdateNewCards',
'ankiConnect.knownWords.highlightEnabled',
'ankiConnect.knownWords.refreshMinutes',
@@ -306,9 +306,11 @@ test('vocabulary charts use complete top-word and lexical rollup data', () => {
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, '', 1700000000, 1700000000, ?)`,
);
db.exec('BEGIN');
for (let index = 0; index < 501; index += 1) {
insertWord.run(`${index}`, `${index}`, index === 500 ? 10_000 : 1);
}
db.exec('COMMIT');
const charts = getVocabularyChartData(db);
+77
View File
@@ -648,6 +648,83 @@ test('registerIpcHandlers exposes playback window activation request', async ()
assert.deepEqual(calls, ['activate']);
});
test('registerIpcHandlers accepts the keep-without-media timing decision', async () => {
const { registrar, handlers } = createFakeIpcRegistrar();
const requests: unknown[] = [];
registerIpcHandlers(
createRegisterIpcDeps({
resolveMediaTimingReview: async (request) => {
requests.push(request);
return { ok: true };
},
}),
registrar,
);
const handler = handlers.handle.get(IPC_CHANNELS.request.mediaTimingReviewResolve);
assert.ok(handler);
assert.deepEqual(
await handler!({}, { reviewId: 'review-1', decision: { action: 'skip-media' } }),
{ ok: true },
);
assert.deepEqual(requests, [{ reviewId: 'review-1', decision: { action: 'skip-media' } }]);
});
test('registerIpcHandlers validates and forwards combined timing review text', async () => {
const { registrar, handlers } = createFakeIpcRegistrar();
const requests: unknown[] = [];
registerIpcHandlers(
createRegisterIpcDeps({
resolveMediaTimingReview: async (request) => {
requests.push(request);
return { ok: true };
},
}),
registrar,
);
const handler = handlers.handle.get(IPC_CHANNELS.request.mediaTimingReviewResolve);
assert.ok(handler);
assert.deepEqual(
await handler!(
{},
{
reviewId: 'review-1',
decision: {
action: 'confirm',
startTime: 10,
endTime: 12,
text: '前の行 対象の行',
},
},
),
{ ok: true },
);
assert.deepEqual(requests, [
{
reviewId: 'review-1',
decision: {
action: 'confirm',
startTime: 10,
endTime: 12,
text: '前の行 対象の行',
},
},
]);
assert.deepEqual(
await handler!(
{},
{
reviewId: 'review-1',
decision: { action: 'confirm', startTime: 10, endTime: 12, text: ' ' },
},
),
{ ok: false, message: 'Timing review is unavailable.' },
);
assert.equal(requests.length, 1);
});
test('registerIpcHandlers forwards yomitan lookup tracking commands to immersion tracker', () => {
const { registrar, handlers } = createFakeIpcRegistrar();
const calls: string[] = [];
+131
View File
@@ -19,6 +19,13 @@ import type {
YoutubePickerResolveRequest,
YoutubePickerResolveResult,
} from '../../types';
import type {
MediaTimingReviewActionResult,
MediaTimingReviewPreviewRequest,
MediaTimingReviewResolveRequest,
MediaTimingReviewWaveformRequest,
MediaTimingReviewWaveformResult,
} from '../../types/anki';
import { IPC_CHANNELS, type OverlayHostedModal } from '../../shared/ipc/contracts';
import {
parseMpvCommand,
@@ -99,6 +106,16 @@ export interface IpcServiceDeps {
onYoutubePickerResolve: (
request: YoutubePickerResolveRequest,
) => Promise<YoutubePickerResolveResult>;
previewMediaTimingReview?: (
request: MediaTimingReviewPreviewRequest,
) => Promise<MediaTimingReviewActionResult>;
getMediaTimingReviewWaveform?: (
request: MediaTimingReviewWaveformRequest,
) => Promise<MediaTimingReviewWaveformResult>;
stopMediaTimingReviewPreview?: (reviewId: string) => Promise<MediaTimingReviewActionResult>;
resolveMediaTimingReview?: (
request: MediaTimingReviewResolveRequest,
) => MediaTimingReviewActionResult | Promise<MediaTimingReviewActionResult>;
getAnkiConnectStatus: () => boolean;
getRuntimeOptions: () => unknown;
setRuntimeOption: (id: RuntimeOptionId, value: RuntimeOptionValue) => unknown;
@@ -222,6 +239,72 @@ function parseOverlayNotificationActionPayload(
return { notificationId, actionId, ...(typeof noteId === 'number' ? { noteId } : {}) };
}
function parseMediaTimingReviewPreviewRequest(
payload: unknown,
): MediaTimingReviewPreviewRequest | null {
if (!payload || typeof payload !== 'object') return null;
const record = payload as Record<string, unknown>;
if (
typeof record.reviewId !== 'string' ||
!record.reviewId ||
typeof record.startTime !== 'number' ||
!Number.isFinite(record.startTime) ||
typeof record.endTime !== 'number' ||
!Number.isFinite(record.endTime)
) {
return null;
}
return {
reviewId: record.reviewId,
startTime: record.startTime,
endTime: record.endTime,
};
}
function parseMediaTimingReviewWaveformRequest(
payload: unknown,
): MediaTimingReviewWaveformRequest | null {
return parseMediaTimingReviewPreviewRequest(payload);
}
function parseMediaTimingReviewResolveRequest(
payload: unknown,
): MediaTimingReviewResolveRequest | null {
if (!payload || typeof payload !== 'object') return null;
const record = payload as Record<string, unknown>;
if (typeof record.reviewId !== 'string' || !record.reviewId) return null;
const decision = record.decision;
if (!decision || typeof decision !== 'object') return null;
const decisionRecord = decision as Record<string, unknown>;
if (
decisionRecord.action === 'use-original' ||
decisionRecord.action === 'skip-media' ||
decisionRecord.action === 'discard'
) {
return { reviewId: record.reviewId, decision: { action: decisionRecord.action } };
}
if (
decisionRecord.action === 'confirm' &&
typeof decisionRecord.startTime === 'number' &&
Number.isFinite(decisionRecord.startTime) &&
typeof decisionRecord.endTime === 'number' &&
Number.isFinite(decisionRecord.endTime) &&
(decisionRecord.text === undefined ||
(typeof decisionRecord.text === 'string' && decisionRecord.text.trim().length > 0))
) {
return {
reviewId: record.reviewId,
decision: {
action: 'confirm',
startTime: decisionRecord.startTime,
endTime: decisionRecord.endTime,
...(decisionRecord.text === undefined ? {} : { text: decisionRecord.text }),
},
};
}
return null;
}
export interface IpcDepsRuntimeOptions {
getMainWindow: () => WindowLike | null;
getVisibleOverlayVisibility: () => boolean;
@@ -278,6 +361,10 @@ export interface IpcDepsRuntimeOptions {
onYoutubePickerResolve: (
request: YoutubePickerResolveRequest,
) => Promise<YoutubePickerResolveResult>;
previewMediaTimingReview?: IpcServiceDeps['previewMediaTimingReview'];
getMediaTimingReviewWaveform?: IpcServiceDeps['getMediaTimingReviewWaveform'];
stopMediaTimingReviewPreview?: IpcServiceDeps['stopMediaTimingReviewPreview'];
resolveMediaTimingReview?: IpcServiceDeps['resolveMediaTimingReview'];
getAnkiConnectStatus: () => boolean;
getRuntimeOptions: () => unknown;
setRuntimeOption: (id: RuntimeOptionId, value: RuntimeOptionValue) => unknown;
@@ -371,6 +458,10 @@ export function createIpcDepsRuntime(options: IpcDepsRuntimeOptions): IpcService
options.activatePlaybackWindowForOverlayInteraction ?? (() => false),
runSubsyncManual: options.runSubsyncManual,
onYoutubePickerResolve: options.onYoutubePickerResolve,
previewMediaTimingReview: options.previewMediaTimingReview,
getMediaTimingReviewWaveform: options.getMediaTimingReviewWaveform,
stopMediaTimingReviewPreview: options.stopMediaTimingReviewPreview,
resolveMediaTimingReview: options.resolveMediaTimingReview,
getAnkiConnectStatus: options.getAnkiConnectStatus,
getRuntimeOptions: options.getRuntimeOptions,
setRuntimeOption: options.setRuntimeOption,
@@ -498,6 +589,46 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar
},
);
ipc.handle(
IPC_CHANNELS.request.mediaTimingReviewPreview,
async (_event: unknown, payload: unknown) => {
const request = parseMediaTimingReviewPreviewRequest(payload);
if (!request || !deps.previewMediaTimingReview) {
return { ok: false, message: 'Timing preview is unavailable.' };
}
return await deps.previewMediaTimingReview(request);
},
);
ipc.handle(
IPC_CHANNELS.request.mediaTimingReviewWaveform,
async (_event: unknown, payload: unknown) => {
const request = parseMediaTimingReviewWaveformRequest(payload);
if (!request || !deps.getMediaTimingReviewWaveform) {
return { ok: false, message: 'Timing waveform is unavailable.' };
}
return await deps.getMediaTimingReviewWaveform(request);
},
);
ipc.handle(
IPC_CHANNELS.request.mediaTimingReviewStopPreview,
async (_event: unknown, reviewId: unknown) => {
if (typeof reviewId !== 'string' || !reviewId || !deps.stopMediaTimingReviewPreview) {
return { ok: false, message: 'Timing preview is unavailable.' };
}
return await deps.stopMediaTimingReviewPreview(reviewId);
},
);
ipc.handle(
IPC_CHANNELS.request.mediaTimingReviewResolve,
async (_event: unknown, payload: unknown) => {
const request = parseMediaTimingReviewResolveRequest(payload);
if (!request || !deps.resolveMediaTimingReview) {
return { ok: false, message: 'Timing review is unavailable.' };
}
return await deps.resolveMediaTimingReview(request);
},
);
ipc.on(IPC_CHANNELS.command.openYomitanSettings, () => {
deps.openYomitanSettings();
});
@@ -0,0 +1,291 @@
import assert from 'node:assert/strict';
import { EventEmitter } from 'node:events';
import net from 'node:net';
import { describe, test } from 'node:test';
import { buildMediaTimingPreviewArgs, MediaTimingPreviewSession } from './media-timing-preview';
describe('buildMediaTimingPreviewArgs', () => {
test('creates a hidden audio-only reusable mpv session', () => {
const args = buildMediaTimingPreviewArgs('/tmp/review.sock', {
mediaPath: '/video/show.mkv',
audioTrackId: 3,
volume: 55,
});
assert.ok(args.includes('--no-video'));
assert.ok(args.includes('--force-window=no'));
assert.ok(args.includes('--idle=yes'));
assert.ok(args.includes('--pause=yes'));
assert.ok(args.includes('--input-ipc-server=/tmp/review.sock'));
assert.ok(args.includes('--aid=3'));
assert.ok(args.includes('--volume=55'));
assert.equal(args.at(-2), '--');
assert.equal(args.at(-1), '/video/show.mkv');
});
test('keeps source timestamps for cached remote windows', () => {
const args = buildMediaTimingPreviewArgs('/tmp/review.sock', {
mediaPath: '/tmp/window.mkv',
absoluteTimestamps: true,
});
assert.ok(args.includes('--rebase-start-time=no'));
assert.equal(
buildMediaTimingPreviewArgs('/tmp/review.sock', { mediaPath: '/video/show.mkv' }).includes(
'--rebase-start-time=no',
),
false,
);
});
test('separates an option-like media path without adding optional audio arguments', () => {
const args = buildMediaTimingPreviewArgs('/tmp/review.sock', {
mediaPath: '--fullscreen',
});
assert.equal(args.at(-2), '--');
assert.equal(args.at(-1), '--fullscreen');
assert.equal(
args.some((arg) => arg.startsWith('--aid=')),
false,
);
assert.equal(
args.some((arg) => arg.startsWith('--volume=')),
false,
);
});
});
test('preview session handles socket errors after connecting', async () => {
const socket = new net.Socket();
const child = new EventEmitter() as EventEmitter & { kill: () => boolean };
child.kill = () => true;
const session = new MediaTimingPreviewSession({
platform: 'linux',
spawnProcess: () => child as never,
connectSocket: () => {
queueMicrotask(() => socket.emit('connect'));
return socket;
},
removeSocketFile: () => undefined,
createSocketPath: () => '/tmp/review.sock',
});
await session.start({ mediaPath: '/video/show.mkv' });
assert.doesNotThrow(() => socket.emit('error', new Error('pipe closed')));
await assert.rejects(session.play(1, 2), /not ready/);
session.dispose();
});
test('preview session keeps failed connection errors handled through destruction', async () => {
const socket = new EventEmitter() as EventEmitter & {
destroy: () => void;
};
socket.destroy = () => {
socket.emit('error', new Error('socket failed again while closing'));
};
const child = new EventEmitter() as EventEmitter & { kill: () => boolean };
child.kill = () => true;
const times = [0, 0, 0, 6_000];
const session = new MediaTimingPreviewSession({
platform: 'linux',
spawnProcess: () => child as never,
connectSocket: () => {
queueMicrotask(() => socket.emit('error', new Error('connection failed')));
return socket as never;
},
now: () => times.shift() ?? 6_000,
removeSocketFile: () => undefined,
createSocketPath: () => '/tmp/review.sock',
});
await assert.rejects(session.start({ mediaPath: '/video/show.mkv' }), /Timed out starting/);
});
test('preview session rejects a connection that finishes after disposal', async () => {
const socket = new net.Socket();
const child = new EventEmitter() as EventEmitter & { kill: () => boolean };
child.kill = () => true;
const session = new MediaTimingPreviewSession({
platform: 'linux',
spawnProcess: () => child as never,
connectSocket: () => socket,
removeSocketFile: () => undefined,
createSocketPath: () => '/tmp/review.sock',
});
const pendingStart = session.start({ mediaPath: '-playlist' });
session.dispose();
socket.emit('connect');
await assert.rejects(pendingStart, /closed/);
assert.equal(socket.destroyed, true);
});
test('preview session shares one startup across concurrent start calls', async () => {
const socket = new net.Socket();
const child = new EventEmitter() as EventEmitter & { kill: () => boolean };
child.kill = () => true;
let spawnCount = 0;
const session = new MediaTimingPreviewSession({
platform: 'linux',
spawnProcess: () => {
spawnCount += 1;
return child as never;
},
connectSocket: () => socket,
removeSocketFile: () => undefined,
createSocketPath: () => '/tmp/review.sock',
});
const firstStart = session.start({ mediaPath: '/video/show.mkv' });
const secondStart = session.start({ mediaPath: '/video/show.mkv' });
socket.emit('connect');
await Promise.all([firstStart, secondStart]);
assert.equal(spawnCount, 1);
session.dispose();
});
test('preview session can start again after a startup failure', async () => {
const socket = new net.Socket();
const child = new EventEmitter() as EventEmitter & { kill: () => boolean };
child.kill = () => true;
let spawnCount = 0;
const session = new MediaTimingPreviewSession({
platform: 'linux',
spawnProcess: () => {
spawnCount += 1;
if (spawnCount === 1) throw new Error('spawn failed');
return child as never;
},
connectSocket: () => {
queueMicrotask(() => socket.emit('connect'));
return socket;
},
removeSocketFile: () => undefined,
createSocketPath: () => '/tmp/review.sock',
});
await assert.rejects(session.start({ mediaPath: '/video/show.mkv' }), /spawn failed/);
await session.start({ mediaPath: '/video/show.mkv' });
assert.equal(spawnCount, 2);
session.dispose();
});
test('preview session bounds a connection attempt that never settles', async () => {
const child = new EventEmitter() as EventEmitter & { kill: () => boolean };
child.kill = () => true;
let nowMs = 0;
let connectAttempts = 0;
const session = new MediaTimingPreviewSession({
platform: 'linux',
spawnProcess: () => child as never,
connectSocket: () => {
connectAttempts += 1;
const socket = new net.Socket();
socket.destroy = (() => {
socket.emit('error', new Error('socket failed while timing out'));
return socket;
}) as typeof socket.destroy;
return socket;
},
now: () => {
const current = nowMs;
nowMs += 1_000;
return current;
},
schedule: (callback) => setTimeout(callback, 0),
cancelSchedule: (timeout) => clearTimeout(timeout),
removeSocketFile: () => undefined,
createSocketPath: () => '/tmp/review.sock',
});
await assert.rejects(session.start({ mediaPath: '/video/show.mkv' }), /Timed out starting/);
assert.equal(connectAttempts, 1);
});
function createFakeSocket() {
const socket = new EventEmitter() as EventEmitter & {
destroyed: boolean;
write: (data: string) => boolean;
end: () => void;
destroy: () => void;
off: EventEmitter['off'];
};
const writes: string[] = [];
socket.destroyed = false;
socket.write = (data) => {
writes.push(data);
return true;
};
socket.end = () => undefined;
socket.destroy = () => {
socket.destroyed = true;
};
return { socket, writes };
}
test('preview session plays once to the clip end and reports when mpv has drained it', async () => {
const { socket, writes } = createFakeSocket();
const child = new EventEmitter() as EventEmitter & { kill: () => boolean };
child.kill = () => true;
const session = new MediaTimingPreviewSession({
platform: 'linux',
spawnProcess: () => child as never,
connectSocket: () => {
queueMicrotask(() => socket.emit('connect'));
return socket as never;
},
removeSocketFile: () => undefined,
createSocketPath: () => '/tmp/review.sock',
});
let endedCount = 0;
session.onPlaybackEnded(() => {
endedCount += 1;
});
const property = (name: string, data: boolean): string =>
`${JSON.stringify({ event: 'property-change', name, data })}\n`;
await session.start({ mediaPath: '/video/show.mkv' });
assert.deepEqual(
writes.map((line) => JSON.parse(line).command),
[
['observe_property', 1, 'eof-reached'],
['observe_property', 2, 'pause'],
],
);
// The observers' initial replies describe the idle paused player, not a finished preview.
socket.emit('data', property('eof-reached', false) + property('pause', true));
assert.equal(endedCount, 0);
writes.length = 0;
await session.play(12.25, 14.5);
assert.deepEqual(
writes.map((line) => JSON.parse(line).command),
[
['set_property', 'pause', true],
['seek', 12.25, 'absolute+exact'],
['set_property', 'end', '14.500'],
['set_property', 'pause', false],
],
);
// Events may arrive split across chunks. The decoder passing `end` flips eof-reached while
// audio still drains; only the keep-open pause that follows marks the preview as finished.
socket.emit('data', property('eof-reached', false) + property('pause', false).slice(0, 20));
socket.emit('data', property('pause', false).slice(20) + property('eof-reached', true));
assert.equal(endedCount, 0);
socket.emit('data', property('pause', true));
assert.equal(endedCount, 1);
socket.emit('data', property('pause', true));
assert.equal(endedCount, 1);
// Stopping early pauses without an end signal, and a later real EOF is not a preview end.
await session.play(1, 2);
socket.emit('data', property('eof-reached', false) + property('pause', false));
await session.stop();
socket.emit('data', property('pause', true) + property('eof-reached', true));
assert.equal(endedCount, 1);
session.dispose();
});
+394
View File
@@ -0,0 +1,394 @@
import { spawn, type ChildProcess } from 'child_process';
import fs from 'fs';
import net, { type Socket } from 'net';
import os from 'os';
import path from 'path';
import { randomUUID } from 'crypto';
const CONNECT_TIMEOUT_MS = 5_000;
const CONNECT_ATTEMPT_TIMEOUT_MS = 500;
const CONNECT_RETRY_MS = 40;
/**
* mpv flips eof-reached as soon as the decoder passes `end`, while its audio buffer is still
* draining; keep-open then pauses once the buffer has played out. A preview has ended when
* both have happened.
*/
const EOF_OBSERVER_ID = 1;
const PAUSE_OBSERVER_ID = 2;
export interface MediaTimingPreviewStartOptions {
mediaPath: string;
executablePath?: string;
audioTrackId?: number;
volume?: number;
/** The file keeps source timestamps (a cached remote window); seek with the original times. */
absoluteTimestamps?: boolean;
}
type PreviewProcess = Pick<ChildProcess, 'kill' | 'once'>;
interface MediaTimingPreviewDeps {
platform: NodeJS.Platform;
spawnProcess: (command: string, args: string[]) => PreviewProcess;
connectSocket: (socketPath: string) => Socket;
now: () => number;
schedule: (callback: () => void, delayMs: number) => ReturnType<typeof setTimeout>;
cancelSchedule: (timeout: ReturnType<typeof setTimeout>) => void;
removeSocketFile: (socketPath: string) => void;
createSocketPath: () => string;
}
export function buildMediaTimingPreviewArgs(
socketPath: string,
options: MediaTimingPreviewStartOptions,
): string[] {
const args = [
'--no-config',
'--no-video',
'--audio-display=no',
'--force-window=no',
'--idle=yes',
'--keep-open=yes',
'--pause=yes',
'--terminal=no',
'--msg-level=all=warn',
`--input-ipc-server=${socketPath}`,
];
if (typeof options.audioTrackId === 'number' && Number.isInteger(options.audioTrackId)) {
args.push(`--aid=${options.audioTrackId}`);
}
if (typeof options.volume === 'number' && Number.isFinite(options.volume)) {
args.push(`--volume=${Math.max(0, options.volume)}`);
}
if (options.absoluteTimestamps) {
args.push('--rebase-start-time=no');
}
args.push('--', options.mediaPath);
return args;
}
function createDefaultSocketPath(): string {
const suffix = `${process.pid}-${randomUUID()}`;
return process.platform === 'win32'
? `\\\\.\\pipe\\subminer-timing-preview-${suffix}`
: path.join(
// macOS limits Unix socket paths to 104 bytes, while its temp directory can be long.
process.platform === 'darwin' ? '/tmp' : os.tmpdir(),
`subminer-timing-preview-${suffix}.sock`,
);
}
function removePosixSocketFile(socketPath: string): void {
if (process.platform === 'win32') return;
try {
fs.unlinkSync(socketPath);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
throw error;
}
}
}
export class MediaTimingPreviewSession {
private readonly deps: MediaTimingPreviewDeps;
private socketPath: string | null = null;
private socket: Socket | null = null;
private process: PreviewProcess | null = null;
private startupError: Error | null = null;
private startPromise: Promise<void> | null = null;
private retryWait: {
timeout: ReturnType<typeof setTimeout>;
resolve: () => void;
} | null = null;
private disposed = false;
private readBuffer = '';
private playing = false;
private eofReached = false;
private paused = true;
private readonly endedListeners = new Set<() => void>();
constructor(deps: Partial<MediaTimingPreviewDeps> = {}) {
this.deps = {
platform: process.platform,
spawnProcess: (command, args) => spawn(command, args, { stdio: 'ignore' }),
connectSocket: (socketPath) => net.createConnection(socketPath),
now: Date.now,
schedule: (callback, delayMs) => setTimeout(callback, delayMs),
cancelSchedule: (timeout) => clearTimeout(timeout),
removeSocketFile: removePosixSocketFile,
createSocketPath: createDefaultSocketPath,
...deps,
};
}
async start(options: MediaTimingPreviewStartOptions): Promise<void> {
if (this.disposed) throw new Error('Preview session is closed');
if (this.socket) return;
if (this.startPromise) return await this.startPromise;
const startPromise = this.startOnce(options);
this.startPromise = startPromise;
try {
await startPromise;
} catch (error) {
this.releaseResources();
throw error;
} finally {
if (this.startPromise === startPromise) this.startPromise = null;
}
}
private async startOnce(options: MediaTimingPreviewStartOptions): Promise<void> {
const mediaPath = options.mediaPath.trim();
if (!mediaPath) throw new Error('No media source is available for preview');
const socketPath = this.deps.createSocketPath();
this.socketPath = socketPath;
if (this.deps.platform !== 'win32') {
this.deps.removeSocketFile(socketPath);
}
const command = options.executablePath?.trim() || 'mpv';
this.startupError = null;
const child = this.deps.spawnProcess(
command,
buildMediaTimingPreviewArgs(socketPath, { ...options, mediaPath }),
);
this.process = child;
child.once('error', (error) => {
if (this.process !== child) return;
this.startupError = error;
});
child.once('exit', () => {
if (this.process !== child) return;
if (!this.socket && !this.disposed && !this.startupError) {
this.startupError = new Error('The hidden mpv preview player exited during startup');
}
this.socket?.destroy();
this.socket = null;
this.process = null;
});
await this.connectWithRetry(socketPath);
}
/**
* Plays [startTime, endTime) once. mpv stops itself at `end` and, thanks to keep-open,
* pauses after draining the audio device, so the listener hears the whole clip even on
* high-latency outputs. onPlaybackEnded fires when mpv reports the end was reached.
*/
async play(startTime: number, endTime: number): Promise<void> {
if (!this.socket || this.socket.destroyed) {
throw new Error('Preview player is not ready');
}
if (!Number.isFinite(startTime) || !Number.isFinite(endTime) || endTime <= startTime) {
throw new Error('Preview timing is invalid');
}
this.playing = false;
this.send(['set_property', 'pause', true]);
this.send(['seek', startTime, 'absolute+exact']);
// The option parser wants a time string; a raw JSON number is not accepted for `end`.
this.send(['set_property', 'end', endTime.toFixed(3)]);
this.send(['set_property', 'pause', false]);
// Only the seek's eof-reached=false and the later keep-open pause count for this play.
this.eofReached = false;
this.paused = false;
this.playing = true;
}
async stop(): Promise<void> {
this.playing = false;
if (!this.socket || this.socket.destroyed) return;
this.send(['set_property', 'pause', true]);
}
onPlaybackEnded(listener: () => void): void {
this.endedListeners.add(listener);
}
private finishPlayback(): void {
if (!this.playing) return;
this.playing = false;
for (const listener of this.endedListeners) listener();
}
private handleSocketData(chunk: Buffer | string): void {
this.readBuffer += chunk.toString();
let newline = this.readBuffer.indexOf('\n');
while (newline !== -1) {
const line = this.readBuffer.slice(0, newline).trim();
this.readBuffer = this.readBuffer.slice(newline + 1);
newline = this.readBuffer.indexOf('\n');
if (!line) continue;
let message: unknown;
try {
message = JSON.parse(line);
} catch {
continue;
}
if (
typeof message === 'object' &&
message !== null &&
'event' in message &&
message.event === 'property-change' &&
'name' in message &&
'data' in message
) {
this.handlePropertyChange(message.name, message.data);
}
}
}
private handlePropertyChange(name: unknown, data: unknown): void {
if (name === 'eof-reached') this.eofReached = data === true;
else if (name === 'pause') this.paused = data === true;
else return;
if (this.playing && this.eofReached && this.paused) this.finishPlayback();
}
dispose(): void {
if (this.disposed) return;
this.disposed = true;
this.releaseResources();
}
private releaseResources(): void {
this.cancelRetryWait();
try {
this.send(['quit']);
} catch {
// The process may already have exited.
}
this.socket?.end();
this.socket?.destroy();
this.socket = null;
const child = this.process;
this.process = null;
child?.kill();
if (this.socketPath && this.deps.platform !== 'win32') {
try {
this.deps.removeSocketFile(this.socketPath);
} catch {
// mpv may still be releasing the socket. The OS temp directory owns cleanup.
}
}
this.socketPath = null;
}
private send(command: Array<string | number | boolean>): void {
if (!this.socket || this.socket.destroyed) {
throw new Error('Preview player is not connected');
}
this.socket.write(`${JSON.stringify({ command })}\n`);
}
private async connectWithRetry(socketPath: string): Promise<void> {
const deadline = this.deps.now() + CONNECT_TIMEOUT_MS;
while (!this.disposed && this.deps.now() < deadline) {
if (this.startupError) {
throw this.startupError;
}
try {
const remainingMs = deadline - this.deps.now();
if (remainingMs <= 0) break;
const socket = await this.connectOnce(
socketPath,
Math.min(CONNECT_ATTEMPT_TIMEOUT_MS, remainingMs),
);
if (this.disposed) {
socket.destroy();
throw new Error('Preview session is closed');
}
this.socket = socket;
this.readBuffer = '';
socket.on('data', (chunk: Buffer | string) => {
if (this.socket === socket) this.handleSocketData(chunk);
});
socket.once('close', () => this.finishPlayback());
this.send(['observe_property', EOF_OBSERVER_ID, 'eof-reached']);
this.send(['observe_property', PAUSE_OBSERVER_ID, 'pause']);
return;
} catch {
if (this.disposed) {
throw new Error('Preview session is closed');
}
const remainingMs = deadline - this.deps.now();
if (remainingMs <= 0) break;
await this.waitForRetry(Math.min(CONNECT_RETRY_MS, remainingMs));
}
}
if (this.startupError) {
throw this.startupError;
}
if (this.disposed) {
throw new Error('Preview session is closed');
}
throw new Error('Timed out starting the hidden mpv preview player');
}
private waitForRetry(delayMs: number): Promise<void> {
return new Promise<void>((resolve) => {
const timeout = this.deps.schedule(() => {
if (this.retryWait?.timeout === timeout) this.retryWait = null;
resolve();
}, delayMs);
this.retryWait = { timeout, resolve };
});
}
private cancelRetryWait(): void {
const pending = this.retryWait;
this.retryWait = null;
if (!pending) return;
this.deps.cancelSchedule(pending.timeout);
pending.resolve();
}
private connectOnce(socketPath: string, timeoutMs: number): Promise<Socket> {
return new Promise<Socket>((resolve, reject) => {
let timeout: ReturnType<typeof setTimeout> | null = null;
let settled = false;
const clearAttemptTimeout = (): void => {
if (timeout !== null) this.deps.cancelSchedule(timeout);
timeout = null;
};
const socket = this.deps.connectSocket(socketPath);
const onConnect = (): void => {
if (settled) return;
settled = true;
clearAttemptTimeout();
socket.off('error', onError);
socket.on('error', () => {
socket.destroy();
if (this.socket === socket) this.socket = null;
});
socket.once('close', () => {
if (this.socket === socket) this.socket = null;
});
resolve(socket);
};
const onError = (error: Error): void => {
if (settled) return;
settled = true;
clearAttemptTimeout();
socket.off('connect', onConnect);
socket.on('error', () => {});
socket.destroy();
reject(error);
};
socket.once('connect', onConnect);
socket.once('error', onError);
timeout = this.deps.schedule(() => {
if (settled) return;
settled = true;
timeout = null;
socket.off('connect', onConnect);
socket.off('error', onError);
socket.on('error', () => {});
socket.destroy();
reject(new Error('Timed out connecting to the hidden mpv preview player'));
}, timeoutMs);
});
}
}
@@ -0,0 +1,125 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
buildSpeechWaveformArgs,
computeWaveformPeaks,
generateSpeechWaveform,
} from './media-timing-waveform';
function pcm(samples: number[]): Buffer {
const result = Buffer.alloc(samples.length * 2);
samples.forEach((sample, index) => result.writeInt16LE(sample, index * 2));
return result;
}
test('speech waveform maps the selected FFmpeg stream and visible range', () => {
const args = buildSpeechWaveformArgs(
{
mediaPath: '/video/show.mkv',
startTime: 8,
endTime: 15,
audioStreamIndex: 3,
},
'center',
);
assert.deepEqual(args.slice(args.indexOf('-ss'), args.indexOf('-t') + 2), [
'-ss',
'8',
'-i',
'/video/show.mkv',
'-t',
'7',
]);
assert.deepEqual(args.slice(args.indexOf('-map'), args.indexOf('-map') + 2), ['-map', '0:3']);
assert.match(args[args.indexOf('-af') + 1] ?? '', /c0=FC/);
});
test('speech waveform seeks cached windows by source timestamps', () => {
const args = buildSpeechWaveformArgs(
{
mediaPath: { path: '/tmp/window.mkv', absoluteTimestamps: true, singleResolvedStream: true },
startTime: 8,
endTime: 15,
},
'downmix',
);
assert.deepEqual(args.slice(args.indexOf('-ss'), args.indexOf('-t') + 2), [
'-ss',
'8',
'-seek_timestamp',
'1',
'-i',
'/tmp/window.mkv',
'-t',
'7',
]);
assert.equal(args.includes('-map'), false);
});
test('waveform levels rise with loudness and top out at the reference level', () => {
const peaks = computeWaveformPeaks(pcm([0, 1_000, -2_000, 4_000, -8_000, 16_000]), 3);
assert.equal(peaks.length, 3);
assert.equal(peaks[0], 0);
assert.ok((peaks[1] ?? 0) > 0);
assert.ok((peaks[1] ?? 0) < (peaks[2] ?? 0));
assert.equal(peaks[2], 1);
});
test('waveform flattens steady background noise and keeps speech bursts tall', () => {
// 20 slices of steady noise at a fixed level with an 18 dB louder "speech" burst in the middle.
const noise = 1_000;
const samples: number[] = [];
for (let slice = 0; slice < 20; slice += 1) {
const level = slice >= 8 && slice < 12 ? noise * 8 : noise;
for (let sample = 0; sample < 50; sample += 1) {
samples.push(sample % 2 === 0 ? level : -level);
}
}
const peaks = computeWaveformPeaks(pcm(samples), 20);
for (const [index, peak] of peaks.entries()) {
if (index >= 8 && index < 12) assert.equal(peak, 1);
else assert.equal(peak, 0);
}
});
test('waveform stays flat when the whole range is a single steady level', () => {
const peaks = computeWaveformPeaks(
pcm(Array.from({ length: 400 }, (_, i) => (i % 2 ? 900 : -900))),
40,
);
assert.ok(peaks.every((peak) => peak === 0));
});
test('speech waveform uses a mono downmix when the source has no center activity', async () => {
const calls: string[][] = [];
const peaks = await generateSpeechWaveform(
{ mediaPath: '/video/show.mkv', startTime: 0, endTime: 2 },
async (args) => {
calls.push(args);
return calls.length === 1 ? pcm([0, 0, 0, 0]) : pcm([0, 4_000, -8_000, 16_000]);
},
);
assert.equal(calls.length, 2);
assert.match(calls[1]?.[calls[1].indexOf('-af') + 1] ?? '', /channel_layouts=mono/);
assert.equal(Math.max(...peaks), 1);
});
test('speech waveform keeps an active center channel without doing a second decode', async () => {
let calls = 0;
await generateSpeechWaveform(
{ mediaPath: '/video/show.mkv', startTime: 0, endTime: 2 },
async () => {
calls += 1;
return pcm([0, 4_000, -8_000, 16_000]);
},
);
assert.equal(calls, 1);
});
+185
View File
@@ -0,0 +1,185 @@
import { spawn } from 'node:child_process';
import { normalizeMediaInput, type MediaInput } from '../../media-input';
const WAVEFORM_SAMPLE_RATE = 8_000;
const WAVEFORM_POINT_COUNT = 480;
const WAVEFORM_TIMEOUT_MS = 15_000;
const MAX_WAVEFORM_BYTES = 16 * 1024 * 1024;
// Keep the band where speech intelligibility lives; bass, drums, and hum sit below it.
const SPEECH_FILTER = 'highpass=f=250,lowpass=f=3500';
const NOISE_FLOOR_PERCENTILE = 0.2;
const REFERENCE_PERCENTILE = 0.95;
const NOISE_GATE_DB = 3;
const MIN_DISPLAY_RANGE_DB = 12;
const SILENCE_DB = -100;
const CENTER_CHANNEL_FILTER = `pan=mono|c0=FC,${SPEECH_FILTER}`;
const DOWNMIX_FILTER = `aformat=channel_layouts=mono,${SPEECH_FILTER}`;
export interface SpeechWaveformOptions {
mediaPath: MediaInput;
startTime: number;
endTime: number;
audioStreamIndex?: number;
}
type RunFfmpeg = (args: string[]) => Promise<Buffer>;
export function buildSpeechWaveformArgs(
options: SpeechWaveformOptions,
mode: 'center' | 'downmix',
): string[] {
const duration = options.endTime - options.startTime;
const input = normalizeMediaInput(options.mediaPath);
const args = [
'-hide_banner',
'-nostdin',
'-loglevel',
'error',
'-ss',
String(options.startTime),
...input.inputArgs,
'-i',
input.path,
'-t',
String(duration),
];
if (
options.audioStreamIndex !== undefined &&
Number.isInteger(options.audioStreamIndex) &&
options.audioStreamIndex >= 0
) {
args.push('-map', `0:${options.audioStreamIndex}`);
}
args.push(
'-vn',
'-sn',
'-dn',
'-af',
mode === 'center' ? CENTER_CHANNEL_FILTER : DOWNMIX_FILTER,
'-ac',
'1',
'-ar',
String(WAVEFORM_SAMPLE_RATE),
'-f',
's16le',
'pipe:1',
);
return args;
}
function runFfmpeg(args: string[]): Promise<Buffer> {
return new Promise((resolve, reject) => {
const child = spawn('ffmpeg', args, { stdio: ['ignore', 'pipe', 'pipe'] });
const chunks: Buffer[] = [];
let byteLength = 0;
let stderr = '';
let settled = false;
const timeout = setTimeout(() => {
if (settled) return;
settled = true;
child.kill('SIGKILL');
reject(new Error(`FFmpeg waveform analysis timed out after ${WAVEFORM_TIMEOUT_MS}ms`));
}, WAVEFORM_TIMEOUT_MS);
const settle = (callback: () => void): void => {
if (settled) return;
settled = true;
clearTimeout(timeout);
callback();
};
child.stdout.on('data', (chunk: Buffer) => {
if (settled) return;
byteLength += chunk.byteLength;
if (byteLength > MAX_WAVEFORM_BYTES) {
settle(() => {
child.kill('SIGKILL');
reject(new Error('The visible waveform range is too large to analyze.'));
});
return;
}
chunks.push(chunk);
});
child.stderr.setEncoding('utf8');
child.stderr.on('data', (chunk) => {
if (stderr.length < 4_000) stderr += String(chunk);
});
child.once('error', (error) => settle(() => reject(error)));
child.once('close', (code) => {
settle(() => {
if (code === 0) {
resolve(Buffer.concat(chunks, byteLength));
return;
}
reject(new Error(stderr.trim() || `FFmpeg exited with status ${code ?? 'unknown'}`));
});
});
});
}
function percentile(sortedValues: number[], fraction: number): number {
const index = Math.min(sortedValues.length - 1, Math.floor(sortedValues.length * fraction));
return sortedValues[index] ?? SILENCE_DB;
}
/**
* Turns mono PCM into 0..1 display heights. Each point is the RMS level of its slice in
* dB, measured against the clip's own noise floor (a low percentile of the slices), so
* constant background noise draws flat and sustained speech stands out. Peak sampling
* would instead follow music transients and lift the floor to nearly speech height.
*/
export function computeWaveformPeaks(pcm: Buffer, pointCount = WAVEFORM_POINT_COUNT): number[] {
const sampleCount = Math.floor(pcm.byteLength / 2);
if (sampleCount === 0 || pointCount <= 0) return [];
const resolvedPointCount = Math.min(pointCount, sampleCount);
const levelsDb = Array.from({ length: resolvedPointCount }, () => SILENCE_DB);
for (let point = 0; point < resolvedPointCount; point += 1) {
const sampleStart = Math.floor((point * sampleCount) / resolvedPointCount);
const sampleEnd = Math.max(
sampleStart + 1,
Math.floor(((point + 1) * sampleCount) / resolvedPointCount),
);
let energy = 0;
for (let sample = sampleStart; sample < sampleEnd; sample += 1) {
const value = pcm.readInt16LE(sample * 2) / 32_768;
energy += value * value;
}
const rms = Math.sqrt(energy / (sampleEnd - sampleStart));
levelsDb[point] = rms > 0 ? Math.max(SILENCE_DB, 20 * Math.log10(rms)) : SILENCE_DB;
}
const sortedLevels = [...levelsDb].sort((left, right) => left - right);
const floorDb = percentile(sortedLevels, NOISE_FLOOR_PERCENTILE) + NOISE_GATE_DB;
const referenceDb = Math.max(
percentile(sortedLevels, REFERENCE_PERCENTILE),
floorDb + MIN_DISPLAY_RANGE_DB,
);
return levelsDb.map(
(levelDb) =>
Math.round(Math.min(1, Math.max(0, (levelDb - floorDb) / (referenceDb - floorDb))) * 1_000) /
1_000,
);
}
function hasAudibleSamples(pcm: Buffer): boolean {
for (let offset = 0; offset + 1 < pcm.byteLength; offset += 2) {
if (Math.abs(pcm.readInt16LE(offset)) >= 164) return true;
}
return false;
}
export async function generateSpeechWaveform(
options: SpeechWaveformOptions,
execute: RunFfmpeg = runFfmpeg,
): Promise<number[]> {
try {
const centerPcm = await execute(buildSpeechWaveformArgs(options, 'center'));
if (hasAudibleSamples(centerPcm)) return computeWaveformPeaks(centerPcm);
} catch {
// Sources without a named center channel can reject the center-only filter.
}
const downmixPcm = await execute(buildSpeechWaveformArgs(options, 'downmix'));
return computeWaveformPeaks(downmixPcm);
}
@@ -0,0 +1,253 @@
import assert from 'node:assert/strict';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import test from 'node:test';
import {
buildRemoteMediaWindowArgs,
RemoteMediaWindowCache,
REMOTE_MEDIA_WINDOW_MAX_SECONDS,
type RemoteMediaWindowCacheOptions,
} from './remote-media-window-cache';
const SOURCE = {
path: 'https://jellyfin.example/Videos/abc/stream?static=true',
audioStreamIndex: 2,
};
type ExecFileStub = NonNullable<RemoteMediaWindowCacheOptions['execFile']>;
function createStub(options: { fail?: boolean; empty?: boolean; defer?: boolean } = {}) {
const calls: string[][] = [];
const pendingCallbacks: Array<() => void> = [];
const execFile: ExecFileStub = (_file, args, _options, callback) => {
calls.push([...args]);
const finish = (): void => {
const outputPath = args.at(-1);
assert.ok(outputPath);
if (options.fail) {
callback(Object.assign(new Error('boom'), { code: 1 }));
return;
}
if (!options.empty) {
fs.writeFileSync(outputPath, 'mkv', 'utf8');
}
callback(null);
};
if (options.defer) {
pendingCallbacks.push(finish);
} else {
queueMicrotask(finish);
}
};
return {
calls,
execFile,
flush: () => {
for (const finish of pendingCallbacks.splice(0)) finish();
},
};
}
async function withCache(
stubOptions: Parameters<typeof createStub>[0],
cacheOptions: Omit<RemoteMediaWindowCacheOptions, 'execFile' | 'tempDir'>,
run: (cache: RemoteMediaWindowCache, stub: ReturnType<typeof createStub>) => Promise<void>,
): Promise<void> {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-media-window-test-'));
const stub = createStub(stubOptions);
const cache = new RemoteMediaWindowCache({
tempDir,
execFile: stub.execFile,
idleTtlMs: 0,
logDebug: () => undefined,
...cacheOptions,
});
try {
await run(cache, stub);
} finally {
cache.cleanup();
fs.rmSync(tempDir, { recursive: true, force: true });
}
}
function argValue(args: string[], flag: string): string | undefined {
const index = args.indexOf(flag);
return index === -1 ? undefined : args[index + 1];
}
test('buildRemoteMediaWindowArgs stream-copies the window with source timestamps intact', () => {
const args = buildRemoteMediaWindowArgs(
{ ...SOURCE, inputOptions: { reconnect: true, headers: { Referer: 'https://a.example/' } } },
{ startTime: 22.75, endTime: 33 },
'/tmp/window.mkv',
);
const inputIndex = args.indexOf('-i');
assert.equal(args[inputIndex + 1], SOURCE.path);
assert.ok(args.indexOf('-reconnect') < inputIndex);
assert.ok(args.indexOf('-headers') < inputIndex);
assert.equal(argValue(args, '-ss'), '22.75');
assert.equal(argValue(args, '-t'), '10.25');
assert.ok(args.indexOf('-t') < inputIndex);
assert.deepEqual(args.slice(args.indexOf('-map'), args.indexOf('-map') + 4), [
'-map',
'0:v:0?',
'-map',
'0:2',
]);
assert.equal(argValue(args, '-c'), 'copy');
assert.ok(args.includes('-copyts'));
assert.ok(args.includes('-start_at_zero'));
assert.equal(argValue(args, '-f'), 'matroska');
assert.equal(args.at(-1), '/tmp/window.mkv');
});
test('buildRemoteMediaWindowArgs keeps every audio stream when none is selected', () => {
const args = buildRemoteMediaWindowArgs(
{ path: SOURCE.path, audioStreamIndex: null },
{ startTime: 0, endTime: 5 },
'/tmp/window.mkv',
);
assert.equal(args[args.lastIndexOf('-map') + 1], '0:a');
});
test('acquire downloads once and reuses the window for covered ranges', async () => {
await withCache({}, {}, async (cache, stub) => {
const window = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
assert.equal(stub.calls.length, 1);
assert.equal(argValue(stub.calls[0]!, '-ss'), '9.75');
assert.equal(argValue(stub.calls[0]!, '-t'), '5.25');
assert.equal(window.startTime, 9.75);
assert.equal(window.endTime, 15);
assert.equal(window.audioStreamIndex, 2);
assert.ok(fs.existsSync(window.path));
assert.deepEqual(window.media, {
path: window.path,
source: 'remote-window',
singleResolvedStream: true,
absoluteTimestamps: true,
});
assert.equal(await cache.acquire(SOURCE, { startTime: 11, endTime: 15 }), window);
assert.equal(await cache.lookup(SOURCE, { startTime: 12, endTime: 12 }), window);
assert.equal(
await cache.lookup(
{ path: SOURCE.path, audioStreamIndex: null },
{ startTime: 12, endTime: 13 },
),
window,
);
assert.equal(stub.calls.length, 1);
});
});
test('lookup never downloads and misses on other ranges, sources, or audio streams', async () => {
await withCache({}, {}, async (cache, stub) => {
assert.equal(await cache.lookup(SOURCE, { startTime: 10, endTime: 14 }), null);
assert.equal(stub.calls.length, 0);
await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
assert.equal(await cache.lookup(SOURCE, { startTime: 14, endTime: 16 }), null);
assert.equal(
await cache.lookup(
{ path: 'https://other.example/stream', audioStreamIndex: 2 },
{
startTime: 11,
endTime: 12,
},
),
null,
);
assert.equal(
await cache.lookup(
{ path: SOURCE.path, audioStreamIndex: 3 },
{ startTime: 11, endTime: 12 },
),
null,
);
assert.equal(stub.calls.length, 1);
});
});
test('acquire widens to the union of the old window and replaces the old file', async () => {
await withCache({}, {}, async (cache, stub) => {
const first = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
const second = await cache.acquire(SOURCE, { startTime: 8, endTime: 12 });
assert.equal(stub.calls.length, 2);
assert.equal(argValue(stub.calls[1]!, '-ss'), '7.75');
assert.equal(second.startTime, 7.75);
assert.equal(second.endTime, 15);
assert.notEqual(second.path, first.path);
assert.equal(fs.existsSync(first.path), false);
assert.ok(fs.existsSync(second.path));
assert.equal(cache.currentWindow, second);
});
});
test('acquire shares an in-flight download between concurrent callers', async () => {
await withCache({ defer: true }, {}, async (cache, stub) => {
const first = cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
await Promise.resolve();
const second = cache.acquire(SOURCE, { startTime: 11, endTime: 13 });
const lookup = cache.lookup(SOURCE, { startTime: 12, endTime: 12 });
await Promise.resolve();
assert.equal(stub.calls.length, 1);
stub.flush();
const [a, b, c] = await Promise.all([first, second, lookup]);
assert.equal(a, b);
assert.equal(a, c);
assert.equal(stub.calls.length, 1);
});
});
test('acquire rejects on ffmpeg failure, leaves no file, and can retry', async () => {
await withCache({ fail: true }, {}, async (cache, stub) => {
await assert.rejects(
cache.acquire(SOURCE, { startTime: 10, endTime: 14 }),
/FFmpeg media window failed: boom/,
);
assert.equal(cache.currentWindow, null);
assert.equal(await cache.lookup(SOURCE, { startTime: 10, endTime: 14 }), null);
await assert.rejects(cache.acquire(SOURCE, { startTime: 10, endTime: 14 }));
assert.equal(stub.calls.length, 2);
});
await withCache({ empty: true }, {}, async (cache) => {
await assert.rejects(
cache.acquire(SOURCE, { startTime: 10, endTime: 14 }),
/exited without creating a media window/,
);
});
});
test('acquire refuses invalid and oversized ranges without spawning ffmpeg', async () => {
await withCache({}, {}, async (cache, stub) => {
await assert.rejects(cache.acquire(SOURCE, { startTime: 10, endTime: 10 }), /invalid/);
await assert.rejects(cache.acquire(SOURCE, { startTime: -1, endTime: 10 }), /invalid/);
await assert.rejects(
cache.acquire(SOURCE, { startTime: 0, endTime: REMOTE_MEDIA_WINDOW_MAX_SECONDS + 1 }),
/too long/,
);
assert.equal(stub.calls.length, 0);
});
});
test('the window is deleted after the idle timeout and on cleanup', async () => {
await withCache({}, { idleTtlMs: 20 }, async (cache) => {
const window = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
await new Promise((resolve) => setTimeout(resolve, 60));
assert.equal(cache.currentWindow, null);
assert.equal(fs.existsSync(window.path), false);
const again = await cache.acquire(SOURCE, { startTime: 10, endTime: 14 });
cache.cleanup();
assert.equal(fs.existsSync(again.path), false);
assert.equal(fs.existsSync(path.dirname(again.path)), false);
});
});
@@ -0,0 +1,377 @@
import { execFile as nodeExecFile, type ExecFileException } from 'child_process';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { createLogger } from '../../logger';
import { normalizeMediaInput, type MediaInput, type MediaInputOptions } from '../../media-input';
const log = createLogger('media-window');
export const REMOTE_MEDIA_WINDOW_TIMEOUT_MS = 120_000;
export const REMOTE_MEDIA_WINDOW_MAX_SECONDS = 180;
const HEAD_SLACK_SECONDS = 0.25;
const TAIL_SLACK_SECONDS = 1;
const DEFAULT_IDLE_TTL_MS = 10 * 60_000;
const COVERAGE_EPSILON_SECONDS = 0.01;
export interface RemoteMediaWindowSource {
path: string;
inputOptions?: MediaInputOptions;
/** FFmpeg stream index to keep; `null`/undefined keeps every audio stream. */
audioStreamIndex?: number | null;
}
export interface RemoteMediaWindowRange {
startTime: number;
endTime: number;
}
export interface RemoteMediaWindow {
path: string;
startTime: number;
endTime: number;
sourcePath: string;
audioStreamIndex: number | null;
/** Input descriptor for FFmpeg reads; timestamps stay absolute so callers keep source times. */
media: MediaInput;
}
type WindowExecFile = (
file: string,
args: readonly string[],
options: { timeout: number },
callback: (error: ExecFileException | null) => void,
) => void;
export interface RemoteMediaWindowCacheOptions {
tempDir?: string;
execFile?: WindowExecFile;
idleTtlMs?: number;
logDebug?: (message: string) => void;
}
interface PendingFetch extends RemoteMediaWindowRange {
sourcePath: string;
audioStreamIndex: number | null;
promise: Promise<RemoteMediaWindow>;
}
export function isRemoteMediaWindowSourcePath(value: string): boolean {
return /^https?:\/\//i.test(value.trim());
}
function describeSourceForDebugLog(sourcePath: string): string {
try {
return `remote:${new URL(sourcePath).hostname.toLowerCase() || 'unknown'}`;
} catch {
return 'remote:unknown';
}
}
function isUsableRange(range: RemoteMediaWindowRange, allowEmpty: boolean): boolean {
return (
Number.isFinite(range.startTime) &&
Number.isFinite(range.endTime) &&
range.startTime >= 0 &&
(allowEmpty ? range.endTime >= range.startTime : range.endTime > range.startTime)
);
}
function audioStreamMatches(
windowIndex: number | null,
requested: number | null | undefined,
): boolean {
return requested == null || windowIndex === requested;
}
function covers(
candidate: RemoteMediaWindowRange & { sourcePath: string; audioStreamIndex: number | null },
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): boolean {
return (
candidate.sourcePath === source.path &&
audioStreamMatches(candidate.audioStreamIndex, source.audioStreamIndex) &&
candidate.startTime <= range.startTime + COVERAGE_EPSILON_SECONDS &&
candidate.endTime >= range.endTime - COVERAGE_EPSILON_SECONDS
);
}
/**
* Stream-copies `[startTime, endTime]` of a remote source into a local Matroska file.
* `-copyts -start_at_zero` keeps the source timestamps, so later reads seek with the
* original times via `-seek_timestamp 1` (see `MediaInput.absoluteTimestamps`).
*/
export function buildRemoteMediaWindowArgs(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
outputPath: string,
): string[] {
const input = normalizeMediaInput({ path: source.path, inputOptions: source.inputOptions });
const audioMap =
typeof source.audioStreamIndex === 'number' && Number.isInteger(source.audioStreamIndex)
? `0:${source.audioStreamIndex}`
: '0:a';
return [
'-hide_banner',
'-nostdin',
'-loglevel',
'error',
'-ss',
String(range.startTime),
'-t',
String(range.endTime - range.startTime),
...input.inputArgs,
'-i',
input.path,
'-map',
'0:v:0?',
'-map',
audioMap,
'-c',
'copy',
'-sn',
'-dn',
'-copyts',
'-start_at_zero',
'-f',
'matroska',
'-y',
outputPath,
];
}
/**
* Holds one downloaded window of the current remote stream so the timing review,
* audio extraction, and screenshot all read the same local bytes instead of each
* re-fetching the clip over HTTP. A new window replaces the old one; the file is
* deleted after `idleTtlMs` without use, on `clear()`, or on `cleanup()`.
*/
export class RemoteMediaWindowCache {
private readonly tempDir: string;
private readonly execFile: WindowExecFile;
private readonly idleTtlMs: number;
private readonly logDebug: (message: string) => void;
private current: RemoteMediaWindow | null = null;
private pending: PendingFetch | null = null;
private idleTimer: ReturnType<typeof setTimeout> | null = null;
private sequence = 0;
constructor(options: RemoteMediaWindowCacheOptions = {}) {
this.tempDir = options.tempDir ?? path.join(os.tmpdir(), 'subminer-media-windows');
this.execFile = options.execFile ?? nodeExecFile;
this.idleTtlMs = options.idleTtlMs ?? DEFAULT_IDLE_TTL_MS;
this.logDebug = options.logDebug ?? ((message) => log.debug(message));
}
get currentWindow(): RemoteMediaWindow | null {
return this.current;
}
/** Returns a ready or in-flight window covering the range; never starts a download. */
async lookup(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow | null> {
if (!isUsableRange(range, true)) return null;
if (this.current && covers(this.current, source, range)) {
this.touch();
return this.current;
}
const pending = this.pending;
if (pending && covers(pending, source, range)) {
try {
const window = await pending.promise;
this.touch();
return window;
} catch {
return null;
}
}
return null;
}
/** Returns a window covering the range, downloading (and widening) one when needed. */
async acquire(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow> {
if (!isUsableRange(range, false)) {
throw new Error('Media window range is invalid.');
}
if (range.endTime - range.startTime > REMOTE_MEDIA_WINDOW_MAX_SECONDS) {
throw new Error('Media window range is too long to download.');
}
for (;;) {
const hit = await this.lookup(source, range);
if (hit) return hit;
const pending = this.pending;
if (!pending) break;
// Another caller is already downloading; wait for it, then re-check coverage.
await pending.promise.catch(() => null);
}
return this.fetch(source, this.planFetchRange(source, range));
}
clear(): void {
this.cancelIdleTimer();
const current = this.current;
this.current = null;
if (current) this.removeFile(current.path);
}
cleanup(): void {
this.clear();
try {
fs.rmSync(this.tempDir, { recursive: true, force: true });
} catch (error) {
log.error('Failed to cleanup media window directory:', error);
}
}
private planFetchRange(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): RemoteMediaWindowRange {
let startTime = Math.max(0, range.startTime - HEAD_SLACK_SECONDS);
let endTime = range.endTime + TAIL_SLACK_SECONDS;
const current = this.current;
if (
current &&
current.sourcePath === source.path &&
audioStreamMatches(current.audioStreamIndex, source.audioStreamIndex)
) {
// Keep what was already downloaded when the review timeline grows in one direction.
const unionStart = Math.min(startTime, current.startTime);
const unionEnd = Math.max(endTime, current.endTime);
if (unionEnd - unionStart <= REMOTE_MEDIA_WINDOW_MAX_SECONDS) {
startTime = unionStart;
endTime = unionEnd;
}
}
return { startTime, endTime };
}
private fetch(
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow> {
fs.mkdirSync(this.tempDir, { recursive: true });
this.sequence += 1;
const outputPath = path.join(this.tempDir, `window_${Date.now()}_${this.sequence}.mkv`);
const audioStreamIndex =
typeof source.audioStreamIndex === 'number' ? source.audioStreamIndex : null;
const description = describeSourceForDebugLog(source.path);
const startedAt = Date.now();
this.logDebug(
`[media-window] fetch start ${description} start=${range.startTime} end=${range.endTime} audioStream=${audioStreamIndex ?? 'all'}`,
);
const promise = new Promise<RemoteMediaWindow>((resolve, reject) => {
this.execFile(
'ffmpeg',
buildRemoteMediaWindowArgs(source, range, outputPath),
{ timeout: REMOTE_MEDIA_WINDOW_TIMEOUT_MS },
(error) => {
const elapsedMs = Math.max(0, Date.now() - startedAt);
const size = error ? 0 : this.fileSize(outputPath);
if (error || size === 0) {
this.removeFile(outputPath);
const reason = error
? error.code === 'ENOENT'
? 'FFmpeg not found. Install FFmpeg to enable media generation.'
: `FFmpeg media window failed: ${error.message}`
: 'FFmpeg exited without creating a media window.';
this.logDebug(`[media-window] fetch failed ${description} elapsedMs=${elapsedMs}`);
reject(new Error(reason));
return;
}
const window: RemoteMediaWindow = {
path: outputPath,
startTime: range.startTime,
endTime: range.endTime,
sourcePath: source.path,
audioStreamIndex,
media: {
path: outputPath,
source: 'remote-window',
singleResolvedStream: true,
absoluteTimestamps: true,
},
};
this.logDebug(
`[media-window] fetch complete ${description} elapsedMs=${elapsedMs} bytes=${size}`,
);
this.replaceCurrent(window);
resolve(window);
},
);
});
const pending: PendingFetch = {
sourcePath: source.path,
audioStreamIndex,
startTime: range.startTime,
endTime: range.endTime,
promise,
};
this.pending = pending;
promise
.catch(() => undefined)
.then(() => {
if (this.pending === pending) this.pending = null;
});
return promise;
}
private replaceCurrent(window: RemoteMediaWindow): void {
const previous = this.current;
this.current = window;
if (previous && previous.path !== window.path) this.removeFile(previous.path);
this.touch();
}
private touch(): void {
this.cancelIdleTimer();
if (this.idleTtlMs <= 0 || !this.current) return;
const timer = setTimeout(() => {
if (this.idleTimer === timer) this.idleTimer = null;
this.clear();
}, this.idleTtlMs);
timer.unref?.();
this.idleTimer = timer;
}
private cancelIdleTimer(): void {
if (this.idleTimer) clearTimeout(this.idleTimer);
this.idleTimer = null;
}
private fileSize(filePath: string): number {
try {
return fs.statSync(filePath).size;
} catch {
return 0;
}
}
private removeFile(filePath: string): void {
try {
fs.unlinkSync(filePath);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
log.debug(`Failed to remove media window ${filePath}:`, (error as Error).message);
}
}
}
}
let sharedCache: RemoteMediaWindowCache | null = null;
/** Process-wide cache so the review modal and card media generation share one download. */
export function getSharedRemoteMediaWindowCache(): RemoteMediaWindowCache {
sharedCache ??= new RemoteMediaWindowCache();
return sharedCache;
}
+83 -12
View File
@@ -235,7 +235,10 @@ import {
createCycleSecondarySubModeRuntimeHandler,
} from './main/runtime/domains/mpv';
import { buildSubtitleTrackDiagnostics } from './main/runtime/mpv-track-diagnostics';
import { resolveCanonicalPrimarySubtitle } from './main/runtime/primary-subtitle-text';
import {
resolveCanonicalPrimarySubtitle,
resolvePrimarySubtitle,
} from './main/runtime/primary-subtitle-text';
import {
createBuildCopyCurrentSubtitleMainDepsHandler,
createBuildHandleMineSentenceDigitMainDepsHandler,
@@ -463,6 +466,15 @@ import { createMainBootServices, type MainBootServicesResult } from './main/boot
import { handleCliCommandRuntimeServiceWithContext } from './main/cli-runtime';
import { createOverlayModalRuntimeService } from './main/overlay-runtime';
import { createOverlayModalInputState } from './main/runtime/overlay-modal-input-state';
import { MediaTimingPreviewSession } from './core/services/media-timing-preview';
import { getSharedRemoteMediaWindowCache } from './core/services/remote-media-window-cache';
import { resolveMediaGenerationInput } from './anki-integration/media-source';
import { generateSpeechWaveform } from './core/services/media-timing-waveform';
import {
collectMediaTimingContextLines,
createMediaTimingReviewRuntime,
} from './main/runtime/media-timing-review';
import { openMediaTimingReviewModal } from './main/runtime/media-timing-review-open';
import { openYoutubeTrackPicker } from './main/runtime/youtube-picker-open';
import { openRuntimeOptionsModal as openRuntimeOptionsModalRuntime } from './main/runtime/runtime-options-open';
import { openJimakuModal as openJimakuModalRuntime } from './main/runtime/jimaku-open';
@@ -1820,28 +1832,31 @@ function withCurrentSubtitleTiming(payload: SubtitleData): SubtitleData {
}
function captureCurrentPrimarySubtitleMiningContext(): SubtitleMiningContext | null {
const canonical = resolveCanonicalPrimarySubtitle({
// Mine what the overlay shows, not raw mpv `sub-text`: the raw text lists every active
// event, so a finished caption row lingering beside a fresh line would end up on the
// card. The parsed view also carries the cue's own timings for the clip range.
const resolved = resolvePrimarySubtitle({
liveText: appState.mpvClient?.currentSubText ?? '',
currentTimeSec: Number(appState.mpvClient?.currentTimePos),
cues: appState.activeParsedSubtitleCues,
});
// Same validity bar as the live capture path: an unusable canonical span must fall
// Same validity bar as the live capture path: an unusable resolved span must fall
// back rather than hand mining an empty line or an inverted range.
const canonicalText = canonical?.text.trim();
const resolvedText = resolved?.text.replace(/\n{2,}/g, '\n').trim();
if (
!canonical ||
!canonicalText ||
!Number.isFinite(canonical.startTime) ||
!Number.isFinite(canonical.endTime) ||
canonical.endTime <= canonical.startTime
!resolved ||
!resolvedText ||
!Number.isFinite(resolved.startTime) ||
!Number.isFinite(resolved.endTime) ||
resolved.endTime <= resolved.startTime
) {
return captureLiveSubtitleMiningContext(appState.mpvClient);
}
return {
source: 'overlay',
text: canonicalText,
startTime: canonical.startTime,
endTime: canonical.endTime,
text: resolvedText,
startTime: resolved.startTime,
endTime: resolved.endTime,
capturedAtMs: Date.now(),
};
}
@@ -2881,6 +2896,49 @@ function createOverlayHostedModalOpenDeps(): {
};
}
const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({
getMpvClient: () => appState.mpvClient,
getCurrentMediaPath: () =>
appState.currentMediaPath?.trim() || appState.mpvClient?.currentVideoPath?.trim() || null,
getMpvExecutablePath: () =>
configService.getConfig().mpv.executablePath || process.env.SUBMINER_MPV_PATH?.trim() || '',
createPreviewSession: () => new MediaTimingPreviewSession(),
generateWaveform: (options) => generateSpeechWaveform(options),
resolveMediaSource: async () => {
const resolved = await resolveMediaGenerationInput(appState.mpvClient, 'audio', {
getCachedMediaPath: (currentVideoPath, kind) =>
getCachedYoutubeMediaPathForCurrentPlayback(currentVideoPath, kind),
remoteCacheMode: shouldRequireYoutubeMediaCacheForCurrentPlayback() ? 'required' : 'optional',
});
return resolved
? {
path: resolved.path,
...(resolved.inputOptions ? { inputOptions: resolved.inputOptions } : {}),
singleResolvedStream: resolved.singleResolvedStream,
}
: null;
},
acquireMediaWindow: (source, range) => getSharedRemoteMediaWindowCache().acquire(source, range),
getSubtitleContextLines: (range) =>
collectMediaTimingContextLines({
cues: appState.activeParsedSubtitleCues,
fallbackPrevious: appState.subtitleTimingTracker?.getRecentEntries(40) ?? [],
startTime: range.startTime,
endTime: range.endTime,
}),
openModal: (payload) => openMediaTimingReviewModal(createOverlayHostedModalOpenDeps(), payload),
onPreviewEnded: (reviewId) => {
// The review may live in either overlay window; the renderer ignores foreign review ids.
for (const window of [overlayManager.getMainWindow(), overlayManager.getModalWindow()]) {
if (window && !window.isDestroyed()) {
window.webContents.send(IPC_CHANNELS.event.mediaTimingReviewPreviewEnded, reviewId);
}
}
},
showStatus: (message) =>
overlayNotificationsRuntime.showConfiguredStatusNotification(message, { variant: 'warning' }),
});
function openOverlayHostedModalWithOsd(
openModal: (deps: ReturnType<typeof createOverlayHostedModalOpenDeps>) => Promise<boolean>,
unavailableMessage: string,
@@ -3946,6 +4004,7 @@ const {
cleanupInternalSubtitleTrackCache: () => cachedInternalSubtitleTrackExtractor.clear(),
cleanupYoutubeSubtitleTempDirs: () => youtubeFlowRuntime.cleanupSubtitleTempDirs(),
cleanupYoutubeMediaCache: () => youtubeMediaCache.cleanup(),
cleanupRemoteMediaWindows: () => getSharedRemoteMediaWindowCache().cleanup(),
cleanupJellyfinSubtitleCache: () => cleanupJellyfinSubtitleCache(),
stopDiscordPresenceService: () => {
void appState.discordPresenceService?.stop();
@@ -5098,6 +5157,7 @@ function initializeOverlayRuntime(): void {
appState.ankiIntegration?.setRecordCardsMinedCallback(recordTrackedCardsMined);
appState.ankiIntegration?.setKnownWordCacheUpdatedCallback(refreshCurrentSubtitleAnnotations);
appState.ankiIntegration?.setSubtitleMiningContextConsumer(consumePendingSubtitleMiningContext);
appState.ankiIntegration?.setMediaTimingReviewCallback(mediaTimingReviewRuntime.requestReview);
syncOverlayMpvSubtitleSuppression();
}
@@ -5507,6 +5567,10 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
showMpvOsd: (text: string) => showConfiguredPlaybackFeedback(text),
},
mainDeps: {
previewMediaTimingReview: (request) => mediaTimingReviewRuntime.previewRange(request),
getMediaTimingReviewWaveform: (request) => mediaTimingReviewRuntime.getWaveform(request),
stopMediaTimingReviewPreview: (reviewId) => mediaTimingReviewRuntime.stopPreview(reviewId),
resolveMediaTimingReview: (request) => mediaTimingReviewRuntime.resolveReview(request),
getMainWindow: () => overlayManager.getMainWindow(),
getVisibleOverlayVisibility: () => overlayManager.getVisibleOverlayVisible(),
focusMainWindow: () => {
@@ -5540,6 +5604,9 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
}
},
onOverlayModalClosed: (modal, senderWindow) => {
if (modal === 'media-timing-review') {
void mediaTimingReviewRuntime.dispose();
}
if (modal === 'subtitle-sidebar' && senderWindow === overlayManager.getMainWindow()) {
subtitleSidebarRequestedOpen = false;
}
@@ -5893,6 +5960,9 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
appState.ankiIntegration?.setSubtitleMiningContextConsumer(
consumePendingSubtitleMiningContext,
);
appState.ankiIntegration?.setMediaTimingReviewCallback(
mediaTimingReviewRuntime.requestReview,
);
},
getKnownWordCacheStatePath: () => path.join(USER_DATA_PATH, 'known-words-cache.json'),
getCachedMediaPath: (currentVideoPath, kind) =>
@@ -6217,6 +6287,7 @@ const { createMainWindow: createMainWindowHandler, createModalWindow: createModa
if (overlayManager.getModalWindow() !== window) {
return;
}
void mediaTimingReviewRuntime.dispose();
overlayManager.setModalWindow(null);
}
},
+8
View File
@@ -62,6 +62,10 @@ export interface MainIpcRuntimeServiceDepsParams {
onOverlayInteractiveHint?: IpcDepsRuntimeOptions['onOverlayInteractiveHint'];
handleOverlayNotificationAction?: IpcDepsRuntimeOptions['handleOverlayNotificationAction'];
onYoutubePickerResolve: IpcDepsRuntimeOptions['onYoutubePickerResolve'];
previewMediaTimingReview?: IpcDepsRuntimeOptions['previewMediaTimingReview'];
getMediaTimingReviewWaveform?: IpcDepsRuntimeOptions['getMediaTimingReviewWaveform'];
stopMediaTimingReviewPreview?: IpcDepsRuntimeOptions['stopMediaTimingReviewPreview'];
resolveMediaTimingReview?: IpcDepsRuntimeOptions['resolveMediaTimingReview'];
openYomitanSettings: IpcDepsRuntimeOptions['openYomitanSettings'];
quitApp: IpcDepsRuntimeOptions['quitApp'];
toggleVisibleOverlay: IpcDepsRuntimeOptions['toggleVisibleOverlay'];
@@ -257,6 +261,10 @@ export function createMainIpcRuntimeServiceDeps(
onOverlayInteractiveHint: params.onOverlayInteractiveHint,
handleOverlayNotificationAction: params.handleOverlayNotificationAction,
onYoutubePickerResolve: params.onYoutubePickerResolve,
previewMediaTimingReview: params.previewMediaTimingReview,
getMediaTimingReviewWaveform: params.getMediaTimingReviewWaveform,
stopMediaTimingReviewPreview: params.stopMediaTimingReviewPreview,
resolveMediaTimingReview: params.resolveMediaTimingReview,
openYomitanSettings: params.openYomitanSettings,
quitApp: params.quitApp,
toggleVisibleOverlay: params.toggleVisibleOverlay,
+60
View File
@@ -828,6 +828,7 @@ test('modal fallback reveal skips showing window when content is not ready', asy
setModalWindowBounds: () => {},
},
{
platform: 'darwin',
scheduleRevealFallback: (callback) => {
scheduledReveal = callback;
return { scheduled: true } as never;
@@ -1363,3 +1364,62 @@ test('modal placement reconcile cancels stale retry ladder after a newer visible
globalThis.clearTimeout = originalClearTimeout;
}
});
test('Linux keeps the dedicated modal window unmapped until the renderer opens the modal, then hides the overlay before revealing it', () => {
const mainWindow = createMockWindow();
mainWindow.visible = true;
const modalWindow = createMockWindow();
const order: string[] = [];
const hideMain = mainWindow.hide;
mainWindow.hide = () => {
order.push('main:hide');
hideMain();
};
const showModal = modalWindow.show;
modalWindow.show = () => {
order.push('modal:show');
showModal();
};
let revealScheduled = false;
const runtime = createOverlayModalRuntimeService(
{
getMainWindow: () => mainWindow as never,
getModalWindow: () => modalWindow as never,
createModalWindow: () => modalWindow as never,
getModalGeometry: () => ({ x: 0, y: 0, width: 400, height: 300 }),
setModalWindowBounds: () => {},
},
{
platform: 'linux',
scheduleRevealFallback: () => {
revealScheduled = true;
return { scheduled: true } as never;
},
clearRevealFallback: () => {},
},
);
const open = () =>
runtime.sendToActiveOverlayWindow(
'media-timing-review:open',
{ reviewId: 'review' },
{ restoreOnModalClose: 'media-timing-review', preferModalWindow: true },
);
assert.equal(open(), true);
assert.deepEqual(modalWindow.sent, [['media-timing-review:open', { reviewId: 'review' }]]);
assert.equal(revealScheduled, false);
assert.equal(modalWindow.getShowCount(), 0);
assert.equal(mainWindow.getHideCount(), 0);
// The open retry must not map the window before the renderer answers either.
assert.equal(open(), true);
assert.equal(modalWindow.getShowCount(), 0);
runtime.notifyOverlayModalOpened('media-timing-review');
assert.deepEqual(order, ['main:hide', 'modal:show']);
assert.equal(mainWindow.isVisible(), false);
assert.equal(modalWindow.isVisible(), true);
assert.equal(modalWindow.ignoreMouseEvents, false);
});
+21 -5
View File
@@ -90,6 +90,12 @@ export function createOverlayModalRuntimeService(
const platform = options.platform ?? process.platform;
const shouldPrimeModalWindow = platform === 'darwin' || platform === 'win32';
const reuseModalWindowAfterClose = platform === 'darwin';
// On Linux (Hyprland) every placement dispatch on a mapped window (resize, move, set_prop)
// blanks the still-visible overlay for a few frames while mpv is fullscreen. Revealing the
// dedicated modal window before its renderer has the modal open runs the placement ladder,
// and the open retry, against a visible overlay, which the user sees as flicker. Keep the
// window unmapped until the renderer acknowledges the open, then hide the overlay first.
const deferModalRevealUntilOpened = platform === 'linux';
const focusApplication = options.focusApplication ?? requestOverlayApplicationFocus;
const scheduleRevealFallback = (callback: () => void, delayMs: number): RevealFallbackHandle =>
(options.scheduleRevealFallback ?? globalThis.setTimeout)(callback, delayMs);
@@ -457,7 +463,9 @@ export function createOverlayModalRuntimeService(
deps.setModalWindowBounds(deps.getModalGeometry());
const wasVisible = modalWindow.isVisible();
if (!wasVisible) {
if (modalWindowPrimedForImmediateShow && isWindowReadyForIpc(modalWindow)) {
if (deferModalRevealUntilOpened) {
// notifyOverlayModalOpened reveals the window once the renderer has the modal open.
} else if (modalWindowPrimedForImmediateShow && isWindowReadyForIpc(modalWindow)) {
showModalWindow(modalWindow);
} else {
scheduleModalWindowReveal(modalWindow);
@@ -560,15 +568,23 @@ export function createOverlayModalRuntimeService(
}
const modalWindow = deps.getModalWindow();
const targetIsModalWindow =
modalWindow !== null && !modalWindow.isDestroyed() && targetWindow === modalWindow;
const handOffMainWindowToModal = (): void => {
setMainWindowMousePassthroughForModal(true);
setMainWindowVisibilityForModal(true);
};
if (targetIsModalWindow && deferModalRevealUntilOpened) {
handOffMainWindowToModal();
}
if (targetWindow.isVisible()) {
ensureModalWindowInteractive(targetWindow);
} else {
showModalWindow(targetWindow);
}
if (modalWindow && !modalWindow.isDestroyed() && targetWindow === modalWindow) {
setMainWindowMousePassthroughForModal(true);
setMainWindowVisibilityForModal(true);
if (targetIsModalWindow && !deferModalRevealUntilOpened) {
handOffMainWindowToModal();
}
};
@@ -46,12 +46,13 @@ test('on will quit cleanup handler runs all cleanup steps', () => {
cleanupInternalSubtitleTrackCache: () => calls.push('cleanup-internal-subtitles'),
cleanupYoutubeSubtitleTempDirs: () => calls.push('cleanup-youtube-subtitles'),
cleanupYoutubeMediaCache: () => calls.push('cleanup-youtube-media'),
cleanupRemoteMediaWindows: () => calls.push('cleanup-remote-media-windows'),
cleanupJellyfinSubtitleCache: () => calls.push('cleanup-jellyfin-subtitles'),
stopDiscordPresenceService: () => calls.push('stop-discord-presence'),
});
cleanup();
assert.equal(calls.length, 35);
assert.equal(calls.length, 36);
assert.equal(calls[0], 'destroy-tray');
assert.equal(calls[calls.length - 1], 'stop-discord-presence');
assert.ok(calls.includes('cleanup-jellyfin-subtitles'));
@@ -60,6 +61,7 @@ test('on will quit cleanup handler runs all cleanup steps', () => {
assert.ok(calls.includes('clear-linux-mpv-fullscreen-overlay-refresh-timeouts'));
assert.ok(calls.includes('cleanup-youtube-subtitles'));
assert.ok(calls.includes('cleanup-youtube-media'));
assert.ok(calls.includes('cleanup-remote-media-windows'));
assert.ok(calls.indexOf('flush-mpv-log') < calls.indexOf('destroy-socket'));
});
@@ -102,6 +104,7 @@ test('on will quit cleanup handler cleans jellyfin subtitle cache when stopping
cleanupInternalSubtitleTrackCache: () => calls.push('cleanup-internal-subtitles'),
cleanupYoutubeSubtitleTempDirs: () => calls.push('cleanup-youtube-subtitles'),
cleanupYoutubeMediaCache: () => calls.push('cleanup-youtube-media'),
cleanupRemoteMediaWindows: () => calls.push('cleanup-remote-media-windows'),
cleanupJellyfinSubtitleCache: () => calls.push('cleanup-jellyfin-subtitles'),
stopDiscordPresenceService: () => calls.push('stop-discord-presence'),
});
@@ -32,6 +32,7 @@ export function createOnWillQuitCleanupHandler(deps: {
cleanupInternalSubtitleTrackCache: () => void;
cleanupYoutubeSubtitleTempDirs: () => void;
cleanupYoutubeMediaCache: () => void;
cleanupRemoteMediaWindows: () => void;
cleanupJellyfinSubtitleCache: () => void;
stopDiscordPresenceService: () => void;
}) {
@@ -76,6 +77,7 @@ export function createOnWillQuitCleanupHandler(deps: {
}
deps.cleanupYoutubeSubtitleTempDirs();
deps.cleanupYoutubeMediaCache();
deps.cleanupRemoteMediaWindows();
deps.stopDiscordPresenceService();
return Promise.resolve(stopSyncAutoScheduler);
};
@@ -75,6 +75,7 @@ test('cleanup deps builder returns handlers that guard optional runtime objects'
cleanupInternalSubtitleTrackCache: () => calls.push('cleanup-internal-subtitles'),
cleanupYoutubeSubtitleTempDirs: () => calls.push('cleanup-youtube-subtitles'),
cleanupYoutubeMediaCache: () => calls.push('cleanup-youtube-media'),
cleanupRemoteMediaWindows: () => calls.push('cleanup-remote-media-windows'),
cleanupJellyfinSubtitleCache: () => calls.push('cleanup-jellyfin-subtitles'),
stopDiscordPresenceService: () => calls.push('stop-discord-presence'),
});
@@ -157,6 +158,7 @@ test('cleanup deps builder skips destroyed yomitan window', () => {
cleanupInternalSubtitleTrackCache: () => {},
cleanupYoutubeSubtitleTempDirs: () => {},
cleanupYoutubeMediaCache: () => {},
cleanupRemoteMediaWindows: () => {},
cleanupJellyfinSubtitleCache: () => {},
stopDiscordPresenceService: () => {},
});
@@ -210,6 +212,7 @@ test('cleanup deps builder skips global shortcut cleanup before app ready', () =
cleanupInternalSubtitleTrackCache: () => {},
cleanupYoutubeSubtitleTempDirs: () => {},
cleanupYoutubeMediaCache: () => {},
cleanupRemoteMediaWindows: () => {},
cleanupJellyfinSubtitleCache: () => {},
stopDiscordPresenceService: () => {},
});
@@ -61,6 +61,7 @@ export function createBuildOnWillQuitCleanupDepsHandler(deps: {
cleanupInternalSubtitleTrackCache: () => void;
cleanupYoutubeSubtitleTempDirs: () => void;
cleanupYoutubeMediaCache: () => void;
cleanupRemoteMediaWindows: () => void;
cleanupJellyfinSubtitleCache: () => void;
stopDiscordPresenceService: () => void;
}) {
@@ -148,6 +149,7 @@ export function createBuildOnWillQuitCleanupDepsHandler(deps: {
cleanupInternalSubtitleTrackCache: () => deps.cleanupInternalSubtitleTrackCache(),
cleanupYoutubeSubtitleTempDirs: () => deps.cleanupYoutubeSubtitleTempDirs(),
cleanupYoutubeMediaCache: () => deps.cleanupYoutubeMediaCache(),
cleanupRemoteMediaWindows: () => deps.cleanupRemoteMediaWindows(),
cleanupJellyfinSubtitleCache: () => deps.cleanupJellyfinSubtitleCache(),
stopDiscordPresenceService: () => deps.stopDiscordPresenceService(),
});
@@ -52,6 +52,7 @@ test('composeStartupLifecycleHandlers returns callable startup lifecycle handler
cleanupInternalSubtitleTrackCache: () => {},
cleanupYoutubeSubtitleTempDirs: () => {},
cleanupYoutubeMediaCache: () => {},
cleanupRemoteMediaWindows: () => {},
cleanupJellyfinSubtitleCache: () => {},
stopDiscordPresenceService: () => {},
},
@@ -156,6 +156,7 @@ test('createConfigHotReloadAppliedHandler applies only changed Anki media option
const config = deepCloneConfig(DEFAULT_CONFIG);
config.ankiConnect.media.normalizeAudio = false;
config.ankiConnect.media.mirrorMpvVolume = false;
config.ankiConnect.media.reviewTiming = true;
const ankiPatches: unknown[] = [];
const applyHotReload = createConfigHotReloadAppliedHandler({
@@ -181,10 +182,18 @@ test('createConfigHotReloadAppliedHandler applies only changed Anki media option
},
config,
);
applyHotReload(
{
hotReloadFields: ['ankiConnect.media.reviewTiming'],
restartRequiredFields: [],
},
config,
);
assert.deepEqual(ankiPatches, [
{ media: { normalizeAudio: false } },
{ media: { mirrorMpvVolume: false } },
{ media: { reviewTiming: true } },
]);
});
@@ -100,6 +100,9 @@ function buildAnkiRuntimeConfigPatch(
if (diff.hotReloadFields.includes('ankiConnect.media.mirrorMpvVolume')) {
mediaPatch.mirrorMpvVolume = config.ankiConnect.media.mirrorMpvVolume;
}
if (diff.hotReloadFields.includes('ankiConnect.media.reviewTiming')) {
mediaPatch.reviewTiming = config.ankiConnect.media.reviewTiming;
}
if (Object.keys(mediaPatch).length > 0) {
patch.media = mediaPatch;
}
@@ -0,0 +1,42 @@
import { IPC_CHANNELS, type OverlayHostedModal } from '../../shared/ipc/contracts';
import type { MediaTimingReviewOpenPayload } from '../../types/anki';
import { openOverlayHostedModal, retryOverlayModalOpen } from './overlay-hosted-modal-open';
const MODAL: OverlayHostedModal = 'media-timing-review';
export async function openMediaTimingReviewModal(
deps: {
ensureOverlayStartupPrereqs: () => void;
ensureOverlayWindowsReadyForVisibilityActions: () => void;
sendToActiveOverlayWindow: (
channel: string,
payload?: unknown,
runtimeOptions?: {
restoreOnModalClose?: OverlayHostedModal;
preferModalWindow?: boolean;
},
) => boolean;
waitForModalOpen: (modal: OverlayHostedModal, timeoutMs: number) => Promise<boolean>;
logWarn: (message: string) => void;
},
payload: MediaTimingReviewOpenPayload,
): Promise<boolean> {
return await retryOverlayModalOpen(
{ waitForModalOpen: deps.waitForModalOpen, logWarn: deps.logWarn },
{
modal: MODAL,
// The review renderer regularly needs more than the 1.5 s the other modals allow; a
// premature retry re-sends the payload and reloads the waveform for nothing.
timeoutMs: 4_000,
retryWarning:
'Media timing review did not acknowledge modal open; retrying the dedicated modal window.',
sendOpen: () =>
openOverlayHostedModal(deps, {
channel: IPC_CHANNELS.event.mediaTimingReviewOpen,
modal: MODAL,
payload,
preferModalWindow: true,
}),
},
);
}
@@ -0,0 +1,859 @@
import assert from 'node:assert/strict';
import { describe, test } from 'node:test';
import type { MediaTimingReviewOpenPayload } from '../../types/anki';
import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform';
import type {
RemoteMediaWindow,
RemoteMediaWindowRange,
RemoteMediaWindowSource,
} from '../../core/services/remote-media-window-cache';
import type { MediaTimingPreviewSession } from '../../core/services/media-timing-preview';
type MediaTimingPreviewSessionLike = Pick<MediaTimingPreviewSession, 'start'>;
import {
buildMediaTimingReviewPayload,
collectMediaTimingContextLines,
createMediaTimingReviewRuntime,
} from './media-timing-review';
describe('buildMediaTimingReviewPayload', () => {
test('starts from the padded range and leaves two seconds to drag on each side', () => {
const payload = buildMediaTimingReviewPayload(
{
kind: 'sentence',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0.5,
maxMediaDuration: 30,
},
{ reviewId: 'review-1', mediaDuration: 100 },
);
assert.equal(payload.selectionStartTime, 9.5);
assert.equal(payload.selectionEndTime, 12.5);
assert.equal(payload.timelineStartTime, 7.5);
assert.equal(payload.timelineEndTime, 14.5);
});
test('clamps the padded selection and timeline to media bounds', () => {
const payload = buildMediaTimingReviewPayload(
{
kind: 'word',
text: '字幕',
startTime: 0.2,
endTime: 9.8,
audioPadding: 1,
maxMediaDuration: 30,
},
{ reviewId: 'review-2', mediaDuration: 10 },
);
assert.equal(payload.selectionStartTime, 0);
assert.equal(payload.selectionEndTime, 10);
assert.equal(payload.timelineStartTime, 0);
assert.equal(payload.timelineEndTime, 10);
});
test('keeps an uncapped selection when max media duration is disabled', () => {
const payload = buildMediaTimingReviewPayload(
{
kind: 'sentence',
text: '字幕',
startTime: 10,
endTime: 55,
audioPadding: 1,
maxMediaDuration: 0,
},
{ reviewId: 'review-unlimited', mediaDuration: 100 },
);
assert.equal(payload.selectionStartTime, 9);
assert.equal(payload.selectionEndTime, 56);
assert.equal(payload.maxMediaDuration, 0);
});
});
async function startActiveMediaTimingReview(
options: {
maxMediaDuration?: number;
decisionTimeoutMs?: number;
generateWaveform?: () => Promise<number[]>;
play?: () => Promise<void>;
} = {},
) {
const previewCalls: Array<[number, number]> = [];
let publishPayload!: (payload: MediaTimingReviewOpenPayload) => void;
const openedPayload = new Promise<MediaTimingReviewOpenPayload>((resolve) => {
publishPayload = resolve;
});
const runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
connected: true,
currentVideoPath: '/video/show.mkv',
requestProperty: async (name) => (name === 'duration' ? 100 : name === 'pause' ? true : null),
send: () => undefined,
}),
getCurrentMediaPath: () => '/video/show.mkv',
getMpvExecutablePath: () => 'mpv',
generateWaveform: options.generateWaveform ?? (async () => []),
decisionTimeoutMs: options.decisionTimeoutMs,
createPreviewSession: () => ({
start: async () => undefined,
play: async (startTime, endTime) => {
previewCalls.push([startTime, endTime]);
await options.play?.();
},
stop: async () => undefined,
onPlaybackEnded: () => undefined,
dispose: () => undefined,
}),
openModal: async (payload) => {
publishPayload(payload);
return true;
},
showStatus: () => undefined,
});
const pendingDecision = runtime.requestReview({
kind: 'sentence',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0,
maxMediaDuration: options.maxMediaDuration ?? 30,
});
return { runtime, payload: await openedPayload, pendingDecision, previewCalls };
}
test('media timing review pauses playback, resolves exact timing, and restores playing state', async () => {
const commands: Array<Array<string | number>> = [];
const previewCalls: Array<[number, number]> = [];
let runtime: ReturnType<typeof createMediaTimingReviewRuntime>;
runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
connected: true,
currentVideoPath: '/video/show.mkv',
requestProperty: async (name) =>
({ pause: false, duration: 100, aid: 2, volume: 60 })[
name as 'pause' | 'duration' | 'aid' | 'volume'
],
send: ({ command }) => commands.push(command),
}),
getCurrentMediaPath: () => '/video/show.mkv',
getMpvExecutablePath: () => 'mpv',
generateWaveform: async () => [],
createPreviewSession: () => ({
start: async () => undefined,
play: async (startTime, endTime) => {
previewCalls.push([startTime, endTime]);
},
stop: async () => undefined,
onPlaybackEnded: () => undefined,
dispose: () => undefined,
}),
openModal: async (payload) => {
queueMicrotask(() => {
void runtime
.previewRange({
reviewId: payload.reviewId,
startTime: 9.5,
endTime: 12.5,
})
.then(() => {
runtime.resolveReview({
reviewId: payload.reviewId,
decision: { action: 'confirm', startTime: 9.5, endTime: 12.5 },
});
});
});
return true;
},
showStatus: () => undefined,
});
const decision = await runtime.requestReview({
kind: 'word',
text: '字幕',
startTime: 10,
endTime: 12,
noteId: 42,
audioPadding: 0.5,
maxMediaDuration: 30,
});
assert.deepEqual(decision, { action: 'confirm', startTime: 9.5, endTime: 12.5 });
assert.deepEqual(commands, [
['set_property', 'pause', 'yes'],
['set_property', 'pause', 'no'],
]);
assert.deepEqual(previewCalls, [[9.5, 12.5]]);
});
test('media timing review analyzes the visible range on the selected audio stream', async () => {
const waveformCalls: SpeechWaveformOptions[] = [];
let runtime: ReturnType<typeof createMediaTimingReviewRuntime>;
runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
connected: true,
currentVideoPath: '/video/show.mkv',
currentAudioStreamIndex: 4,
requestProperty: async (name) => (name === 'duration' ? 100 : name === 'pause' ? true : null),
send: () => undefined,
}),
getCurrentMediaPath: () => '/video/show.mkv',
getMpvExecutablePath: () => 'mpv',
generateWaveform: async (options) => {
waveformCalls.push(options);
return [0.1, 0.8, 0.2];
},
createPreviewSession: () => ({
start: async () => undefined,
play: async () => undefined,
stop: async () => undefined,
onPlaybackEnded: () => undefined,
dispose: () => undefined,
}),
openModal: async (payload) => {
const waveform = await runtime.getWaveform({
reviewId: payload.reviewId,
startTime: payload.timelineStartTime,
endTime: payload.timelineEndTime,
});
assert.deepEqual(waveform, { ok: true, peaks: [0.1, 0.8, 0.2] });
runtime.resolveReview({
reviewId: payload.reviewId,
decision: { action: 'use-original' },
});
return true;
},
showStatus: () => undefined,
});
await runtime.requestReview({
kind: 'sentence',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0.5,
maxMediaDuration: 30,
});
assert.deepEqual(waveformCalls, [
{
mediaPath: '/video/show.mkv',
startTime: 7.5,
endTime: 14.5,
audioStreamIndex: 4,
},
]);
});
const REMOTE_STREAM_URL = 'https://jellyfin.example/Videos/abc/stream?static=true';
function createWindowStub(options: { fail?: boolean } = {}) {
const calls: Array<{ source: RemoteMediaWindowSource; range: RemoteMediaWindowRange }> = [];
const acquireMediaWindow = async (
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow> => {
calls.push({ source, range });
if (options.fail) throw new Error('offline');
const windowPath = `/tmp/window-${range.startTime}-${range.endTime}.mkv`;
return {
path: windowPath,
startTime: range.startTime,
endTime: range.endTime,
sourcePath: source.path,
audioStreamIndex: source.audioStreamIndex ?? null,
media: {
path: windowPath,
source: 'remote-window',
singleResolvedStream: true,
absoluteTimestamps: true,
},
};
};
return { calls, acquireMediaWindow };
}
function createRemoteReviewRuntime(options: {
windowStub: ReturnType<typeof createWindowStub>;
waveformCalls: SpeechWaveformOptions[];
previewStarts: Array<Parameters<MediaTimingPreviewSessionLike['start']>[0]>;
previewPlays: Array<[string, number, number]>;
disposed: string[];
openModal: (
runtime: ReturnType<typeof createMediaTimingReviewRuntime>,
payload: MediaTimingReviewOpenPayload,
) => Promise<void>;
}) {
let runtime!: ReturnType<typeof createMediaTimingReviewRuntime>;
runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
connected: true,
currentVideoPath: REMOTE_STREAM_URL,
currentAudioStreamIndex: 2,
requestProperty: async (name) =>
({ pause: true, duration: 100, aid: 3, volume: 60 })[
name as 'pause' | 'duration' | 'aid' | 'volume'
] ?? null,
send: () => undefined,
}),
getCurrentMediaPath: () => REMOTE_STREAM_URL,
getMpvExecutablePath: () => 'mpv',
resolveMediaSource: async () => ({
path: REMOTE_STREAM_URL,
inputOptions: { reconnect: true },
}),
acquireMediaWindow: options.windowStub.acquireMediaWindow,
generateWaveform: async (waveformOptions) => {
options.waveformCalls.push(waveformOptions);
return [0.1, 0.8, 0.2];
},
createPreviewSession: () => {
let mediaPath = '';
return {
start: async (startOptions) => {
mediaPath = startOptions.mediaPath;
options.previewStarts.push(startOptions);
},
play: async (startTime, endTime) => {
options.previewPlays.push([mediaPath, startTime, endTime]);
},
stop: async () => undefined,
onPlaybackEnded: () => undefined,
dispose: () => {
options.disposed.push(mediaPath);
},
};
},
openModal: async (payload) => {
await options.openModal(runtime, payload);
return true;
},
showStatus: () => undefined,
});
return runtime;
}
test('media timing review downloads one window of a remote stream for the waveform and preview', async () => {
const windowStub = createWindowStub();
const waveformCalls: SpeechWaveformOptions[] = [];
const previewStarts: Array<Parameters<MediaTimingPreviewSessionLike['start']>[0]> = [];
const previewPlays: Array<[string, number, number]> = [];
const disposed: string[] = [];
const runtime = createRemoteReviewRuntime({
windowStub,
waveformCalls,
previewStarts,
previewPlays,
disposed,
openModal: async (active, payload) => {
const waveform = await active.getWaveform({
reviewId: payload.reviewId,
startTime: payload.timelineStartTime,
endTime: payload.timelineEndTime,
});
assert.deepEqual(waveform, { ok: true, peaks: [0.1, 0.8, 0.2] });
assert.deepEqual(
await active.previewRange({ reviewId: payload.reviewId, startTime: 9.5, endTime: 12.5 }),
{ ok: true },
);
active.resolveReview({
reviewId: payload.reviewId,
decision: { action: 'confirm', startTime: 9.5, endTime: 12.5 },
});
},
});
const decision = await runtime.requestReview({
kind: 'word',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0.5,
maxMediaDuration: 30,
});
assert.deepEqual(decision, { action: 'confirm', startTime: 9.5, endTime: 12.5 });
assert.deepEqual(windowStub.calls, [
{
source: { path: REMOTE_STREAM_URL, inputOptions: { reconnect: true }, audioStreamIndex: 2 },
range: { startTime: 7.5, endTime: 14.5 },
},
]);
assert.deepEqual(waveformCalls, [
{
mediaPath: {
path: '/tmp/window-7.5-14.5.mkv',
source: 'remote-window',
singleResolvedStream: true,
absoluteTimestamps: true,
},
startTime: 7.5,
endTime: 14.5,
},
]);
assert.deepEqual(previewStarts, [
{
mediaPath: '/tmp/window-7.5-14.5.mkv',
executablePath: 'mpv',
volume: 60,
absoluteTimestamps: true,
},
]);
assert.deepEqual(previewPlays, [['/tmp/window-7.5-14.5.mkv', 9.5, 12.5]]);
assert.deepEqual(disposed, ['/tmp/window-7.5-14.5.mkv']);
});
test('media timing review restarts the preview on a wider window when the timeline grows', async () => {
const windowStub = createWindowStub();
const waveformCalls: SpeechWaveformOptions[] = [];
const previewStarts: Array<Parameters<MediaTimingPreviewSessionLike['start']>[0]> = [];
const previewPlays: Array<[string, number, number]> = [];
const disposed: string[] = [];
const runtime = createRemoteReviewRuntime({
windowStub,
waveformCalls,
previewStarts,
previewPlays,
disposed,
openModal: async (active, payload) => {
await active.previewRange({ reviewId: payload.reviewId, startTime: 9.5, endTime: 12.5 });
// The user revealed two more seconds before the clip.
await active.getWaveform({ reviewId: payload.reviewId, startTime: 5.5, endTime: 14.5 });
await active.previewRange({ reviewId: payload.reviewId, startTime: 6, endTime: 12.5 });
active.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } });
},
});
await runtime.requestReview({
kind: 'sentence',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0.5,
maxMediaDuration: 30,
});
assert.deepEqual(
windowStub.calls.map((call) => call.range),
[
{ startTime: 7.5, endTime: 14.5 },
{ startTime: 5.5, endTime: 14.5 },
],
);
assert.deepEqual(
previewStarts.map((start) => start.mediaPath),
['/tmp/window-7.5-14.5.mkv', '/tmp/window-5.5-14.5.mkv'],
);
assert.deepEqual(previewPlays, [
['/tmp/window-7.5-14.5.mkv', 9.5, 12.5],
['/tmp/window-5.5-14.5.mkv', 6, 12.5],
]);
assert.deepEqual(disposed, ['/tmp/window-7.5-14.5.mkv', '/tmp/window-5.5-14.5.mkv']);
assert.equal(waveformCalls[0]?.startTime, 5.5);
});
test('media timing review falls back to the remote stream after one failed window download', async () => {
const windowStub = createWindowStub({ fail: true });
const waveformCalls: SpeechWaveformOptions[] = [];
const previewStarts: Array<Parameters<MediaTimingPreviewSessionLike['start']>[0]> = [];
const previewPlays: Array<[string, number, number]> = [];
const disposed: string[] = [];
const runtime = createRemoteReviewRuntime({
windowStub,
waveformCalls,
previewStarts,
previewPlays,
disposed,
openModal: async (active, payload) => {
await active.getWaveform({
reviewId: payload.reviewId,
startTime: payload.timelineStartTime,
endTime: payload.timelineEndTime,
});
await active.previewRange({ reviewId: payload.reviewId, startTime: 9.5, endTime: 12.5 });
active.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } });
},
});
await runtime.requestReview({
kind: 'word',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0.5,
maxMediaDuration: 30,
});
assert.equal(windowStub.calls.length, 1);
assert.deepEqual(waveformCalls, [
{
mediaPath: { path: REMOTE_STREAM_URL, inputOptions: { reconnect: true } },
startTime: 7.5,
endTime: 14.5,
audioStreamIndex: 2,
},
]);
assert.deepEqual(previewStarts, [
{ mediaPath: REMOTE_STREAM_URL, executablePath: 'mpv', volume: 60, audioTrackId: 3 },
]);
assert.deepEqual(previewPlays, [[REMOTE_STREAM_URL, 9.5, 12.5]]);
});
test('media timing review never downloads windows for local media', async () => {
const windowStub = createWindowStub();
let runtime!: ReturnType<typeof createMediaTimingReviewRuntime>;
runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
connected: true,
currentVideoPath: '/video/show.mkv',
requestProperty: async (name) => (name === 'duration' ? 100 : name === 'pause' ? true : null),
send: () => undefined,
}),
getCurrentMediaPath: () => '/video/show.mkv',
getMpvExecutablePath: () => 'mpv',
resolveMediaSource: async () => ({ path: '/video/show.mkv' }),
acquireMediaWindow: windowStub.acquireMediaWindow,
generateWaveform: async () => [0.1, 0.8, 0.2],
createPreviewSession: () => ({
start: async () => undefined,
play: async () => undefined,
stop: async () => undefined,
onPlaybackEnded: () => undefined,
dispose: () => undefined,
}),
openModal: async (payload) => {
await runtime.getWaveform({ reviewId: payload.reviewId, startTime: 7.5, endTime: 14.5 });
runtime.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } });
return true;
},
showStatus: () => undefined,
});
await runtime.requestReview({
kind: 'sentence',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0.5,
maxMediaDuration: 30,
});
assert.equal(windowStub.calls.length, 0);
});
test('media timing review rejects stale and out-of-range actions before allowing discard', async () => {
const { runtime, payload, pendingDecision, previewCalls } = await startActiveMediaTimingReview({
maxMediaDuration: 3,
});
assert.deepEqual(
await runtime.previewRange({ reviewId: 'stale-review', startTime: 10, endTime: 12 }),
{ ok: false, stale: true, message: 'This timing review is no longer active.' },
);
assert.deepEqual(
runtime.resolveReview({
reviewId: 'stale-review',
decision: { action: 'confirm', startTime: 10, endTime: 12 },
}),
{ ok: false, stale: true, message: 'This timing review is no longer active.' },
);
assert.deepEqual(
runtime.resolveReview({
reviewId: payload.reviewId,
decision: { action: 'confirm', startTime: 10, endTime: 14 },
}),
{ ok: false, message: 'The selected timing range is invalid.' },
);
assert.deepEqual(
runtime.resolveReview({
reviewId: payload.reviewId,
decision: { action: 'confirm', startTime: 99, endTime: 100.5 },
}),
{ ok: false, message: 'The selected timing range is invalid.' },
);
assert.deepEqual(
runtime.resolveReview({
reviewId: payload.reviewId,
decision: { action: 'confirm', startTime: 10, endTime: 12, text: ' ' },
}),
{ ok: false, message: 'The combined sentence text is invalid.' },
);
assert.deepEqual(
runtime.resolveReview({ reviewId: payload.reviewId, decision: { action: 'discard' } }),
{ ok: true },
);
assert.deepEqual(await pendingDecision, { action: 'discard' });
assert.deepEqual(previewCalls, []);
});
test('collectMediaTimingContextLines splits cues around the mined range', () => {
const cues = [
{ text: '一行目', startTime: 0, endTime: 2 },
{ text: '二行目', startTime: 2.5, endTime: 4 },
{ text: '', startTime: 4.2, endTime: 4.4 },
{ text: '採掘行', startTime: 5, endTime: 7 },
{ text: '四行目', startTime: 7.5, endTime: 9 },
{ text: '五行目', startTime: 9.5, endTime: 11 },
];
const context = collectMediaTimingContextLines({ cues, startTime: 5, endTime: 7 });
assert.deepEqual(context.previous, [
{ text: '一行目', startTime: 0, endTime: 2 },
{ text: '二行目', startTime: 2.5, endTime: 4 },
]);
assert.deepEqual(context.next, [
{ text: '四行目', startTime: 7.5, endTime: 9 },
{ text: '五行目', startTime: 9.5, endTime: 11 },
]);
});
test('collectMediaTimingContextLines falls back to played history when no cues are loaded', () => {
const context = collectMediaTimingContextLines({
cues: [],
fallbackPrevious: [
{ displayText: '前の行', startTime: 1, endTime: 2 },
{ displayText: '採掘行', startTime: 5, endTime: 7 },
],
startTime: 5,
endTime: 7,
});
assert.deepEqual(context.previous, [{ text: '前の行', startTime: 1, endTime: 2 }]);
assert.deepEqual(context.next, []);
});
test('media timing review watchdog falls back when the renderer stops responding', async () => {
const { pendingDecision } = await startActiveMediaTimingReview({ decisionTimeoutMs: 0 });
assert.deepEqual(await pendingDecision, { action: 'use-original' });
});
test('media timing review does not resume playback when the prior state is unavailable', async () => {
const commands: Array<Array<string | number>> = [];
let runtime: ReturnType<typeof createMediaTimingReviewRuntime>;
runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
connected: true,
currentVideoPath: '/video/show.mkv',
requestProperty: async () => null,
send: ({ command }) => commands.push(command),
}),
getCurrentMediaPath: () => '/video/show.mkv',
getMpvExecutablePath: () => '',
generateWaveform: async () => [],
createPreviewSession: () => ({
start: async () => {
throw new Error('preview unavailable');
},
play: async () => undefined,
stop: async () => undefined,
onPlaybackEnded: () => undefined,
dispose: () => undefined,
}),
openModal: async (payload) => {
queueMicrotask(() => {
runtime.resolveReview({
reviewId: payload.reviewId,
decision: { action: 'use-original' },
});
});
return true;
},
showStatus: () => undefined,
});
assert.deepEqual(
await runtime.requestReview({
kind: 'sentence',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0,
maxMediaDuration: 30,
}),
{ action: 'use-original' },
);
assert.deepEqual(commands, [['set_property', 'pause', 'yes']]);
});
test('media timing review restores playback when setup fails after pausing', async () => {
const commands: Array<Array<string | number>> = [];
const runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
connected: true,
currentVideoPath: '/video/show.mkv',
requestProperty: async (name) => (name === 'pause' ? false : null),
send: ({ command }) => commands.push(command),
}),
getCurrentMediaPath: () => '/video/show.mkv',
getMpvExecutablePath: () => {
throw new Error('preview setup failed');
},
generateWaveform: async () => [],
createPreviewSession: () => ({
start: async () => undefined,
play: async () => undefined,
stop: async () => undefined,
onPlaybackEnded: () => undefined,
dispose: () => undefined,
}),
openModal: async () => true,
showStatus: () => undefined,
});
assert.deepEqual(
await runtime.requestReview({
kind: 'word',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0,
maxMediaDuration: 30,
}),
{ action: 'use-original' },
);
assert.deepEqual(commands, [
['set_property', 'pause', 'yes'],
['set_property', 'pause', 'no'],
]);
});
test('disposing an open review settles it with original timing and restores playback', async () => {
const commands: Array<Array<string | number>> = [];
const runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
connected: true,
currentVideoPath: '/video/show.mkv',
requestProperty: async (name) => (name === 'pause' ? false : null),
send: ({ command }) => commands.push(command),
}),
getCurrentMediaPath: () => '/video/show.mkv',
getMpvExecutablePath: () => 'mpv',
generateWaveform: async () => [],
createPreviewSession: () => ({
start: async () => undefined,
play: async () => undefined,
stop: async () => undefined,
onPlaybackEnded: () => undefined,
dispose: () => undefined,
}),
openModal: async () => true,
showStatus: () => undefined,
});
const pending = runtime.requestReview({
kind: 'word',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0,
maxMediaDuration: 30,
});
await new Promise<void>((resolve) => setImmediate(resolve));
await runtime.dispose();
assert.deepEqual(await pending, { action: 'use-original' });
assert.deepEqual(commands, [
['set_property', 'pause', 'yes'],
['set_property', 'pause', 'no'],
]);
});
test('media timing review forwards the hidden player finishing a preview to the modal', async () => {
const endedReviewIds: string[] = [];
const playback: { ended?: () => void } = {};
let publishPayload!: (payload: MediaTimingReviewOpenPayload) => void;
const openedPayload = new Promise<MediaTimingReviewOpenPayload>((resolve) => {
publishPayload = resolve;
});
const runtime = createMediaTimingReviewRuntime({
getMpvClient: () => ({
connected: true,
currentVideoPath: '/video/show.mkv',
requestProperty: async (name) => (name === 'duration' ? 100 : null),
send: () => undefined,
}),
getCurrentMediaPath: () => '/video/show.mkv',
getMpvExecutablePath: () => 'mpv',
generateWaveform: async () => [],
createPreviewSession: () => ({
start: async () => undefined,
play: async () => undefined,
stop: async () => undefined,
onPlaybackEnded: (listener) => {
playback.ended = listener;
},
dispose: () => undefined,
}),
openModal: async (payload) => {
publishPayload(payload);
return true;
},
onPreviewEnded: (reviewId) => {
endedReviewIds.push(reviewId);
},
showStatus: () => undefined,
});
const pendingDecision = runtime.requestReview({
kind: 'sentence',
text: '字幕',
startTime: 10,
endTime: 12,
audioPadding: 0,
maxMediaDuration: 30,
});
const payload = await openedPayload;
assert.deepEqual(
await runtime.previewRange({ reviewId: payload.reviewId, startTime: 10, endTime: 12 }),
{
ok: true,
},
);
assert.ok(playback.ended);
playback.ended();
assert.deepEqual(endedReviewIds, [payload.reviewId]);
runtime.resolveReview({ reviewId: payload.reviewId, decision: { action: 'use-original' } });
await pendingDecision;
playback.ended();
assert.deepEqual(endedReviewIds, [payload.reviewId]);
});
test('preview reports a stale review when the review ends during playback', async () => {
let endReview: (() => Promise<void>) | null = null;
const { runtime, payload, pendingDecision } = await startActiveMediaTimingReview({
play: async () => {
await endReview?.();
},
});
endReview = () => runtime.dispose();
assert.deepEqual(
await runtime.previewRange({ reviewId: payload.reviewId, startTime: 10, endTime: 12 }),
{ ok: false, stale: true, message: 'This timing review is no longer active.' },
);
await pendingDecision;
});
test('waveform reports a stale review when the review ends during analysis', async () => {
let endReview: (() => Promise<void>) | null = null;
const { runtime, payload, pendingDecision } = await startActiveMediaTimingReview({
generateWaveform: async () => {
await endReview?.();
return [0.1, 0.9, 0.2];
},
});
endReview = () => runtime.dispose();
assert.deepEqual(
await runtime.getWaveform({ reviewId: payload.reviewId, startTime: 8, endTime: 14 }),
{ ok: false, stale: true, message: 'This timing review is no longer active.' },
);
await pendingDecision;
});
+589
View File
@@ -0,0 +1,589 @@
import { randomUUID } from 'crypto';
import type {
MediaTimingReviewActionResult,
MediaTimingReviewContextLine,
MediaTimingReviewDecision,
MediaTimingReviewOpenPayload,
MediaTimingReviewPreviewRequest,
MediaTimingReviewRequest,
MediaTimingReviewResolveRequest,
MediaTimingReviewWaveformRequest,
MediaTimingReviewWaveformResult,
} from '../../types/anki';
import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform';
import {
isRemoteMediaWindowSourcePath,
type RemoteMediaWindow,
type RemoteMediaWindowRange,
type RemoteMediaWindowSource,
} from '../../core/services/remote-media-window-cache';
import type { MediaInput, MediaInputOptions } from '../../media-input';
const INITIAL_TIMELINE_MARGIN_SECONDS = 2;
const REVIEW_DECISION_TIMEOUT_MS = 5 * 60_000;
const CONTEXT_LINE_LIMIT = 12;
const CONTEXT_LINE_EPSILON_SECONDS = 0.05;
interface ReviewMpvClient {
connected: boolean;
currentVideoPath: string;
currentAudioStreamIndex?: number | null;
requestProperty?: (name: string) => Promise<unknown>;
send: (payload: { command: Array<string | number> }) => void;
}
interface PreviewSession {
start(options: {
mediaPath: string;
executablePath?: string;
audioTrackId?: number;
volume?: number;
absoluteTimestamps?: boolean;
}): Promise<void>;
play(startTime: number, endTime: number): Promise<void>;
stop(): Promise<void>;
/** Fires when the player reaches the end of the clip started by play(). */
onPlaybackEnded(listener: () => void): void;
dispose(): void;
}
interface ReviewMediaSource {
path: string;
inputOptions?: MediaInputOptions;
singleResolvedStream?: boolean;
}
interface ActiveReview {
payload: MediaTimingReviewOpenPayload;
/** What the hidden mpv preview plays when no cached window is available. */
mediaPath: string;
/** What the waveform reads when no cached window is available. */
waveformMedia: MediaInput;
audioStreamIndex?: number;
/** Remote source to download windows of; null for local media or without a cache. */
windowSource: RemoteMediaWindowSource | null;
/** Latest window returned for this review; reused while it still covers the request. */
window: RemoteMediaWindow | null;
windowRequest: (RemoteMediaWindowRange & { promise: Promise<RemoteMediaWindow | null> }) | null;
windowFailed: boolean;
previewOptions: { executablePath?: string; audioTrackId?: number; volume?: number };
preview: { path: string; session: Promise<PreviewSession> } | null;
mpvClient: ReviewMpvClient;
restorePlayback: boolean;
resolve: (decision: MediaTimingReviewDecision) => void;
}
export interface MediaTimingReviewRuntimeDeps {
getMpvClient: () => ReviewMpvClient | null;
getCurrentMediaPath: () => string | null;
getMpvExecutablePath: () => string;
createPreviewSession: () => PreviewSession;
generateWaveform: (options: SpeechWaveformOptions) => Promise<number[]>;
/** Resolves the FFmpeg-readable stream URL and headers behind the current media path. */
resolveMediaSource?: () => Promise<ReviewMediaSource | null>;
/** Downloads (or reuses) a local window of a remote source covering the range. */
acquireMediaWindow?: (
source: RemoteMediaWindowSource,
range: RemoteMediaWindowRange,
) => Promise<RemoteMediaWindow>;
getSubtitleContextLines?: (range: { startTime: number; endTime: number }) => {
previous: MediaTimingReviewContextLine[];
next: MediaTimingReviewContextLine[];
};
decisionTimeoutMs?: number;
openModal: (payload: MediaTimingReviewOpenPayload) => Promise<boolean>;
/** Tells the modal that the hidden player finished the previewed clip. */
onPreviewEnded?: (reviewId: string) => void;
showStatus: (message: string) => void;
}
function finiteNumber(value: unknown): number | null {
return typeof value === 'number' && Number.isFinite(value) ? value : null;
}
function booleanProperty(value: unknown): boolean | null {
if (typeof value === 'boolean') return value;
if (value === 'yes' || value === 1) return true;
if (value === 'no' || value === 0) return false;
return null;
}
/**
* Picks the subtitle lines adjacent to the mined range that the review modal can pull
* onto the card. Parsed cues cover both directions; when none are loaded (e.g. the
* active track was never parsed) the timing tracker's history still provides the
* lines that already played, so only "next" is unavailable.
*/
export function collectMediaTimingContextLines(options: {
cues: readonly { text: string; startTime: number; endTime: number }[];
fallbackPrevious?: readonly { displayText: string; startTime: number; endTime: number }[];
startTime: number;
endTime: number;
}): { previous: MediaTimingReviewContextLine[]; next: MediaTimingReviewContextLine[] } {
const usable = options.cues
.filter(
(cue) =>
cue.text.trim().length > 0 &&
Number.isFinite(cue.startTime) &&
Number.isFinite(cue.endTime) &&
cue.endTime > cue.startTime,
)
.sort((a, b) => a.startTime - b.startTime || a.endTime - b.endTime);
let previous = usable
.filter((cue) => cue.endTime <= options.startTime + CONTEXT_LINE_EPSILON_SECONDS)
.slice(-CONTEXT_LINE_LIMIT)
.map(({ text, startTime, endTime }) => ({ text: text.trim(), startTime, endTime }));
const next = usable
.filter((cue) => cue.startTime >= options.endTime - CONTEXT_LINE_EPSILON_SECONDS)
.slice(0, CONTEXT_LINE_LIMIT)
.map(({ text, startTime, endTime }) => ({ text: text.trim(), startTime, endTime }));
if (previous.length === 0 && options.fallbackPrevious) {
previous = options.fallbackPrevious
.filter(
(entry) =>
entry.displayText.trim().length > 0 &&
Number.isFinite(entry.startTime) &&
Number.isFinite(entry.endTime) &&
entry.endTime > entry.startTime &&
entry.endTime <= options.startTime + CONTEXT_LINE_EPSILON_SECONDS,
)
.slice(-CONTEXT_LINE_LIMIT)
.map((entry) => ({
text: entry.displayText.trim(),
startTime: entry.startTime,
endTime: entry.endTime,
}));
}
return { previous, next };
}
/**
* Result for requests that name a review main has already resolved or disposed (decision
* watchdog, overlay teardown, duplicate modal). The renderer closes on it instead of
* leaving the user with controls that can never succeed.
*/
function staleReviewResult(): MediaTimingReviewActionResult {
return { ok: false, stale: true, message: 'This timing review is no longer active.' };
}
function isValidMediaTimingRange(
payload: MediaTimingReviewOpenPayload,
startTime: number,
endTime: number,
): boolean {
return (
Number.isFinite(startTime) &&
Number.isFinite(endTime) &&
startTime >= 0 &&
endTime > startTime &&
(payload.maxMediaDuration <= 0 || endTime - startTime <= payload.maxMediaDuration + 0.001) &&
(payload.mediaDuration === undefined || endTime <= payload.mediaDuration + 0.001)
);
}
export function buildMediaTimingReviewPayload(
request: MediaTimingReviewRequest,
options: {
reviewId: string;
mediaDuration?: number;
contextLines?: {
previous: MediaTimingReviewContextLine[];
next: MediaTimingReviewContextLine[];
};
},
): MediaTimingReviewOpenPayload {
const duration = finiteNumber(options.mediaDuration);
const maxTime = duration !== null && duration > 0 ? duration : Number.POSITIVE_INFINITY;
const paddedStart = Math.max(0, request.startTime - request.audioPadding);
let paddedEnd = Math.min(maxTime, request.endTime + request.audioPadding);
const maxMediaDuration = Math.max(0, request.maxMediaDuration);
if (maxMediaDuration > 0 && paddedEnd - paddedStart > maxMediaDuration) {
paddedEnd = paddedStart + maxMediaDuration;
}
if (paddedEnd <= paddedStart) {
paddedEnd = Math.min(maxTime, paddedStart + 0.1);
}
const timelineStartTime = Math.max(0, paddedStart - INITIAL_TIMELINE_MARGIN_SECONDS);
const timelineEndTime = Math.max(
paddedEnd,
Math.min(maxTime, paddedEnd + INITIAL_TIMELINE_MARGIN_SECONDS),
);
return {
reviewId: options.reviewId,
kind: request.kind,
text: request.text,
previousLines: options.contextLines?.previous ?? [],
nextLines: options.contextLines?.next ?? [],
...(request.noteId !== undefined ? { noteId: request.noteId } : {}),
originalStartTime: request.startTime,
originalEndTime: request.endTime,
selectionStartTime: paddedStart,
selectionEndTime: paddedEnd,
timelineStartTime,
timelineEndTime,
...(duration !== null && duration > 0 ? { mediaDuration: duration } : {}),
maxMediaDuration,
};
}
export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDeps) {
let active: ActiveReview | null = null;
let reviewInProgress = false;
let pendingPauseRestore: ReviewMpvClient | null = null;
function restorePendingPlayback(): void {
const mpvClient = pendingPauseRestore;
pendingPauseRestore = null;
if (mpvClient?.connected) {
mpvClient.send({ command: ['set_property', 'pause', 'no'] });
}
}
function ensureWindow(
review: ActiveReview,
range: RemoteMediaWindowRange,
): Promise<RemoteMediaWindow | null> {
const { windowSource } = review;
if (!windowSource || review.windowFailed || !deps.acquireMediaWindow) {
return Promise.resolve(null);
}
const coversRange = (candidate: RemoteMediaWindowRange): boolean =>
candidate.startTime <= range.startTime && candidate.endTime >= range.endTime;
if (review.window && coversRange(review.window)) return Promise.resolve(review.window);
const inFlight = review.windowRequest;
if (inFlight && coversRange(inFlight)) return inFlight.promise;
const request = {
startTime: range.startTime,
endTime: range.endTime,
promise: Promise.resolve<RemoteMediaWindow | null>(null),
};
request.promise = deps
.acquireMediaWindow(windowSource, { startTime: range.startTime, endTime: range.endTime })
.then((window) => {
review.window = window;
return window;
})
.catch(() => {
// Fall back to the remote source for the rest of this review instead of retrying.
review.windowFailed = true;
return null;
})
.finally(() => {
if (review.windowRequest === request) review.windowRequest = null;
});
review.windowRequest = request;
return request.promise;
}
/**
* Returns the preview player for the range, restarting it when the range needs a
* different file (the first cached window, or a wider one after the timeline grew).
*/
async function previewFor(
review: ActiveReview,
range: RemoteMediaWindowRange,
): Promise<PreviewSession> {
const window = await ensureWindow(review, range);
if (active !== review) {
// The review ended during the download; do not start a player nobody will dispose.
throw new Error('This timing review is no longer active.');
}
const mediaPath = window?.path ?? review.mediaPath;
if (review.preview?.path === mediaPath) return review.preview.session;
const previous = review.preview;
const session = deps.createPreviewSession();
session.onPlaybackEnded(() => {
if (active === review && review.preview?.session === started) {
deps.onPreviewEnded?.(review.payload.reviewId);
}
});
const { audioTrackId, ...previewOptions } = review.previewOptions;
const started = session
.start({
mediaPath,
...previewOptions,
// A cached window keeps one audio stream, so mpv's track id from the source no longer applies.
...(window
? { absoluteTimestamps: true }
: audioTrackId !== undefined
? { audioTrackId }
: {}),
})
.then(() => session)
.catch((error) => {
session.dispose();
throw error;
});
review.preview = { path: mediaPath, session: started };
void started.catch(() => {});
if (previous) void previous.session.then((old) => old.dispose()).catch(() => {});
return started;
}
async function runReview(request: MediaTimingReviewRequest): Promise<MediaTimingReviewDecision> {
const mpvClient = deps.getMpvClient();
const mediaPath =
deps.getCurrentMediaPath()?.trim() || mpvClient?.currentVideoPath?.trim() || '';
if (!mpvClient?.connected || !mediaPath) {
deps.showStatus('Timing review unavailable. Using the original subtitle timing.');
return { action: 'use-original' };
}
const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw, resolvedSource] = await Promise.all([
mpvClient.requestProperty?.('pause').catch(() => null) ?? null,
mpvClient.requestProperty?.('duration').catch(() => null) ?? null,
mpvClient.requestProperty?.('aid').catch(() => null) ?? null,
mpvClient.requestProperty?.('volume').catch(() => null) ?? null,
deps.resolveMediaSource?.().catch(() => null) ?? null,
]);
const pauseState = booleanProperty(pauseRaw);
mpvClient.send({ command: ['set_property', 'pause', 'yes'] });
pendingPauseRestore = pauseState === false ? mpvClient : null;
let contextLines: ReturnType<NonNullable<typeof deps.getSubtitleContextLines>> | undefined;
try {
contextLines = deps.getSubtitleContextLines?.({
startTime: request.startTime,
endTime: request.endTime,
});
} catch {
contextLines = undefined;
}
const payload = buildMediaTimingReviewPayload(request, {
reviewId: randomUUID(),
mediaDuration: finiteNumber(durationRaw) ?? undefined,
...(contextLines ? { contextLines } : {}),
});
const sourcePath = resolvedSource?.path.trim() || mediaPath;
const inputOptions = resolvedSource?.inputOptions;
const audioStreamIndex =
resolvedSource?.singleResolvedStream || mpvClient.currentAudioStreamIndex == null
? undefined
: mpvClient.currentAudioStreamIndex;
const windowSource: RemoteMediaWindowSource | null =
deps.acquireMediaWindow && isRemoteMediaWindowSourcePath(sourcePath)
? {
path: sourcePath,
...(inputOptions ? { inputOptions } : {}),
audioStreamIndex: audioStreamIndex ?? null,
}
: null;
let resolveDecision!: (decision: MediaTimingReviewDecision) => void;
const decisionPromise = new Promise<MediaTimingReviewDecision>((resolve) => {
resolveDecision = resolve;
});
const review: ActiveReview = {
payload,
mediaPath,
waveformMedia: inputOptions ? { path: sourcePath, inputOptions } : sourcePath,
...(audioStreamIndex !== undefined ? { audioStreamIndex } : {}),
windowSource,
window: null,
windowRequest: null,
windowFailed: false,
previewOptions: {
executablePath: deps.getMpvExecutablePath(),
audioTrackId: finiteNumber(audioTrackRaw) ?? undefined,
volume: finiteNumber(volumeRaw) ?? undefined,
},
preview: null,
mpvClient,
restorePlayback: pendingPauseRestore === mpvClient,
resolve: resolveDecision,
};
active = review;
pendingPauseRestore = null;
// Download the visible timeline once now; the waveform and preview both wait on it.
void previewFor(review, {
startTime: payload.timelineStartTime,
endTime: payload.timelineEndTime,
}).catch(() => {});
const opened = await deps.openModal(payload).catch(() => false);
if (!opened) {
await cleanupActiveReview();
deps.showStatus('Timing review could not open. Using the original subtitle timing.');
return { action: 'use-original' };
}
const decisionWatchdog = setTimeout(
() => resolveDecision({ action: 'use-original' }),
Math.max(0, deps.decisionTimeoutMs ?? REVIEW_DECISION_TIMEOUT_MS),
);
let decision: MediaTimingReviewDecision;
try {
decision = await decisionPromise;
} finally {
clearTimeout(decisionWatchdog);
}
await cleanupActiveReview();
return decision;
}
async function requestReview(
request: MediaTimingReviewRequest,
): Promise<MediaTimingReviewDecision> {
if (active || reviewInProgress) {
deps.showStatus('Finish the current timing review before mining another card.');
return { action: 'use-original' };
}
reviewInProgress = true;
try {
return await runReview(request);
} catch {
await cleanupActiveReview();
restorePendingPlayback();
deps.showStatus('Timing review failed. Using the original subtitle timing.');
return { action: 'use-original' };
} finally {
reviewInProgress = false;
}
}
async function previewRange(
request: MediaTimingReviewPreviewRequest,
): Promise<MediaTimingReviewActionResult> {
const current = active;
if (!current || request.reviewId !== current.payload.reviewId) {
return staleReviewResult();
}
if (!isValidMediaTimingRange(current.payload, request.startTime, request.endTime)) {
return { ok: false, message: 'The selected preview range is invalid.' };
}
try {
const previewSession = await previewFor(current, request);
if (active !== current) {
return staleReviewResult();
}
await previewSession.play(request.startTime, request.endTime);
// Playback spans the whole clip, so the review can end (watchdog, teardown) while
// it runs; reporting success would leave the modal open on a dead review.
if (active !== current) {
return staleReviewResult();
}
return { ok: true };
} catch (error) {
if (active !== current) {
return staleReviewResult();
}
return {
ok: false,
message: `Audio preview unavailable: ${error instanceof Error ? error.message : String(error)}`,
};
}
}
async function getWaveform(
request: MediaTimingReviewWaveformRequest,
): Promise<MediaTimingReviewWaveformResult> {
const current = active;
if (!current || request.reviewId !== current.payload.reviewId) {
return staleReviewResult();
}
if (
!Number.isFinite(request.startTime) ||
!Number.isFinite(request.endTime) ||
request.startTime < 0 ||
request.endTime <= request.startTime ||
(current.payload.mediaDuration !== undefined &&
request.endTime > current.payload.mediaDuration + 0.001)
) {
return { ok: false, message: 'The waveform range is invalid.' };
}
try {
const window = await ensureWindow(current, request);
if (active !== current) {
return staleReviewResult();
}
const peaks = await deps.generateWaveform({
mediaPath: window?.media ?? current.waveformMedia,
startTime: request.startTime,
endTime: request.endTime,
...(!window && current.audioStreamIndex !== undefined
? { audioStreamIndex: current.audioStreamIndex }
: {}),
});
// ffmpeg decoding runs long enough for the review to end underneath it.
if (active !== current) {
return staleReviewResult();
}
if (peaks.length < 2 || peaks.some((peak) => !Number.isFinite(peak))) {
return { ok: false, message: 'Timing waveform is unavailable.' };
}
return { ok: true, peaks };
} catch {
if (active !== current) {
return staleReviewResult();
}
return { ok: false, message: 'Timing waveform is unavailable.' };
}
}
async function stopPreview(reviewId: string): Promise<MediaTimingReviewActionResult> {
const current = active;
if (!current || reviewId !== current.payload.reviewId) {
return staleReviewResult();
}
try {
const previewSession = current.preview ? await current.preview.session : null;
await previewSession?.stop();
return { ok: true };
} catch (error) {
return {
ok: false,
message: `Could not stop preview: ${error instanceof Error ? error.message : String(error)}`,
};
}
}
function resolveReview(request: MediaTimingReviewResolveRequest): MediaTimingReviewActionResult {
const current = active;
if (!current || request.reviewId !== current.payload.reviewId) {
return staleReviewResult();
}
if (request.decision.action === 'confirm') {
const { startTime, endTime, text } = request.decision;
if (!isValidMediaTimingRange(current.payload, startTime, endTime)) {
return { ok: false, message: 'The selected timing range is invalid.' };
}
if (text !== undefined && (typeof text !== 'string' || text.trim().length === 0)) {
return { ok: false, message: 'The combined sentence text is invalid.' };
}
}
current.resolve(request.decision);
return { ok: true };
}
async function cleanupActiveReview(): Promise<void> {
const current = active;
active = null;
if (!current) return;
void current.preview?.session.then((session) => session.dispose()).catch(() => {});
if (current.restorePlayback && current.mpvClient.connected) {
current.mpvClient.send({ command: ['set_property', 'pause', 'no'] });
}
}
async function dispose(): Promise<void> {
active?.resolve({ action: 'use-original' });
await cleanupActiveReview();
restorePendingPlayback();
}
return {
requestReview,
previewRange,
getWaveform,
stopPreview,
resolveReview,
dispose,
};
}
@@ -3,6 +3,7 @@ import test from 'node:test';
import { parseSubtitleCues } from '../../core/services/subtitle-cue-parser';
import {
resolveCanonicalPrimarySubtitle,
resolvePrimarySubtitle,
resolvePrimarySubtitleText,
stripCanonicalFragmentLines,
} from './primary-subtitle-text';
@@ -702,3 +703,37 @@ test('resolvePrimarySubtitleText publishes a wrapped caption sentence as one cue
'(東)≪好きだと\n自覚してしまったものの➡',
);
});
test('resolvePrimarySubtitle drops a finished caption row lingering beside a fresh line', () => {
// Broadcast captions give each row its own event, and a row of the previous line can
// outlive its siblings by a frame. mpv's sub-text still lists it, so the mined line
// must come from the parsed cue that is actually running, with that cue's timings.
const ass = [
'[Script Info]',
'PlayResY: 540',
'',
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:14:30.00,0:14:33.00,Default,,0,0,0,,{\\pos(232,437)\\fscx50}{\\fscx100}東{\\fscx50}{\\fscx100}ずっと 言えなかっ',
'Dialogue: 0,0:14:30.00,0:14:33.02,Default,,0,0,0,,{\\pos(232,497)}たが',
'Dialogue: 0,0:14:33.00,0:14:36.00,Default,,0,0,0,,{\\pos(232,437)}⸨もし お互い',
'Dialogue: 0,0:14:33.00,0:14:36.00,Default,,0,0,0,,{\\pos(232,497)}本命 受かったら 大学 近いし➡',
].join('\n');
const cues = parseSubtitleCues(ass, 'polar-opposites-s02e09.ass');
const resolved = resolvePrimarySubtitle({
liveText: 'たが\n⸨もし お互い\n本命 受かったら 大学 近いし➡',
currentTimeSec: 14 * 60 + 33.05,
cues,
});
assert.deepEqual(
{ ...resolved, cues: resolved?.cues.map((cue) => cue.text) },
{
text: '⸨もし お互い\n本命 受かったら 大学 近いし➡',
startTime: 14 * 60 + 33,
endTime: 14 * 60 + 36,
cues: ['⸨もし お互い\n本命 受かったら 大学 近いし➡'],
},
);
});
+21 -9
View File
@@ -319,6 +319,26 @@ export function resolveRecordedPrimarySubtitleText(options: {
);
}
/**
* The parsed view of the live text with its cue timings: a canonical animation when one
* explains the live lines, otherwise the active parsed cues. Null when the parsed cues
* cannot account for every live line, in which case callers keep the raw mpv text.
*/
export function resolvePrimarySubtitle(options: {
liveText: string;
currentTimeSec: number;
cues: readonly SubtitleCue[] | null | undefined;
}): ResolvedPrimarySubtitle | null {
const liveText = decodedLiveText(options.liveText, options.cues);
if (!liveText.trim()) {
return null;
}
return (
resolveCanonicalPrimarySubtitle({ ...options, liveText }) ??
resolveActiveParsedPrimarySubtitle({ ...options, liveText })
);
}
export function resolvePrimarySubtitleText(options: {
liveText: string;
currentTimeSec: number;
@@ -328,13 +348,5 @@ export function resolvePrimarySubtitleText(options: {
if (!liveText.trim()) {
return liveText;
}
return (
resolveCanonicalPrimarySubtitle({
liveText,
currentTimeSec: options.currentTimeSec,
cues: options.cues,
})?.text ??
resolveActiveParsedPrimarySubtitle({ ...options, liveText })?.text ??
removeLiveGlyphFragmentLines(liveText)
);
return resolvePrimarySubtitle(options)?.text ?? removeLiveGlyphFragmentLines(liveText);
}
+141 -31
View File
@@ -10,6 +10,9 @@ import {
MediaGenerator,
type MediaGeneratorOptions,
} from './media-generator';
import { RemoteMediaWindowCache } from './core/services/remote-media-window-cache';
const REMOTE_STREAM_URL = 'https://jellyfin.example/Videos/abc/stream?static=true';
async function withStubbedFfmpeg(
run: (generator: MediaGenerator, argsPath: string) => Promise<void>,
@@ -21,6 +24,7 @@ async function withStubbedFfmpeg(
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-media-generator-test-'));
const binDir = path.join(root, 'bin');
const tempDir = path.join(root, 'media');
const windowsDir = path.join(root, 'windows');
const argsPath = path.join(root, 'ffmpeg-args.txt');
fs.mkdirSync(binDir, { recursive: true });
const ffmpegStubPath = path.join(binDir, 'ffmpeg-stub.cjs');
@@ -34,7 +38,7 @@ async function withStubbedFfmpeg(
" console.log(' V..... libaom-av1');",
' process.exit(0);',
'}',
"fs.writeFileSync(process.env.SUBMINER_TEST_FFMPEG_ARGS, JSON.stringify(args), 'utf8');",
"fs.appendFileSync(process.env.SUBMINER_TEST_FFMPEG_ARGS, JSON.stringify(args) + '\\n', 'utf8');",
'const outputPath = args.at(-1);',
"if (process.env.SUBMINER_TEST_FFMPEG_SKIP_OUTPUT !== '1') {",
" fs.writeFileSync(outputPath, 'avif', 'utf8');",
@@ -61,12 +65,18 @@ async function withStubbedFfmpeg(
} else {
delete process.env.SUBMINER_TEST_FFMPEG_SKIP_OUTPUT;
}
const generator = new MediaGenerator(tempDir, options);
// Each test gets its own window cache so remote inputs never leak windows between tests.
const remoteMediaWindows = new RemoteMediaWindowCache({ tempDir: windowsDir, idleTtlMs: 0 });
const generator = new MediaGenerator(tempDir, {
remoteMediaWindows,
...options,
});
try {
await run(generator, argsPath);
} finally {
generator.cleanup();
remoteMediaWindows.cleanup();
process.env.PATH = originalPath;
if (originalArgsPath === undefined) {
delete process.env.SUBMINER_TEST_FFMPEG_ARGS;
@@ -82,8 +92,17 @@ async function withStubbedFfmpeg(
}
}
function readAllFfmpegArgs(argsPath: string): string[][] {
return fs
.readFileSync(argsPath, 'utf8')
.split('\n')
.filter((line) => line.trim().length > 0)
.map((line) => JSON.parse(line) as string[]);
}
/** Arguments of the most recent ffmpeg invocation. */
function readFfmpegArgs(argsPath: string): string[] {
return JSON.parse(fs.readFileSync(argsPath, 'utf8')) as string[];
return readAllFfmpegArgs(argsPath).at(-1) ?? [];
}
test('buildAnimatedImageVideoFilter holds lead-in until the next frame after the audio boundary', () => {
@@ -272,41 +291,131 @@ test('generateAudio recreates missing temp directory before invoking ffmpeg', as
});
test('generateAudio adds remote input options before the ffmpeg input', async () => {
await withStubbedFfmpeg(async (generator, argsPath) => {
await generator.generateAudio(
{
path: 'https://rr1---sn.example.googlevideo.com/videoplayback?mime=audio%2Fwebm',
inputOptions: {
reconnect: true,
userAgent: 'Mozilla/5.0',
headers: {
Referer: 'https://www.youtube.com/',
Origin: 'https://www.youtube.com',
await withStubbedFfmpeg(
async (generator, argsPath) => {
await generator.generateAudio(
{
path: 'https://rr1---sn.example.googlevideo.com/videoplayback?mime=audio%2Fwebm',
inputOptions: {
reconnect: true,
userAgent: 'Mozilla/5.0',
headers: {
Referer: 'https://www.youtube.com/',
Origin: 'https://www.youtube.com',
},
},
},
},
10,
12,
);
const args = readFfmpegArgs(argsPath);
const inputIndex = args.indexOf('-i');
assert.ok(inputIndex > 0);
assert.ok(args.indexOf('-reconnect') > -1);
assert.ok(args.indexOf('-reconnect') < inputIndex);
assert.equal(args[args.indexOf('-reconnect') + 1], '1');
assert.equal(args[args.indexOf('-reconnect_streamed') + 1], '1');
assert.equal(args[args.indexOf('-reconnect_on_network_error') + 1], '1');
assert.equal(args[args.indexOf('-reconnect_on_http_error') + 1], '403,5xx');
assert.equal(args[args.indexOf('-reconnect_delay_max') + 1], '5');
assert.equal(args[args.indexOf('-user_agent') + 1], 'Mozilla/5.0');
assert.equal(
args[args.indexOf('-headers') + 1],
'Referer: https://www.youtube.com/\r\nOrigin: https://www.youtube.com\r\n',
);
},
{ remoteMediaWindows: null },
);
});
test('generateAudio downloads a remote window once and extracts from it with absolute seeks', async () => {
await withStubbedFfmpeg(async (generator, argsPath) => {
await generator.generateAudio(
{ path: REMOTE_STREAM_URL, inputOptions: { reconnect: true } },
10,
12,
0.5,
2,
);
const args = readFfmpegArgs(argsPath);
const inputIndex = args.indexOf('-i');
assert.ok(inputIndex > 0);
assert.ok(args.indexOf('-reconnect') > -1);
assert.ok(args.indexOf('-reconnect') < inputIndex);
assert.equal(args[args.indexOf('-reconnect') + 1], '1');
assert.equal(args[args.indexOf('-reconnect_streamed') + 1], '1');
assert.equal(args[args.indexOf('-reconnect_on_network_error') + 1], '1');
assert.equal(args[args.indexOf('-reconnect_on_http_error') + 1], '403,5xx');
assert.equal(args[args.indexOf('-reconnect_delay_max') + 1], '5');
assert.equal(args[args.indexOf('-user_agent') + 1], 'Mozilla/5.0');
assert.equal(
args[args.indexOf('-headers') + 1],
'Referer: https://www.youtube.com/\r\nOrigin: https://www.youtube.com\r\n',
);
const calls = readAllFfmpegArgs(argsPath);
assert.equal(calls.length, 2);
const [fetchArgs, audioArgs] = calls as [string[], string[]];
assert.equal(fetchArgs[fetchArgs.indexOf('-i') + 1], REMOTE_STREAM_URL);
assert.ok(fetchArgs.indexOf('-reconnect') < fetchArgs.indexOf('-i'));
assert.equal(fetchArgs[fetchArgs.indexOf('-ss') + 1], '9.25');
assert.equal(fetchArgs[fetchArgs.lastIndexOf('-map') + 1], '0:2');
assert.ok(fetchArgs.includes('-copyts'));
const windowPath = audioArgs[audioArgs.indexOf('-i') + 1];
assert.ok(windowPath?.endsWith('.mkv'));
assert.notEqual(windowPath, REMOTE_STREAM_URL);
assert.equal(audioArgs[audioArgs.indexOf('-ss') + 1], '9.5');
assert.equal(audioArgs[audioArgs.indexOf('-seek_timestamp') + 1], '1');
assert.ok(audioArgs.indexOf('-seek_timestamp') < audioArgs.indexOf('-i'));
assert.equal(audioArgs.includes('-reconnect'), false);
assert.equal(audioArgs.includes('-map'), false);
assert.equal(audioArgs.includes('-probesize'), false);
assert.ok(audioArgs.includes('loudnorm=I=-23:TP=-2:LRA=11'));
});
});
test('generateScreenshot reuses a downloaded window but never downloads one itself', async () => {
await withStubbedFfmpeg(async (generator, argsPath) => {
await generator.generateScreenshot(REMOTE_STREAM_URL, 11, { format: 'jpg' });
let calls = readAllFfmpegArgs(argsPath);
assert.equal(calls.length, 1);
assert.equal(calls[0]![calls[0]!.indexOf('-i') + 1], REMOTE_STREAM_URL);
await generator.generateAudio(REMOTE_STREAM_URL, 10, 12);
await generator.generateScreenshot(REMOTE_STREAM_URL, 11, { format: 'jpg' });
await generator.generateScreenshot(REMOTE_STREAM_URL, 40, { format: 'jpg' });
calls = readAllFfmpegArgs(argsPath);
assert.equal(calls.length, 5);
const insideWindow = calls[3]!;
assert.ok(insideWindow[insideWindow.indexOf('-i') + 1]?.endsWith('.mkv'));
assert.equal(insideWindow[insideWindow.indexOf('-seek_timestamp') + 1], '1');
const outsideWindow = calls[4]!;
assert.equal(outsideWindow[outsideWindow.indexOf('-i') + 1], REMOTE_STREAM_URL);
});
});
test('generateAnimatedImage downloads the clip window before encoding', async () => {
await withStubbedFfmpeg(async (generator, argsPath) => {
await generator.generateAnimatedImage(REMOTE_STREAM_URL, 10, 12, 0, { fps: 10 });
const calls = readAllFfmpegArgs(argsPath).filter(
(args) => args[0] !== '-hide_banner' || args[1] !== '-encoders',
);
assert.equal(calls.length, 2);
assert.equal(calls[0]![calls[0]!.indexOf('-i') + 1], REMOTE_STREAM_URL);
assert.ok(calls[1]![calls[1]!.indexOf('-i') + 1]?.endsWith('.mkv'));
assert.equal(calls[1]![calls[1]!.indexOf('-seek_timestamp') + 1], '1');
});
});
test('generateAudio reads the remote source directly when the window download fails', async () => {
await withStubbedFfmpeg(
async (generator, argsPath) => {
await generator.generateAudio(REMOTE_STREAM_URL, 10, 12);
const args = readFfmpegArgs(argsPath);
assert.equal(args[args.indexOf('-i') + 1], REMOTE_STREAM_URL);
assert.equal(args.includes('-seek_timestamp'), false);
},
{
remoteMediaWindows: new RemoteMediaWindowCache({
execFile: (_file, _args, _options, callback) =>
queueMicrotask(() => callback(Object.assign(new Error('offline'), { code: 1 }))),
idleTtlMs: 0,
logDebug: () => undefined,
}),
},
);
});
test('generateAudio skips stale audio stream maps for single resolved streams', async () => {
await withStubbedFfmpeg(async (generator, argsPath) => {
await generator.generateAudio(
@@ -320,8 +429,9 @@ test('generateAudio skips stale audio stream maps for single resolved streams',
22,
);
const args = readFfmpegArgs(argsPath);
assert.equal(args.includes('-map'), false);
const [fetchArgs, audioArgs] = readAllFfmpegArgs(argsPath) as [string[], string[]];
assert.equal(fetchArgs[fetchArgs.lastIndexOf('-map') + 1], '0:a');
assert.equal(audioArgs.includes('-map'), false);
});
});
+84 -3
View File
@@ -22,6 +22,12 @@ import * as path from 'path';
import * as os from 'os';
import { createLogger } from './logger';
import { normalizeMediaInput, type MediaInput } from './media-input';
import {
getSharedRemoteMediaWindowCache,
isRemoteMediaWindowSourcePath,
type RemoteMediaWindowCache,
type RemoteMediaWindowRange,
} from './core/services/remote-media-window-cache';
const log = createLogger('media');
const AUDIO_NORMALIZATION_FILTER = 'loudnorm=I=-23:TP=-2:LRA=11';
@@ -86,6 +92,11 @@ export interface MediaGeneratorOptions {
logDebug?: (message: string) => void;
now?: () => number;
execFile?: MediaGeneratorExecFile;
/**
* Local window cache for http(s) sources. Defaults to the process-wide cache shared
* with the timing review; pass `null` to always read remote sources directly.
*/
remoteMediaWindows?: RemoteMediaWindowCache | null;
}
function sanitizeDebugToken(value: string, fallback: string): string {
@@ -232,6 +243,54 @@ export class MediaGenerator {
}, delayMs);
}
/**
* Swaps an http(s) input for the locally cached window that covers `range`, so the
* clip is downloaded once instead of per FFmpeg run. `acquire` downloads on a miss;
* `lookup` only reuses a window that another step already fetched. Any failure falls
* back to reading the remote source directly.
*/
private async resolveRemoteWindowInput(
input: MediaInput,
range: RemoteMediaWindowRange,
audioStreamIndex: number | null | undefined,
mode: 'acquire' | 'lookup',
): Promise<MediaInput> {
const cache =
this.options.remoteMediaWindows === undefined
? getSharedRemoteMediaWindowCache()
: this.options.remoteMediaWindows;
const sourcePath = typeof input === 'string' ? input : input.path;
if (!cache || !isRemoteMediaWindowSourcePath(sourcePath)) {
return input;
}
const source = {
path: sourcePath,
...(typeof input === 'object' && input.inputOptions
? { inputOptions: input.inputOptions }
: {}),
audioStreamIndex:
typeof input === 'object' && input.singleResolvedStream ? null : (audioStreamIndex ?? null),
};
const description = describeMediaInputForDebugLog(input);
try {
const window =
mode === 'acquire' ? await cache.acquire(source, range) : await cache.lookup(source, range);
if (!window) {
this.logMediaDebug(`window miss ${description} mode=${mode}`);
return input;
}
this.logMediaDebug(
`window hit ${description} mode=${mode} start=${window.startTime} end=${window.endTime}`,
);
return window.media;
} catch (error) {
this.logMediaDebug(
`window failed ${description} mode=${mode} reason=${sanitizeDebugToken((error as Error).message, 'error')}`,
);
return input;
}
}
private ffmpegError(label: string, error: ExecFileException): Error {
if (error.code === 'ENOENT') {
return new Error('FFmpeg not found. Install FFmpeg to enable media generation.');
@@ -281,7 +340,13 @@ export class MediaGenerator {
const safePadding = Number.isFinite(padding) ? Math.max(0, padding) : 0;
const start = Math.max(0, startTime - safePadding);
const duration = endTime - start + safePadding;
const mediaInput = normalizeMediaInput(videoPath);
const sourceInput = await this.resolveRemoteWindowInput(
videoPath,
{ startTime: start, endTime: start + duration },
audioStreamIndex,
'acquire',
);
const mediaInput = normalizeMediaInput(sourceInput);
const inputDescription = describeMediaInputForDebugLog(videoPath);
const hasSelectedAudioStream =
!mediaInput.singleResolvedStream &&
@@ -385,7 +450,15 @@ export class MediaGenerator {
png: 'png',
webp: 'webp',
};
const mediaInput = normalizeMediaInput(videoPath);
// A single frame is cheap to fetch remotely, so only reuse a window another step downloaded.
const mediaInput = normalizeMediaInput(
await this.resolveRemoteWindowInput(
videoPath,
{ startTime: timestamp, endTime: timestamp },
null,
'lookup',
),
);
const inputDescription = describeMediaInputForDebugLog(videoPath);
const args: string[] = [
@@ -533,9 +606,17 @@ export class MediaGenerator {
);
}
const mediaInput = normalizeMediaInput(
await this.resolveRemoteWindowInput(
videoPath,
{ startTime: start, endTime: start + duration },
null,
'acquire',
),
);
return new Promise((resolve, reject) => {
const outputPath = this.createTempOutputPath('animation', 'avif');
const mediaInput = normalizeMediaInput(videoPath);
const startedAt = this.nowMs();
const encoderArgs: string[] = ['-c:v', av1Encoder];
+10
View File
@@ -11,6 +11,12 @@ export type MediaInput =
source?: string;
inputOptions?: MediaInputOptions;
singleResolvedStream?: boolean;
/**
* The file keeps the original media timestamps instead of starting at zero (a
* stream-copied window of a longer source). Seek with `-ss` against those
* absolute timestamps rather than relative to the file's own start time.
*/
absoluteTimestamps?: boolean;
};
export type NormalizedMediaInput = {
@@ -89,6 +95,10 @@ export function normalizeMediaInput(input: MediaInput): NormalizedMediaInput {
inputArgs.push('-headers', headers);
}
if (input.absoluteTimestamps) {
inputArgs.push('-seek_timestamp', '1');
}
return {
path: input.path,
inputArgs,
+28
View File
@@ -69,6 +69,11 @@ import type {
OverlayNotificationEventPayload,
OverlayNotificationPosition,
ChangelogSnapshot,
MediaTimingReviewActionResult,
MediaTimingReviewOpenPayload,
MediaTimingReviewPreviewRequest,
MediaTimingReviewResolveRequest,
MediaTimingReviewWaveformRequest,
} from './types';
import { IPC_CHANNELS } from './shared/ipc/contracts';
@@ -181,6 +186,15 @@ const onOpenYoutubeTrackPickerEvent = createQueuedIpcListenerWithPayload<Youtube
IPC_CHANNELS.event.youtubePickerOpen,
(payload) => payload as YoutubePickerOpenPayload,
);
const onOpenMediaTimingReviewEvent =
createQueuedIpcListenerWithPayload<MediaTimingReviewOpenPayload>(
IPC_CHANNELS.event.mediaTimingReviewOpen,
(payload) => payload as MediaTimingReviewOpenPayload,
);
const onMediaTimingReviewPreviewEndedEvent = createQueuedIpcListenerWithPayload<string>(
IPC_CHANNELS.event.mediaTimingReviewPreviewEnded,
(payload) => (typeof payload === 'string' ? payload : ''),
);
const onOpenPlaylistBrowserEvent = createQueuedIpcListener(IPC_CHANNELS.event.playlistBrowserOpen);
const onCancelYoutubeTrackPickerEvent = createQueuedIpcListener(
IPC_CHANNELS.event.youtubePickerCancel,
@@ -458,6 +472,20 @@ const electronAPI: ElectronAPI = {
onOpenJimaku: onOpenJimakuEvent,
onOpenTsukihime: onOpenTsukihimeEvent,
onOpenYoutubeTrackPicker: onOpenYoutubeTrackPickerEvent,
onOpenMediaTimingReview: onOpenMediaTimingReviewEvent,
onMediaTimingReviewPreviewEnded: onMediaTimingReviewPreviewEndedEvent,
previewMediaTimingReview: (
request: MediaTimingReviewPreviewRequest,
): Promise<MediaTimingReviewActionResult> =>
ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewPreview, request),
getMediaTimingReviewWaveform: (request: MediaTimingReviewWaveformRequest) =>
ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewWaveform, request),
stopMediaTimingReviewPreview: (reviewId: string): Promise<MediaTimingReviewActionResult> =>
ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewStopPreview, reviewId),
resolveMediaTimingReview: (
request: MediaTimingReviewResolveRequest,
): Promise<MediaTimingReviewActionResult> =>
ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewResolve, request),
onOpenPlaylistBrowser: onOpenPlaylistBrowserEvent,
onOpenCharacterDictionaryManager: onOpenCharacterDictionaryManagerEvent,
onSubtitleSidebarToggle: onSubtitleSidebarToggleEvent,
+29
View File
@@ -452,6 +452,7 @@ function createKeyboardHandlerHarness() {
const testGlobals = installKeyboardTestGlobals();
const subtitleRootClassList = createClassList();
const subtitleContainerClassList = createClassList();
let mediaTimingReviewKeydownCount = 0;
let controllerSelectKeydownCount = 0;
let openControllerSelectCount = 0;
let openControllerDebugCount = 0;
@@ -494,6 +495,10 @@ function createKeyboardHandlerHarness() {
handleKikuKeydown: () => false,
handleJimakuKeydown: () => false,
handleTsukihimeKeydown: () => false,
handleMediaTimingReviewKeydown: () => {
mediaTimingReviewKeydownCount += 1;
return false;
},
handleControllerSelectKeydown: () => {
controllerSelectKeydownCount += 1;
return true;
@@ -523,6 +528,7 @@ function createKeyboardHandlerHarness() {
ctx,
handlers,
testGlobals,
mediaTimingReviewKeydownCount: () => mediaTimingReviewKeydownCount,
controllerSelectKeydownCount: () => controllerSelectKeydownCount,
openControllerSelectCount: () => openControllerSelectCount,
openControllerDebugCount: () => openControllerDebugCount,
@@ -1367,6 +1373,29 @@ test('keyboard mode: controller select modal handles arrow keys before yomitan p
}
});
test('media timing review modal handles keys before later modal handlers', async () => {
const {
ctx,
testGlobals,
handlers,
mediaTimingReviewKeydownCount,
controllerSelectKeydownCount,
} = createKeyboardHandlerHarness();
try {
await handlers.setupMpvInputForwarding();
ctx.state.mediaTimingReviewModalOpen = true;
ctx.state.controllerSelectModalOpen = true;
testGlobals.dispatchKeydown({ key: 'ArrowDown', code: 'ArrowDown' });
assert.equal(mediaTimingReviewKeydownCount(), 1);
assert.equal(controllerSelectKeydownCount(), 0);
} finally {
testGlobals.restore();
}
});
test('keyboard mode: playlist browser modal handles arrow keys before yomitan popup', async () => {
const { ctx, testGlobals, handlers, playlistBrowserKeydownCount } =
createKeyboardHandlerHarness();
+6
View File
@@ -18,6 +18,7 @@ export function createKeyboardHandlers(
handleJimakuKeydown: (e: KeyboardEvent) => boolean;
handleTsukihimeKeydown: (e: KeyboardEvent) => boolean;
handleYoutubePickerKeydown: (e: KeyboardEvent) => boolean;
handleMediaTimingReviewKeydown: (e: KeyboardEvent) => boolean;
handlePlaylistBrowserKeydown: (e: KeyboardEvent) => boolean;
handleControllerSelectKeydown: (e: KeyboardEvent) => boolean;
handleControllerDebugKeydown: (e: KeyboardEvent) => boolean;
@@ -1078,6 +1079,11 @@ export function createKeyboardHandlers(
);
document.addEventListener('keydown', (e: KeyboardEvent) => {
if (ctx.state.mediaTimingReviewModalOpen) {
options.handleMediaTimingReviewKeydown(e);
return;
}
if (isKeyboardDrivenModeToggle(e) && ctx.platform.isModalLayer) {
e.preventDefault();
handleKeyboardModeToggleRequested();
+294
View File
@@ -196,6 +196,300 @@
</div>
</div>
</div>
<div
id="mediaTimingReviewModal"
class="modal media-timing-review-modal hidden"
aria-hidden="true"
>
<div
class="modal-content media-timing-review-content"
role="dialog"
aria-modal="true"
aria-labelledby="mediaTimingReviewTitle"
>
<div class="media-timing-review-header">
<div class="media-timing-review-heading">
<div id="mediaTimingReviewTitle" class="media-timing-review-title">
Review media timing
</div>
<div id="mediaTimingReviewKind" class="media-timing-review-kind">Sentence card</div>
</div>
<button
id="mediaTimingReviewCancel"
class="media-timing-review-quiet-button"
type="button"
>
Cancel
</button>
</div>
<div id="mediaTimingReviewEditor" class="media-timing-review-editor">
<div class="media-timing-review-sentence-header">
<div class="media-timing-review-sentence-heading">
<span class="media-timing-review-sentence-label">Sentence on card</span>
<span id="mediaTimingReviewLineCount" class="media-timing-review-line-count">
1 line
</span>
</div>
<div
id="mediaTimingReviewLineControls"
class="media-timing-review-line-controls hidden"
>
<div class="media-timing-review-line-stepper">
<span>Prev</span>
<button
id="mediaTimingReviewPrevRemove"
type="button"
aria-label="Remove the earliest added previous subtitle line from the card sentence"
title="Remove the earliest previous line (Shift+P)"
>
&minus;
</button>
<button
id="mediaTimingReviewPrevAdd"
type="button"
aria-label="Add the previous subtitle line to the card sentence"
title="Add the previous line (P)"
>
+
</button>
</div>
<div class="media-timing-review-line-stepper">
<span>Next</span>
<button
id="mediaTimingReviewNextRemove"
type="button"
aria-label="Remove the latest added next subtitle line from the card sentence"
title="Remove the latest next line (Shift+N)"
>
&minus;
</button>
<button
id="mediaTimingReviewNextAdd"
type="button"
aria-label="Add the next subtitle line to the card sentence"
title="Add the next line (N)"
>
+
</button>
</div>
</div>
</div>
<blockquote id="mediaTimingReviewText" class="media-timing-review-text"></blockquote>
<div class="media-timing-review-readout" aria-live="polite">
<div>
<span>Starts</span>
<strong id="mediaTimingReviewStartValue">00:00.000</strong>
</div>
<div class="media-timing-review-duration">
<span>Clip length</span>
<strong id="mediaTimingReviewDuration">0.00s</strong>
</div>
<div>
<span>Ends</span>
<strong id="mediaTimingReviewEndValue">00:00.000</strong>
</div>
</div>
<div class="media-timing-review-timeline-shell">
<div class="media-timing-review-timeline-labels">
<span id="mediaTimingReviewTimelineStart">00:00</span>
<span id="mediaTimingReviewWaveformLabel">speech-weighted waveform</span>
<span id="mediaTimingReviewTimelineEnd">00:00</span>
</div>
<div id="mediaTimingReviewSelectionTrack" class="media-timing-review-track">
<svg
class="media-timing-review-waveform"
viewBox="0 0 1000 100"
preserveAspectRatio="none"
aria-hidden="true"
>
<defs>
<linearGradient id="mediaTimingReviewWaveformFill" x1="0" y1="0" x2="0" y2="1">
<stop class="media-timing-review-waveform-edge-stop" offset="0%" />
<stop class="media-timing-review-waveform-core-stop" offset="50%" />
<stop class="media-timing-review-waveform-edge-stop" offset="100%" />
</linearGradient>
</defs>
<path id="mediaTimingReviewWaveformPath"></path>
</svg>
<div
class="media-timing-review-original-range"
title="Mined subtitle timing"
aria-hidden="true"
>
<span class="media-timing-review-original-boundary is-start"></span>
<span class="media-timing-review-original-boundary is-end"></span>
</div>
<span class="media-timing-review-original-label is-start" aria-hidden="true">
Line start
</span>
<span class="media-timing-review-original-label is-end" aria-hidden="true">
Line end
</span>
<div
id="mediaTimingReviewSelectedRange"
class="media-timing-review-selected-range"
aria-hidden="true"
></div>
<div class="media-timing-review-playhead" aria-hidden="true"></div>
<div
id="mediaTimingReviewStartHandle"
class="media-timing-review-handle media-timing-review-handle-start"
role="slider"
tabindex="0"
aria-label="Clip start"
aria-orientation="horizontal"
></div>
<div
id="mediaTimingReviewEndHandle"
class="media-timing-review-handle media-timing-review-handle-end"
role="slider"
tabindex="0"
aria-label="Clip end"
aria-orientation="horizontal"
></div>
</div>
<div class="media-timing-review-expand-row">
<button
id="mediaTimingReviewShowEarlier"
class="media-timing-review-expand-button"
type="button"
aria-label="Show two more seconds before the visible timeline without moving the selected clip"
title="Show 2 more seconds before the visible timeline. The selected clip does not move."
>
Earlier &minus;2s
</button>
<span>Drag an edge to trim, or drag the highlighted clip to move it.</span>
<button
id="mediaTimingReviewShowLater"
class="media-timing-review-expand-button"
type="button"
aria-label="Show two more seconds after the visible timeline without moving the selected clip"
title="Show 2 more seconds after the visible timeline. The selected clip does not move."
>
Later +2s
</button>
</div>
</div>
<div class="media-timing-review-fine-grid">
<div class="media-timing-review-fine-control">
<span>Start</span>
<button
id="mediaTimingReviewStartBack"
type="button"
aria-label="Move start backward 100 milliseconds"
>
&minus;0.1s
</button>
<button
id="mediaTimingReviewStartForward"
type="button"
aria-label="Move start forward 100 milliseconds"
>
+0.1s
</button>
</div>
<div class="media-timing-review-fine-control">
<span>End</span>
<button
id="mediaTimingReviewEndBack"
type="button"
aria-label="Move end backward 100 milliseconds"
>
&minus;0.1s
</button>
<button
id="mediaTimingReviewEndForward"
type="button"
aria-label="Move end forward 100 milliseconds"
>
+0.1s
</button>
</div>
</div>
<div
id="mediaTimingReviewStatus"
class="media-timing-review-status"
role="status"
aria-live="polite"
></div>
<div class="media-timing-review-footer">
<div class="media-timing-review-preview-actions">
<button
id="mediaTimingReviewPlay"
class="media-timing-review-play-button"
type="button"
>
<span class="media-timing-review-play-glyph" aria-hidden="true"></span>
<span id="mediaTimingReviewPlayLabel">Play selection</span>
</button>
<button
id="mediaTimingReviewReset"
class="media-timing-review-quiet-button"
type="button"
>
Reset timing
</button>
</div>
<button
id="mediaTimingReviewConfirm"
class="media-timing-review-confirm-button"
type="button"
>
Use this timing
</button>
</div>
<div class="media-timing-review-hints" aria-hidden="true">
<span><kbd>Space</kbd> preview</span>
<span><kbd>&larr;</kbd><kbd>&rarr;</kbd> nudge focused edge</span>
<span><kbd>P</kbd>/<kbd>N</kbd> add prev/next line</span>
<span><kbd>Enter</kbd> confirm</span>
<span><kbd>Esc</kbd> cancel</span>
</div>
</div>
<div id="mediaTimingReviewCancelStep" class="media-timing-review-cancel-step hidden">
<div class="media-timing-review-cancel-mark" aria-hidden="true">?</div>
<h2>Stop reviewing?</h2>
<p id="mediaTimingReviewCancelMessage">Choose what should happen to this card.</p>
<div class="media-timing-review-cancel-actions">
<button
id="mediaTimingReviewCancelBack"
class="media-timing-review-quiet-button"
type="button"
>
Keep editing
</button>
<button
id="mediaTimingReviewUseOriginal"
class="media-timing-review-original-button"
type="button"
>
Use original timing
</button>
<button
id="mediaTimingReviewSkipMedia"
class="media-timing-review-skip-button"
type="button"
title="Keep this card but do not add audio or an image."
>
Keep without media
</button>
<button
id="mediaTimingReviewDiscard"
class="media-timing-review-discard-button"
type="button"
>
Delete card
</button>
</div>
</div>
</div>
</div>
<div id="kikuFieldGroupingModal" class="modal hidden" aria-hidden="true">
<div class="modal-content">
<div class="modal-header">
@@ -0,0 +1,157 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
buildMediaTimingLineSelection,
buildMediaTimingWaveformPath,
constrainMediaTimingSelection,
createMediaTimingPreviewRequestGuard,
formatMediaTimingTimestamp,
mediaTimingTimeFromPointer,
slideMediaTimingSelection,
trimMediaTimingSelectionEnd,
} from './media-timing-review';
test('waveform path mirrors normalized peaks around its center line', () => {
const path = buildMediaTimingWaveformPath([0, 0.5, 1]);
assert.match(path, /^M 0\.00 50\.00 L 500\.00 28\.00 L 1000\.00 6\.00/);
assert.match(path, /1000\.00 94\.00 L 500\.00 72\.00 L 0\.00 50\.00 Z$/);
assert.equal(buildMediaTimingWaveformPath([1]), '');
});
test('formatMediaTimingTimestamp renders stable review readouts', () => {
assert.equal(formatMediaTimingTimestamp(65.4321), '01:05.432');
assert.equal(formatMediaTimingTimestamp(65.4321, false), '01:05');
assert.equal(formatMediaTimingTimestamp(-1), '00:00.000');
assert.equal(formatMediaTimingTimestamp(119.9999), '02:00.000');
assert.equal(formatMediaTimingTimestamp(119.6, false), '02:00');
});
test('selection constraints preserve the handle that did not move', () => {
assert.deepEqual(
constrainMediaTimingSelection({
nextStart: 3,
nextEnd: 2,
currentStart: 1,
timelineStart: 0,
timelineEnd: 10,
mediaEnd: 10,
maxMediaDuration: 30,
}),
{ start: 1.9, end: 2 },
);
assert.deepEqual(
constrainMediaTimingSelection({
nextStart: 1,
nextEnd: 0.5,
currentStart: 1,
timelineStart: 0,
timelineEnd: 10,
mediaEnd: 10,
maxMediaDuration: 30,
}),
{ start: 1, end: 1.1 },
);
});
test('pointer positions map onto the visible timeline and clamp past its edges', () => {
const track = { trackLeft: 100, trackWidth: 400, timelineStart: 10, timelineEnd: 20 };
assert.equal(mediaTimingTimeFromPointer({ clientX: 100, ...track }), 10);
assert.equal(mediaTimingTimeFromPointer({ clientX: 300, ...track }), 15);
assert.equal(mediaTimingTimeFromPointer({ clientX: -40, ...track }), 10);
assert.equal(mediaTimingTimeFromPointer({ clientX: 900, ...track }), 20);
assert.equal(mediaTimingTimeFromPointer({ ...track, clientX: 300, trackWidth: 0 }), 10);
});
test('sliding keeps the clip length and stops at the timeline and media bounds', () => {
const timeline = { span: 2, timelineStart: 4, timelineEnd: 12, mediaEnd: 10 };
assert.deepEqual(slideMediaTimingSelection({ nextStart: 6, ...timeline }), { start: 6, end: 8 });
assert.deepEqual(slideMediaTimingSelection({ nextStart: 1, ...timeline }), { start: 4, end: 6 });
assert.deepEqual(slideMediaTimingSelection({ nextStart: 99, ...timeline }), {
start: 8,
end: 10,
});
});
test('line selection combines adjacent lines around the mined one and tracks their range', () => {
const base = {
previousLines: [
{ text: '一行目', startTime: 0, endTime: 2 },
{ text: '二行目', startTime: 2.5, endTime: 4 },
],
nextLines: [
{ text: '四行目', startTime: 7.5, endTime: 9 },
{ text: '五行目', startTime: 9.5, endTime: 11 },
],
text: '採掘行',
originalStartTime: 5,
originalEndTime: 7,
};
const none = buildMediaTimingLineSelection({ ...base, previousCount: 0, nextCount: 0 });
assert.deepEqual(none.lineTexts, ['採掘行']);
assert.equal(none.currentLineIndex, 0);
assert.equal(none.rangeStart, 5);
assert.equal(none.rangeEnd, 7);
const expanded = buildMediaTimingLineSelection({ ...base, previousCount: 1, nextCount: 2 });
assert.deepEqual(expanded.lineTexts, ['二行目', '採掘行', '四行目', '五行目']);
assert.equal(expanded.currentLineIndex, 1);
assert.equal(expanded.sentence, '二行目 採掘行 四行目 五行目');
assert.equal(expanded.rangeStart, 2.5);
assert.equal(expanded.rangeEnd, 11);
});
test('preview request guard blocks overlap and invalidates stale responses', () => {
const guard = createMediaTimingPreviewRequestGuard();
const first = guard.begin();
assert.equal(typeof first, 'number');
assert.equal(guard.begin(), null);
assert.equal(guard.isCurrent(first!), true);
guard.invalidate();
assert.equal(guard.isCurrent(first!), false);
assert.equal(guard.isInFlight(), false);
const second = guard.begin();
assert.equal(typeof second, 'number');
guard.finish(first!);
assert.equal(guard.isCurrent(second!), true);
guard.finish(second!);
assert.equal(guard.isInFlight(), false);
});
test('trailing silence trim follows the last speech slice and keeps cut-off or silent lines', () => {
// 10 slices over a 10 s timeline: one slice per second, speech in seconds 2-4 only.
const peaks = [0, 0, 0.9, 0.8, 0.7, 0.1, 0.2, 0, 0, 0];
const base = { peaks, timelineStart: 0, timelineEnd: 10, lineStart: 2, lineEnd: 8 };
const trimmed = trimMediaTimingSelectionEnd({ ...base, selectionEnd: 8, endPadSeconds: 0 });
assert.ok(trimmed !== null && Math.abs(trimmed - 5.15) < 1e-9);
const padded = trimMediaTimingSelectionEnd({ ...base, selectionEnd: 8.5, endPadSeconds: 0.5 });
assert.ok(padded !== null && Math.abs(padded - 5.65) < 1e-9);
// Speech running through the line end means the subtitle cuts the audio off; keep it.
assert.equal(
trimMediaTimingSelectionEnd({ ...base, lineEnd: 5, selectionEnd: 5, endPadSeconds: 0 }),
null,
);
// No speech inside the line at all: keep the subtitle timing rather than guess.
assert.equal(
trimMediaTimingSelectionEnd({ ...base, lineStart: 6, selectionEnd: 8, endPadSeconds: 0 }),
null,
);
// A saving below the minimum trim is not worth moving the handle for.
assert.equal(
trimMediaTimingSelectionEnd({ ...base, lineEnd: 5.2, selectionEnd: 5.2, endPadSeconds: 0 }),
null,
);
assert.equal(
trimMediaTimingSelectionEnd({ ...base, peaks: [1], selectionEnd: 8, endPadSeconds: 0 }),
null,
);
});
+960
View File
@@ -0,0 +1,960 @@
import type {
MediaTimingReviewContextLine,
MediaTimingReviewDecision,
MediaTimingReviewOpenPayload,
} from '../../types/anki';
import type { ModalStateReader, RendererContext } from '../context';
import { createModalFocusGuard } from './modal-focus-guard';
const MINIMUM_CLIP_SECONDS = 0.1;
const FINE_ADJUST_SECONDS = 0.1;
const COARSE_ADJUST_SECONDS = 0.5;
const TIMELINE_EXPANSION_SECONDS = 2;
const LINE_REVEAL_MARGIN_SECONDS = 1;
/**
* Subtitles usually linger past the dialogue for readability, so an untouched clip end
* follows the last speech-weighted waveform slice above this level (0..1, relative to the
* clip's own noise floor) plus a short tail, when that saves at least the minimum trim.
*/
const SPEECH_LEVEL_THRESHOLD = 0.3;
const SPEECH_TAIL_SECONDS = 0.15;
const MINIMUM_TRAILING_TRIM_SECONDS = 0.1;
/** Slack past the clip length before the UI gives up waiting for mpv's end-of-clip signal. */
const PREVIEW_END_GRACE_MS = 2_500;
function clamp(value: number, minimum: number, maximum: number): number {
return Math.min(maximum, Math.max(minimum, value));
}
export function formatMediaTimingTimestamp(seconds: number, includeMilliseconds = true): string {
const unitsPerSecond = includeMilliseconds ? 1_000 : 1;
const totalUnits = Math.max(0, Math.round(seconds * unitsPerSecond));
const unitsPerMinute = 60 * unitsPerSecond;
const minutes = Math.floor(totalUnits / unitsPerMinute);
const remainingUnits = totalUnits % unitsPerMinute;
const remaining = includeMilliseconds
? (remainingUnits / unitsPerSecond).toFixed(3)
: String(remainingUnits);
return `${String(minutes).padStart(2, '0')}:${remaining.padStart(includeMilliseconds ? 6 : 2, '0')}`;
}
/**
* Resolves which subtitle lines the card sentence currently includes. The counts say
* how many adjacent lines were pulled in on each side; the range covers those lines'
* subtitle timings so the clip edges can follow them.
*/
export function buildMediaTimingLineSelection(options: {
previousLines: MediaTimingReviewContextLine[];
nextLines: MediaTimingReviewContextLine[];
text: string;
originalStartTime: number;
originalEndTime: number;
previousCount: number;
nextCount: number;
}): {
lineTexts: string[];
currentLineIndex: number;
sentence: string;
rangeStart: number;
rangeEnd: number;
} {
const previous =
options.previousCount > 0 ? options.previousLines.slice(-options.previousCount) : [];
const next = options.nextCount > 0 ? options.nextLines.slice(0, options.nextCount) : [];
const lineTexts = [
...previous.map((line) => line.text),
options.text,
...next.map((line) => line.text),
];
return {
lineTexts,
currentLineIndex: previous.length,
sentence: lineTexts.join(' '),
rangeStart: previous[0]?.startTime ?? options.originalStartTime,
rangeEnd: next[next.length - 1]?.endTime ?? options.originalEndTime,
};
}
export function createMediaTimingPreviewRequestGuard() {
let sequence = 0;
let activeRequestId: number | null = null;
return {
begin(): number | null {
if (activeRequestId !== null) return null;
sequence += 1;
activeRequestId = sequence;
return activeRequestId;
},
invalidate(): void {
sequence += 1;
activeRequestId = null;
},
isCurrent(requestId: number): boolean {
return requestId === activeRequestId;
},
finish(requestId: number): void {
if (activeRequestId === requestId) activeRequestId = null;
},
isInFlight(): boolean {
return activeRequestId !== null;
},
};
}
export function buildMediaTimingWaveformPath(peaks: number[]): string {
if (peaks.length < 2) return '';
const points = peaks.map((peak, index) => ({
x: (index / (peaks.length - 1)) * 1_000,
amplitude: clamp(Number.isFinite(peak) ? peak : 0, 0, 1) * 44,
}));
const upper = points.map(({ x, amplitude }) => `${x.toFixed(2)} ${(50 - amplitude).toFixed(2)}`);
const lower = [...points]
.reverse()
.map(({ x, amplitude }) => `${x.toFixed(2)} ${(50 + amplitude).toFixed(2)}`);
return `M ${upper.join(' L ')} L ${lower.join(' L ')} Z`;
}
/**
* Where an untouched clip should end once the waveform is known: just after the line's
* last speech slice, plus the configured end padding. Null keeps the subtitle timing when
* no speech shows inside the line, when speech runs through the line end (the subtitle is
* cutting the audio off, not lingering), or when the saving is too small to matter.
*/
export function trimMediaTimingSelectionEnd(options: {
peaks: readonly number[];
timelineStart: number;
timelineEnd: number;
lineStart: number;
lineEnd: number;
selectionEnd: number;
endPadSeconds: number;
}): number | null {
const pointCount = options.peaks.length;
const span = options.timelineEnd - options.timelineStart;
if (pointCount < 2 || span <= 0 || options.lineEnd <= options.lineStart) return null;
// Point i covers [i, i + 1) / pointCount of the timeline (see computeWaveformPeaks).
const sliceEnd = (index: number): number =>
options.timelineStart + ((index + 1) / pointCount) * span;
const firstIndex = Math.max(
0,
Math.floor(((options.lineStart - options.timelineStart) / span) * pointCount),
);
const lastIndex = Math.min(
pointCount - 1,
Math.ceil(((options.lineEnd - options.timelineStart) / span) * pointCount) - 1,
);
let lastSpeechIndex = -1;
for (let index = lastIndex; index >= firstIndex; index -= 1) {
if ((options.peaks[index] ?? 0) >= SPEECH_LEVEL_THRESHOLD) {
lastSpeechIndex = index;
break;
}
}
if (lastSpeechIndex === -1 || lastSpeechIndex >= lastIndex) return null;
const trimmedEnd = sliceEnd(lastSpeechIndex) + SPEECH_TAIL_SECONDS + options.endPadSeconds;
if (options.selectionEnd - trimmedEnd < MINIMUM_TRAILING_TRIM_SECONDS) return null;
return trimmedEnd;
}
export function constrainMediaTimingSelection(options: {
nextStart: number;
nextEnd: number;
currentStart: number;
timelineStart: number;
timelineEnd: number;
mediaEnd: number;
maxMediaDuration: number;
}): { start: number; end: number } {
let start = clamp(options.nextStart, options.timelineStart, options.timelineEnd);
let end = clamp(options.nextEnd, options.timelineStart, options.timelineEnd);
const startMoved = options.nextStart !== options.currentStart;
if (end - start < MINIMUM_CLIP_SECONDS) {
if (startMoved) start = end - MINIMUM_CLIP_SECONDS;
else end = start + MINIMUM_CLIP_SECONDS;
}
if (options.maxMediaDuration > 0 && end - start > options.maxMediaDuration) {
if (startMoved) start = end - options.maxMediaDuration;
else end = start + options.maxMediaDuration;
}
return {
start: Math.max(0, start),
end: Math.min(options.mediaEnd, end),
};
}
/** Maps a pointer position over the timeline track back onto a media timestamp. */
export function mediaTimingTimeFromPointer(options: {
clientX: number;
trackLeft: number;
trackWidth: number;
timelineStart: number;
timelineEnd: number;
}): number {
if (options.trackWidth <= 0) return options.timelineStart;
const ratio = clamp((options.clientX - options.trackLeft) / options.trackWidth, 0, 1);
return options.timelineStart + ratio * (options.timelineEnd - options.timelineStart);
}
/** Slides the selection without resizing it, keeping the whole clip inside the visible timeline. */
export function slideMediaTimingSelection(options: {
nextStart: number;
span: number;
timelineStart: number;
timelineEnd: number;
mediaEnd: number;
}): { start: number; end: number } {
const latestEnd = Math.min(options.timelineEnd, options.mediaEnd);
const maxStart = Math.max(options.timelineStart, latestEnd - options.span);
const start = clamp(options.nextStart, options.timelineStart, maxStart);
return { start, end: start + options.span };
}
export function createMediaTimingReviewModal(
ctx: RendererContext,
options: {
modalStateReader: Pick<ModalStateReader, 'isAnyModalOpen'>;
syncSettingsModalSubtitleSuppression: () => void;
},
) {
let payload: MediaTimingReviewOpenPayload | null = null;
let selectionStart = 0;
let selectionEnd = 0;
let timelineStart = 0;
let timelineEnd = 0;
let previousCount = 0;
let nextCount = 0;
let startPadSeconds = 0;
let endPadSeconds = 0;
/** True until the user moves the clip; the first waveform then trims trailing silence. */
let trailingTrimPending = false;
let resolveInFlight = false;
let previewPlaying = false;
let previewTimer: ReturnType<typeof setTimeout> | null = null;
let waveformTimer: ReturnType<typeof setTimeout> | null = null;
let waveformSequence = 0;
let drag: {
edge: 'start' | 'end' | 'both';
pointerId: number;
trackLeft: number;
trackWidth: number;
grabOffset: number;
} | null = null;
const focus = createModalFocusGuard({
isOpen: () => ctx.state.mediaTimingReviewModalOpen,
getModalRoot: () => ctx.dom.mediaTimingReviewModal,
getPreferredFocusTargets: () => [
ctx.dom.mediaTimingReviewStartHandle,
ctx.dom.mediaTimingReviewCancelBack,
],
getFallbackFocusTarget: () => ctx.dom.mediaTimingReviewCancel,
isModalLayer: ctx.platform.isModalLayer,
});
const previewRequest = createMediaTimingPreviewRequestGuard();
function setStatus(message: string, isError = false): void {
ctx.dom.mediaTimingReviewStatus.textContent = message;
ctx.dom.mediaTimingReviewStatus.classList.toggle('is-error', isError);
}
function clearPreviewTimer(): void {
if (previewTimer !== null) clearTimeout(previewTimer);
previewTimer = null;
}
/**
* Drives the play button label plus the playhead sweep that mirrors the hidden audio player.
* mpv reports when the clip actually finishes (see handlePreviewEnded), which accounts for
* output latency such as Bluetooth headphones; the timer only covers a player that never does.
*/
function setPreviewPlaying(playing: boolean): void {
previewPlaying = playing;
ctx.dom.mediaTimingReviewPlayLabel.textContent = playing ? 'Stop preview' : 'Play selection';
ctx.dom.mediaTimingReviewPlay.classList.toggle('is-playing', playing);
clearPreviewTimer();
const track = ctx.dom.mediaTimingReviewSelectionTrack;
track.classList.remove('is-previewing');
if (!playing) return;
const clipSeconds = Math.max(MINIMUM_CLIP_SECONDS, selectionEnd - selectionStart);
track.style.setProperty('--playhead-duration', `${clipSeconds}s`);
void track.offsetWidth;
track.classList.add('is-previewing');
previewTimer = setTimeout(() => stopPreview(), clipSeconds * 1000 + PREVIEW_END_GRACE_MS);
}
/** The hidden player reached the end of the clip and paused itself. */
function handlePreviewEnded(reviewId: string): void {
if (!payload || payload.reviewId !== reviewId || !previewPlaying) return;
setPreviewPlaying(false);
setStatus('');
}
/** Callers that need to report a failure set their own status after stopping the preview. */
function stopPreview(): void {
previewRequest.invalidate();
setPreviewPlaying(false);
setStatus('');
if (payload) {
void window.electronAPI.stopMediaTimingReviewPreview(payload.reviewId).catch(() => {});
}
}
function renderHandle(handle: HTMLDivElement, value: number): void {
handle.setAttribute('aria-valuemin', timelineStart.toFixed(3));
handle.setAttribute('aria-valuemax', timelineEnd.toFixed(3));
handle.setAttribute('aria-valuenow', value.toFixed(3));
handle.setAttribute('aria-valuetext', formatMediaTimingTimestamp(value));
}
function renderSelection(): void {
const span = Math.max(MINIMUM_CLIP_SECONDS, timelineEnd - timelineStart);
const startPercent = ((selectionStart - timelineStart) / span) * 100;
const endPercent = ((selectionEnd - timelineStart) / span) * 100;
const lineRange = currentLineSelection();
const originalStartPercent = payload
? ((lineRange.rangeStart - timelineStart) / span) * 100
: 0;
const originalEndPercent = payload ? ((lineRange.rangeEnd - timelineStart) / span) * 100 : 0;
ctx.dom.mediaTimingReviewSelectionTrack.style.setProperty(
'--selection-start',
`${clamp(startPercent, 0, 100)}%`,
);
ctx.dom.mediaTimingReviewSelectionTrack.style.setProperty(
'--selection-end',
`${clamp(endPercent, 0, 100)}%`,
);
ctx.dom.mediaTimingReviewSelectionTrack.style.setProperty(
'--original-start',
`${clamp(originalStartPercent, 0, 100)}%`,
);
ctx.dom.mediaTimingReviewSelectionTrack.style.setProperty(
'--original-end',
`${clamp(originalEndPercent, 0, 100)}%`,
);
renderHandle(ctx.dom.mediaTimingReviewStartHandle, selectionStart);
renderHandle(ctx.dom.mediaTimingReviewEndHandle, selectionEnd);
ctx.dom.mediaTimingReviewStartValue.textContent = formatMediaTimingTimestamp(selectionStart);
ctx.dom.mediaTimingReviewEndValue.textContent = formatMediaTimingTimestamp(selectionEnd);
ctx.dom.mediaTimingReviewDuration.textContent = `${(selectionEnd - selectionStart).toFixed(2)}s`;
ctx.dom.mediaTimingReviewTimelineStart.textContent = formatMediaTimingTimestamp(
timelineStart,
false,
);
ctx.dom.mediaTimingReviewTimelineEnd.textContent = formatMediaTimingTimestamp(
timelineEnd,
false,
);
ctx.dom.mediaTimingReviewShowEarlier.disabled = timelineStart <= 0;
ctx.dom.mediaTimingReviewShowLater.disabled =
payload?.mediaDuration !== undefined && timelineEnd >= payload.mediaDuration;
}
function setWaveformState(state: 'loading' | 'ready' | 'unavailable'): void {
ctx.dom.mediaTimingReviewSelectionTrack.classList.toggle('is-loading', state === 'loading');
ctx.dom.mediaTimingReviewSelectionTrack.classList.toggle(
'is-waveform-unavailable',
state === 'unavailable',
);
ctx.dom.mediaTimingReviewWaveformLabel.textContent =
state === 'loading'
? 'isolating dialogue...'
: state === 'ready'
? 'speech-weighted waveform'
: 'waveform unavailable';
}
async function loadWaveform(): Promise<void> {
if (!payload || !ctx.state.mediaTimingReviewModalOpen) return;
waveformSequence += 1;
const sequence = waveformSequence;
const reviewId = payload.reviewId;
const startTime = timelineStart;
const endTime = timelineEnd;
setWaveformState('loading');
ctx.dom.mediaTimingReviewWaveformPath.setAttribute('d', '');
try {
const result = await window.electronAPI.getMediaTimingReviewWaveform({
reviewId,
startTime,
endTime,
});
if (
sequence !== waveformSequence ||
payload?.reviewId !== reviewId ||
timelineStart !== startTime ||
timelineEnd !== endTime
) {
return;
}
if (!result.ok && result.stale) {
closeResolvedReview();
return;
}
const path = result.ok ? buildMediaTimingWaveformPath(result.peaks ?? []) : '';
if (!path) {
setWaveformState('unavailable');
return;
}
ctx.dom.mediaTimingReviewWaveformPath.setAttribute('d', path);
setWaveformState('ready');
trimTrailingSilence(result.peaks ?? []);
} catch {
if (sequence === waveformSequence && payload?.reviewId === reviewId) {
setWaveformState('unavailable');
}
}
}
/**
* Moves an untouched clip end back to where the line's dialogue ends. The Line end
* rail keeps marking the subtitle timing, and Reset restores it.
*/
function trimTrailingSilence(peaks: readonly number[]): void {
if (!payload || !trailingTrimPending || previewPlaying || previewRequest.isInFlight()) {
return;
}
const lineRange = currentLineSelection();
const trimmedEnd = trimMediaTimingSelectionEnd({
peaks,
timelineStart,
timelineEnd,
lineStart: lineRange.rangeStart,
lineEnd: lineRange.rangeEnd,
selectionEnd,
endPadSeconds,
});
if (trimmedEnd === null) return;
updateSelection(selectionStart, trimmedEnd);
setStatus('Clip end moved to where the dialogue ends. Reset restores the subtitle timing.');
}
function queueWaveformLoad(delayMs = 0): void {
if (waveformTimer !== null) clearTimeout(waveformTimer);
waveformSequence += 1;
if (payload && ctx.state.mediaTimingReviewModalOpen) {
ctx.dom.mediaTimingReviewWaveformPath.setAttribute('d', '');
setWaveformState('loading');
}
waveformTimer = setTimeout(() => {
waveformTimer = null;
void loadWaveform();
}, delayMs);
}
function currentLineSelection(): ReturnType<typeof buildMediaTimingLineSelection> {
return buildMediaTimingLineSelection({
previousLines: payload?.previousLines ?? [],
nextLines: payload?.nextLines ?? [],
text: payload?.text ?? '',
originalStartTime: payload?.originalStartTime ?? 0,
originalEndTime: payload?.originalEndTime ?? 0,
previousCount,
nextCount,
});
}
function renderSentence(): void {
if (!payload) return;
const selection = currentLineSelection();
ctx.dom.mediaTimingReviewText.replaceChildren(
...selection.lineTexts.map((text, index) => {
const line = document.createElement('span');
line.className =
index === selection.currentLineIndex
? 'media-timing-review-line is-current'
: 'media-timing-review-line';
line.textContent = text;
return line;
}),
);
const total = selection.lineTexts.length;
ctx.dom.mediaTimingReviewLineCount.textContent = total === 1 ? '1 line' : `${total} lines`;
const hasContext = payload.previousLines.length > 0 || payload.nextLines.length > 0;
ctx.dom.mediaTimingReviewLineControls.classList.toggle('hidden', !hasContext);
ctx.dom.mediaTimingReviewPrevAdd.disabled = previousCount >= payload.previousLines.length;
ctx.dom.mediaTimingReviewPrevRemove.disabled = previousCount <= 0;
ctx.dom.mediaTimingReviewNextAdd.disabled = nextCount >= payload.nextLines.length;
ctx.dom.mediaTimingReviewNextRemove.disabled = nextCount <= 0;
}
/**
* Adds or removes an adjacent subtitle line from the card sentence, then follows the
* affected clip edge to the new outermost line while keeping the user's other edge.
*/
function adjustLines(direction: 'previous' | 'next', delta: number): void {
if (!payload || resolveInFlight) return;
const available =
direction === 'previous' ? payload.previousLines.length : payload.nextLines.length;
const current = direction === 'previous' ? previousCount : nextCount;
const updated = clamp(current + delta, 0, available);
if (updated === current) return;
if (direction === 'previous') previousCount = updated;
else nextCount = updated;
const selection = currentLineSelection();
const mediaEnd = payload.mediaDuration ?? Number.POSITIVE_INFINITY;
let timelineChanged = false;
let capped = false;
if (direction === 'previous') {
const target = Math.max(0, selection.rangeStart - startPadSeconds);
if (target < timelineStart) {
timelineStart = Math.max(0, target - LINE_REVEAL_MARGIN_SECONDS);
timelineChanged = true;
}
updateSelection(target, selectionEnd);
capped = selectionStart > target + 0.001;
} else {
const target = Math.min(mediaEnd, selection.rangeEnd + endPadSeconds);
if (target > timelineEnd) {
timelineEnd = Math.min(mediaEnd, target + LINE_REVEAL_MARGIN_SECONDS);
timelineChanged = true;
}
updateSelection(selectionStart, target);
capped = selectionEnd < target - 0.001;
}
renderSentence();
if (capped && payload.maxMediaDuration > 0) {
setStatus(
`Clip length is capped at ${payload.maxMediaDuration}s, so the audio cannot cover every added line.`,
);
}
if (timelineChanged) queueWaveformLoad(120);
}
function updateSelection(nextStart: number, nextEnd: number): void {
if (!payload) return;
const mediaEnd = payload.mediaDuration ?? Number.POSITIVE_INFINITY;
const nextSelection = constrainMediaTimingSelection({
nextStart,
nextEnd,
currentStart: selectionStart,
timelineStart,
timelineEnd,
mediaEnd,
maxMediaDuration: payload.maxMediaDuration,
});
selectionStart = nextSelection.start;
selectionEnd = nextSelection.end;
trailingTrimPending = false;
if (previewPlaying || previewRequest.isInFlight()) stopPreview();
setStatus('');
renderSelection();
}
/** Shifts the whole clip without changing its length. */
function moveSelection(nextStart: number): void {
if (!payload) return;
const slid = slideMediaTimingSelection({
nextStart,
span: selectionEnd - selectionStart,
timelineStart,
timelineEnd,
mediaEnd: payload.mediaDuration ?? Number.POSITIVE_INFINITY,
});
updateSelection(slid.start, slid.end);
}
function setEdge(edge: 'start' | 'end', value: number): void {
if (edge === 'start') updateSelection(value, selectionEnd);
else updateSelection(selectionStart, value);
}
function applyDrag(clientX: number): void {
if (!drag) return;
const pointerTime = mediaTimingTimeFromPointer({
clientX,
trackLeft: drag.trackLeft,
trackWidth: drag.trackWidth,
timelineStart,
timelineEnd,
});
const time = pointerTime - drag.grabOffset;
if (drag.edge === 'both') moveSelection(time);
else setEdge(drag.edge, time);
}
/**
* Grabbing a handle drags that edge, grabbing the highlighted clip slides the whole selection,
* and pressing anywhere else snaps the nearest edge to that point and keeps dragging it.
*/
function beginDrag(event: PointerEvent): void {
if (!payload || resolveInFlight || drag !== null || event.button !== 0) return;
const track = ctx.dom.mediaTimingReviewSelectionTrack;
const rect = track.getBoundingClientRect();
const pointerTime = mediaTimingTimeFromPointer({
clientX: event.clientX,
trackLeft: rect.left,
trackWidth: rect.width,
timelineStart,
timelineEnd,
});
const target = event.target;
let edge: 'start' | 'end' | 'both';
let grabOffset: number;
if (target === ctx.dom.mediaTimingReviewStartHandle) {
edge = 'start';
grabOffset = pointerTime - selectionStart;
} else if (target === ctx.dom.mediaTimingReviewEndHandle) {
edge = 'end';
grabOffset = pointerTime - selectionEnd;
} else if (target === ctx.dom.mediaTimingReviewSelectedRange) {
edge = 'both';
grabOffset = pointerTime - selectionStart;
} else {
edge =
Math.abs(pointerTime - selectionStart) <= Math.abs(pointerTime - selectionEnd)
? 'start'
: 'end';
grabOffset = 0;
}
event.preventDefault();
drag = {
edge,
pointerId: event.pointerId,
trackLeft: rect.left,
trackWidth: rect.width,
grabOffset,
};
track.setPointerCapture(event.pointerId);
ctx.dom.mediaTimingReviewModal.classList.add(edge === 'both' ? 'is-sliding' : 'is-scrubbing');
if (edge !== 'both') {
const handle =
edge === 'start'
? ctx.dom.mediaTimingReviewStartHandle
: ctx.dom.mediaTimingReviewEndHandle;
handle.focus();
if (grabOffset === 0) applyDrag(event.clientX);
}
}
function endDrag(event: PointerEvent): void {
if (!drag || drag.pointerId !== event.pointerId) return;
const track = ctx.dom.mediaTimingReviewSelectionTrack;
if (track.hasPointerCapture(event.pointerId)) track.releasePointerCapture(event.pointerId);
drag = null;
ctx.dom.mediaTimingReviewModal.classList.remove('is-scrubbing', 'is-sliding');
}
function cancelDrag(): void {
drag = null;
ctx.dom.mediaTimingReviewModal.classList.remove('is-scrubbing', 'is-sliding');
}
function handleEdgeKeydown(event: KeyboardEvent, edge: 'start' | 'end'): void {
const step = event.shiftKey ? COARSE_ADJUST_SECONDS : FINE_ADJUST_SECONDS;
const current = edge === 'start' ? selectionStart : selectionEnd;
if (event.key === 'ArrowLeft' || event.key === 'ArrowDown') setEdge(edge, current - step);
else if (event.key === 'ArrowRight' || event.key === 'ArrowUp') setEdge(edge, current + step);
else if (event.key === 'Home') setEdge(edge, timelineStart);
else if (event.key === 'End') setEdge(edge, timelineEnd);
else return;
event.preventDefault();
}
function showEditor(): void {
ctx.dom.mediaTimingReviewCancelStep.classList.add('hidden');
ctx.dom.mediaTimingReviewEditor.classList.remove('hidden');
ctx.dom.mediaTimingReviewCancel.classList.remove('hidden');
}
function requestCancel(): void {
if (!ctx.state.mediaTimingReviewModalOpen || !payload || resolveInFlight) return;
stopPreview();
ctx.dom.mediaTimingReviewEditor.classList.add('hidden');
ctx.dom.mediaTimingReviewCancel.classList.add('hidden');
ctx.dom.mediaTimingReviewCancelStep.classList.remove('hidden');
ctx.dom.mediaTimingReviewCancelMessage.textContent =
payload.noteId !== undefined
? 'Keep editing, keep this card without media, use its original timing, or delete it.'
: 'Keep editing, create this card without media, use its original timing, or do not create it.';
ctx.dom.mediaTimingReviewSkipMedia.textContent =
payload.noteId !== undefined ? 'Keep without media' : 'Create without media';
ctx.dom.mediaTimingReviewDiscard.textContent =
payload.noteId !== undefined ? 'Delete card' : "Don't create card";
ctx.dom.mediaTimingReviewCancelBack.focus();
}
function closeResolvedReview(): void {
if (!ctx.state.mediaTimingReviewModalOpen) return;
cancelDrag();
clearPreviewTimer();
if (waveformTimer !== null) clearTimeout(waveformTimer);
waveformTimer = null;
waveformSequence += 1;
previewPlaying = false;
ctx.state.mediaTimingReviewModalOpen = false;
ctx.dom.mediaTimingReviewModal.classList.add('hidden');
ctx.dom.mediaTimingReviewModal.setAttribute('aria-hidden', 'true');
focus.detach();
window.electronAPI.notifyOverlayModalClosed('media-timing-review');
options.syncSettingsModalSubtitleSuppression();
payload = null;
if (!options.modalStateReader.isAnyModalOpen()) {
ctx.dom.overlay.classList.remove('interactive');
if (ctx.platform.shouldToggleMouseIgnore) {
window.electronAPI.setIgnoreMouseEvents(true, { forward: true });
}
}
}
async function resolveReview(decision: MediaTimingReviewDecision): Promise<void> {
if (!payload || resolveInFlight) return;
resolveInFlight = true;
stopPreview();
const controls = ctx.dom.mediaTimingReviewModal.querySelectorAll<HTMLButtonElement>('button');
controls.forEach((button) => {
button.disabled = true;
});
try {
const result = await window.electronAPI.resolveMediaTimingReview({
reviewId: payload.reviewId,
decision,
});
if (!result.ok && !result.stale) {
setStatus(result.message ?? 'The timing review could not be resolved.', true);
showEditor();
return;
}
// A stale review was already settled by main; keeping the modal up would leave
// controls that can never succeed over a live mpv window.
closeResolvedReview();
} catch (error) {
setStatus(error instanceof Error ? error.message : String(error), true);
showEditor();
} finally {
resolveInFlight = false;
controls.forEach((button) => {
button.disabled = false;
});
renderSelection();
renderSentence();
}
}
function confirmSelection(): void {
if (!payload) return;
const includesAdjacentLines = previousCount > 0 || nextCount > 0;
void resolveReview({
action: 'confirm',
startTime: selectionStart,
endTime: selectionEnd,
...(includesAdjacentLines ? { text: currentLineSelection().sentence } : {}),
});
}
async function togglePreview(): Promise<void> {
if (!payload || resolveInFlight) return;
if (previewPlaying) {
stopPreview();
return;
}
const requestId = previewRequest.begin();
if (requestId === null) return;
const requestedReviewId = payload.reviewId;
setStatus('Starting audio preview...');
try {
const result = await window.electronAPI.previewMediaTimingReview({
reviewId: requestedReviewId,
startTime: selectionStart,
endTime: selectionEnd,
});
if (!previewRequest.isCurrent(requestId) || payload?.reviewId !== requestedReviewId) {
if (result.ok && !previewRequest.isInFlight() && !previewPlaying) {
void window.electronAPI.stopMediaTimingReviewPreview(requestedReviewId).catch(() => {});
}
return;
}
if (!result.ok && result.stale) {
closeResolvedReview();
return;
}
if (!result.ok) {
setStatus(result.message ?? 'Audio preview is unavailable.', true);
setPreviewPlaying(false);
return;
}
setStatus('Previewing in the hidden audio player.');
setPreviewPlaying(true);
} catch (error) {
if (!previewRequest.isCurrent(requestId) || payload?.reviewId !== requestedReviewId) return;
setStatus(
`Audio preview unavailable: ${error instanceof Error ? error.message : String(error)}`,
true,
);
setPreviewPlaying(false);
} finally {
previewRequest.finish(requestId);
}
}
function openMediaTimingReviewModal(nextPayload: MediaTimingReviewOpenPayload): void {
previewRequest.invalidate();
cancelDrag();
payload = {
...nextPayload,
previousLines: nextPayload.previousLines ?? [],
nextLines: nextPayload.nextLines ?? [],
};
selectionStart = nextPayload.selectionStartTime;
selectionEnd = nextPayload.selectionEndTime;
timelineStart = nextPayload.timelineStartTime;
timelineEnd = nextPayload.timelineEndTime;
previousCount = 0;
nextCount = 0;
startPadSeconds = Math.max(0, nextPayload.originalStartTime - nextPayload.selectionStartTime);
endPadSeconds = Math.max(0, nextPayload.selectionEndTime - nextPayload.originalEndTime);
trailingTrimPending = true;
resolveInFlight = false;
setPreviewPlaying(false);
ctx.dom.mediaTimingReviewKind.textContent =
nextPayload.kind === 'word'
? 'Word card'
: nextPayload.kind === 'audio'
? 'Audio card'
: 'Sentence card';
ctx.dom.mediaTimingReviewKind.dataset.kind = nextPayload.kind;
ctx.dom.mediaTimingReviewDiscard.textContent =
nextPayload.noteId !== undefined ? 'Delete card' : "Don't create card";
setStatus('');
showEditor();
renderSelection();
renderSentence();
ctx.state.mediaTimingReviewModalOpen = true;
options.syncSettingsModalSubtitleSuppression();
ctx.dom.overlay.classList.add('interactive');
if (ctx.platform.shouldToggleMouseIgnore) window.electronAPI.setIgnoreMouseEvents(false);
ctx.dom.mediaTimingReviewModal.classList.remove('hidden');
ctx.dom.mediaTimingReviewModal.setAttribute('aria-hidden', 'false');
window.electronAPI.notifyOverlayModalOpened('media-timing-review');
focus.attach();
focus.requestOverlayFocus();
window.focus();
focus.enforceModalFocus();
queueWaveformLoad();
}
function expandTimeline(direction: 'earlier' | 'later'): void {
if (!payload) return;
if (direction === 'earlier') {
timelineStart = Math.max(0, timelineStart - TIMELINE_EXPANSION_SECONDS);
} else {
timelineEnd = Math.min(
payload.mediaDuration ?? Number.POSITIVE_INFINITY,
timelineEnd + TIMELINE_EXPANSION_SECONDS,
);
}
renderSelection();
queueWaveformLoad(120);
}
function handleMediaTimingReviewKeydown(event: KeyboardEvent): boolean {
if (!ctx.state.mediaTimingReviewModalOpen) return false;
if (event.key === 'Escape') {
event.preventDefault();
if (!ctx.dom.mediaTimingReviewCancelStep.classList.contains('hidden')) showEditor();
else requestCancel();
return true;
}
if (
event.code === 'Space' &&
event.target instanceof Element &&
!event.target.closest('button')
) {
event.preventDefault();
void togglePreview();
return true;
}
if (
event.key === 'Enter' &&
ctx.dom.mediaTimingReviewCancelStep.classList.contains('hidden') &&
!(event.target instanceof Element && event.target.closest('button'))
) {
event.preventDefault();
confirmSelection();
return true;
}
if (
(event.key === 'p' || event.key === 'P' || event.key === 'n' || event.key === 'N') &&
ctx.dom.mediaTimingReviewCancelStep.classList.contains('hidden') &&
!event.ctrlKey &&
!event.metaKey &&
!event.altKey
) {
event.preventDefault();
adjustLines(
event.key === 'p' || event.key === 'P' ? 'previous' : 'next',
event.shiftKey ? -1 : 1,
);
return true;
}
return false;
}
function wireDomEvents(): void {
const track = ctx.dom.mediaTimingReviewSelectionTrack;
track.addEventListener('pointerdown', beginDrag);
track.addEventListener('pointermove', (event) => {
if (drag?.pointerId === event.pointerId) applyDrag(event.clientX);
});
track.addEventListener('pointerup', endDrag);
track.addEventListener('pointercancel', endDrag);
ctx.dom.mediaTimingReviewStartHandle.addEventListener('keydown', (event) =>
handleEdgeKeydown(event, 'start'),
);
ctx.dom.mediaTimingReviewEndHandle.addEventListener('keydown', (event) =>
handleEdgeKeydown(event, 'end'),
);
ctx.dom.mediaTimingReviewShowEarlier.addEventListener('click', () => expandTimeline('earlier'));
ctx.dom.mediaTimingReviewShowLater.addEventListener('click', () => expandTimeline('later'));
ctx.dom.mediaTimingReviewStartBack.addEventListener('click', () =>
updateSelection(selectionStart - FINE_ADJUST_SECONDS, selectionEnd),
);
ctx.dom.mediaTimingReviewStartForward.addEventListener('click', () =>
updateSelection(selectionStart + FINE_ADJUST_SECONDS, selectionEnd),
);
ctx.dom.mediaTimingReviewEndBack.addEventListener('click', () =>
updateSelection(selectionStart, selectionEnd - FINE_ADJUST_SECONDS),
);
ctx.dom.mediaTimingReviewEndForward.addEventListener('click', () =>
updateSelection(selectionStart, selectionEnd + FINE_ADJUST_SECONDS),
);
ctx.dom.mediaTimingReviewPlay.addEventListener('click', () => void togglePreview());
ctx.dom.mediaTimingReviewReset.addEventListener('click', () => {
if (!payload) return;
previousCount = 0;
nextCount = 0;
updateSelection(payload.selectionStartTime, payload.selectionEndTime);
renderSentence();
});
ctx.dom.mediaTimingReviewPrevAdd.addEventListener('click', () => adjustLines('previous', 1));
ctx.dom.mediaTimingReviewPrevRemove.addEventListener('click', () =>
adjustLines('previous', -1),
);
ctx.dom.mediaTimingReviewNextAdd.addEventListener('click', () => adjustLines('next', 1));
ctx.dom.mediaTimingReviewNextRemove.addEventListener('click', () => adjustLines('next', -1));
ctx.dom.mediaTimingReviewCancel.addEventListener('click', requestCancel);
ctx.dom.mediaTimingReviewCancelBack.addEventListener('click', showEditor);
ctx.dom.mediaTimingReviewUseOriginal.addEventListener(
'click',
() => void resolveReview({ action: 'use-original' }),
);
ctx.dom.mediaTimingReviewSkipMedia.addEventListener(
'click',
() => void resolveReview({ action: 'skip-media' }),
);
ctx.dom.mediaTimingReviewDiscard.addEventListener(
'click',
() => void resolveReview({ action: 'discard' }),
);
ctx.dom.mediaTimingReviewConfirm.addEventListener('click', () => confirmSelection());
}
return {
openMediaTimingReviewModal,
handlePreviewEnded,
requestCancel,
handleMediaTimingReviewKeydown,
wireDomEvents,
};
}
+23
View File
@@ -45,6 +45,7 @@ import { createCharacterDictionaryModal } from './modals/character-dictionary.js
import { createRuntimeOptionsModal } from './modals/runtime-options.js';
import { createSubsyncModal } from './modals/subsync.js';
import { createYoutubeTrackPickerModal } from './modals/youtube-track-picker.js';
import { createMediaTimingReviewModal } from './modals/media-timing-review.js';
import { createPositioningController } from './positioning.js';
import { createOverlayContentMeasurementReporter } from './overlay-content-measurement.js';
import { syncOverlayMouseIgnoreState } from './overlay-mouse-ignore.js';
@@ -114,6 +115,12 @@ const modalDescriptors = [
close: () => youtubePickerModal.closeYoutubePickerModal(),
suppressesSubtitles: true,
},
{
id: 'media-timing-review',
isOpen: () => ctx.state.mediaTimingReviewModalOpen,
close: () => mediaTimingReviewModal.requestCancel(),
suppressesSubtitles: true,
},
{
id: 'playlist-browser',
isOpen: () => ctx.state.playlistBrowserModalOpen,
@@ -265,6 +272,10 @@ const playlistBrowserModal = createPlaylistBrowserModal(ctx, {
modalStateReader: { isAnyModalOpen },
syncSettingsModalSubtitleSuppression,
});
const mediaTimingReviewModal = createMediaTimingReviewModal(ctx, {
modalStateReader: { isAnyModalOpen },
syncSettingsModalSubtitleSuppression,
});
const keyboardHandlers = createKeyboardHandlers(ctx, {
handleRuntimeOptionsKeydown: runtimeOptionsModal.handleRuntimeOptionsKeydown,
handleCharacterDictionaryKeydown: characterDictionaryModal.handleCharacterDictionaryKeydown,
@@ -273,6 +284,7 @@ const keyboardHandlers = createKeyboardHandlers(ctx, {
handleJimakuKeydown: jimakuModal.handleJimakuKeydown,
handleTsukihimeKeydown: tsukihimeModal.handleTsukihimeKeydown,
handleYoutubePickerKeydown: youtubePickerModal.handleYoutubePickerKeydown,
handleMediaTimingReviewKeydown: mediaTimingReviewModal.handleMediaTimingReviewKeydown,
handlePlaylistBrowserKeydown: playlistBrowserModal.handlePlaylistBrowserKeydown,
handleControllerSelectKeydown: controllerSelectModal.handleControllerSelectKeydown,
handleControllerDebugKeydown: controllerDebugModal.handleControllerDebugKeydown,
@@ -574,6 +586,16 @@ function registerModalOpenHandlers(): void {
youtubePickerModal.openYoutubePickerModal(payload);
});
});
window.electronAPI.onOpenMediaTimingReview((payload) => {
runGuarded('media-timing-review:open', () => {
mediaTimingReviewModal.openMediaTimingReviewModal(payload);
});
});
window.electronAPI.onMediaTimingReviewPreviewEnded((reviewId) => {
runGuarded('media-timing-review:preview-ended', () => {
mediaTimingReviewModal.handlePreviewEnded(reviewId);
});
});
window.electronAPI.onOpenPlaylistBrowser(() => {
runGuardedAsync('playlist-browser:open', async () => {
await playlistBrowserModal.openPlaylistBrowserModal();
@@ -805,6 +827,7 @@ async function init(): Promise<void> {
jimakuModal.wireDomEvents();
tsukihimeModal.wireDomEvents();
youtubePickerModal.wireDomEvents();
mediaTimingReviewModal.wireDomEvents();
playlistBrowserModal.wireDomEvents();
kikuModal.wireDomEvents();
runtimeOptionsModal.wireDomEvents();
+4
View File
@@ -64,6 +64,8 @@ export type RendererState = {
youtubePickerSecondaryTrackId: string | null;
youtubePickerStatus: string;
mediaTimingReviewModalOpen: boolean;
kikuModalOpen: boolean;
kikuSelectedCard: 1 | 2;
kikuOriginalData: KikuDuplicateCardInfo | null;
@@ -194,6 +196,8 @@ export function createRendererState(): RendererState {
youtubePickerSecondaryTrackId: null,
youtubePickerStatus: '',
mediaTimingReviewModalOpen: false,
kikuModalOpen: false,
kikuSelectedCard: 1,
kikuOriginalData: null,
+789
View File
@@ -1323,6 +1323,795 @@ body:focus-visible,
}
}
/* Media timing review uses the Catppuccin Macchiato palette without opacity-shifted colors. */
.media-timing-review-modal {
background: rgba(24, 25, 38, 0.76);
backdrop-filter: blur(10px);
}
/* Pointer capture keeps events on the track, so the cursor is forced from the modal root. */
.media-timing-review-modal.is-scrubbing,
.media-timing-review-modal.is-scrubbing * {
cursor: ew-resize;
}
.media-timing-review-modal.is-sliding,
.media-timing-review-modal.is-sliding * {
cursor: grabbing;
}
.media-timing-review-content {
width: min(720px, calc(100vw - 32px));
max-height: min(700px, calc(100vh - 32px));
overflow: auto;
gap: 0;
padding: 0;
border: 1px solid var(--ctp-surface1);
border-radius: 16px;
background: var(--ctp-base);
color: var(--ctp-text);
box-shadow:
0 30px 90px rgba(24, 25, 38, 0.8),
0 0 0 1px rgba(183, 189, 248, 0.07) inset;
animation: media-timing-review-enter 180ms cubic-bezier(0.2, 0.9, 0.25, 1);
}
@keyframes media-timing-review-enter {
from {
opacity: 0;
transform: translateY(10px) scale(0.985);
}
to {
opacity: 1;
transform: translateY(0) scale(1);
}
}
.media-timing-review-header {
position: sticky;
z-index: 2;
top: 0;
display: flex;
align-items: center;
justify-content: space-between;
gap: 24px;
padding: 16px 20px 13px;
border-bottom: 1px solid var(--ctp-surface0);
background: var(--ctp-mantle);
}
.media-timing-review-heading {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 10px;
}
.media-timing-review-title {
color: var(--ctp-text);
font-size: 19px;
font-weight: 750;
letter-spacing: -0.025em;
}
.media-timing-review-kind {
--kind-accent: var(--ctp-teal);
padding: 3px 9px;
border: 1px solid color-mix(in srgb, var(--kind-accent) 42%, transparent);
border-radius: 999px;
background: color-mix(in srgb, var(--kind-accent) 13%, transparent);
color: var(--kind-accent);
font-size: 10px;
font-weight: 800;
letter-spacing: 0.12em;
text-transform: uppercase;
}
.media-timing-review-kind[data-kind='word'] {
--kind-accent: var(--ctp-mauve);
}
.media-timing-review-kind[data-kind='audio'] {
--kind-accent: var(--ctp-sky);
}
.media-timing-review-editor {
padding: 14px 20px 18px;
}
.media-timing-review-sentence-header {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
margin-bottom: 6px;
}
.media-timing-review-sentence-heading {
display: flex;
align-items: center;
gap: 8px;
}
.media-timing-review-sentence-label {
color: var(--ctp-overlay1);
font-size: 10px;
font-weight: 750;
letter-spacing: 0.1em;
text-transform: uppercase;
}
.media-timing-review-line-count {
padding: 2px 8px;
border: 1px solid var(--ctp-surface1);
border-radius: 999px;
background: var(--ctp-mantle);
color: var(--ctp-subtext0);
font-size: 10px;
font-weight: 700;
font-variant-numeric: tabular-nums;
}
.media-timing-review-line-controls {
display: flex;
align-items: center;
gap: 10px;
}
.media-timing-review-line-stepper {
display: flex;
align-items: center;
gap: 4px;
}
.media-timing-review-line-stepper span {
margin-right: 2px;
color: var(--ctp-overlay1);
font-size: 10px;
font-weight: 750;
letter-spacing: 0.08em;
text-transform: uppercase;
}
.media-timing-review-line-stepper button {
width: 24px;
height: 22px;
padding: 0;
border: 1px solid var(--ctp-surface2);
border-radius: 6px;
background: var(--ctp-surface0);
color: var(--ctp-text);
font-size: 13px;
font-weight: 750;
line-height: 1;
cursor: pointer;
}
.media-timing-review-text {
display: flex;
flex-direction: column;
gap: 3px;
max-height: 108px;
overflow: auto;
margin: 0 0 12px;
padding: 8px 14px;
border-left: 3px solid var(--ctp-mauve);
border-radius: 0 10px 10px 0;
background: var(--ctp-mantle);
color: var(--ctp-subtext1);
font-size: 15px;
font-weight: 560;
line-height: 1.45;
white-space: pre-wrap;
}
.media-timing-review-line {
color: var(--ctp-overlay1);
}
.media-timing-review-line.is-current {
color: var(--ctp-text);
}
/* Only dim context lines when some are actually added. */
.media-timing-review-text .media-timing-review-line:only-child {
color: var(--ctp-subtext1);
}
.media-timing-review-readout {
display: grid;
grid-template-columns: 1fr auto 1fr;
overflow: hidden;
margin-bottom: 12px;
border: 1px solid var(--ctp-surface1);
border-radius: 12px;
background: var(--ctp-mantle);
}
.media-timing-review-readout > div {
display: flex;
min-width: 0;
flex-direction: column;
gap: 3px;
padding: 8px 12px;
}
.media-timing-review-readout > div + div {
border-left: 1px solid var(--ctp-surface0);
}
.media-timing-review-readout span {
color: var(--ctp-overlay1);
font-size: 10px;
font-weight: 750;
letter-spacing: 0.1em;
text-transform: uppercase;
}
.media-timing-review-readout strong {
color: var(--ctp-lavender);
font-variant-numeric: tabular-nums;
font-size: 16px;
font-weight: 720;
}
.media-timing-review-readout .media-timing-review-duration {
align-items: center;
min-width: 128px;
background: var(--ctp-surface0);
text-align: center;
}
.media-timing-review-readout > div:last-child {
align-items: flex-end;
text-align: right;
}
.media-timing-review-readout .media-timing-review-duration strong {
color: var(--ctp-teal);
}
.media-timing-review-timeline-shell {
padding: 10px 12px 8px;
border: 1px solid var(--ctp-surface1);
border-radius: 14px;
background: var(--ctp-mantle);
}
.media-timing-review-timeline-labels,
.media-timing-review-expand-row {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
color: var(--ctp-overlay1);
font-size: 10px;
font-weight: 650;
letter-spacing: 0.05em;
}
.media-timing-review-timeline-labels span:first-child,
.media-timing-review-timeline-labels span:last-child {
color: var(--ctp-subtext0);
font-variant-numeric: tabular-nums;
}
.media-timing-review-track {
--selection-start: 20%;
--selection-end: 80%;
--original-start: 25%;
--original-end: 75%;
--playhead-duration: 1s;
position: relative;
height: 52px;
margin: 8px 0 9px;
overflow: hidden;
border: 1px solid var(--ctp-surface2);
border-radius: 10px;
background:
linear-gradient(var(--ctp-surface1), var(--ctp-surface1)) center / 100% 1px no-repeat,
repeating-linear-gradient(
90deg,
transparent 0,
transparent calc(10% - 1px),
color-mix(in srgb, var(--ctp-surface1) 65%, transparent) calc(10% - 1px),
color-mix(in srgb, var(--ctp-surface1) 65%, transparent) 10%
),
linear-gradient(180deg, var(--ctp-crust), var(--ctp-mantle));
cursor: ew-resize;
touch-action: none;
user-select: none;
}
.media-timing-review-waveform {
position: absolute;
z-index: 1;
inset: 4px 0;
width: 100%;
height: calc(100% - 8px);
pointer-events: none;
transition: opacity 160ms ease;
}
#mediaTimingReviewWaveformPath {
fill: url('#mediaTimingReviewWaveformFill');
}
.media-timing-review-waveform-edge-stop {
stop-color: var(--ctp-sky);
}
.media-timing-review-waveform-core-stop {
stop-color: var(--ctp-blue);
}
/* Sits above the selection scrim so adjacent dialogue reads as outside the mined subtitle. */
.media-timing-review-original-range {
position: absolute;
z-index: 4;
top: 0;
bottom: 0;
left: var(--original-start);
right: calc(100% - var(--original-end));
min-width: 1px;
background: color-mix(in srgb, var(--ctp-peach) 11%, transparent);
box-shadow: inset 0 0 0 1px color-mix(in srgb, var(--ctp-peach) 28%, transparent);
pointer-events: none;
}
.media-timing-review-original-boundary {
position: absolute;
top: 0;
bottom: 0;
width: 2px;
background: var(--ctp-peach);
box-shadow: 0 0 8px color-mix(in srgb, var(--ctp-peach) 45%, transparent);
}
.media-timing-review-original-boundary.is-start {
left: 0;
}
.media-timing-review-original-boundary.is-end {
right: 0;
}
.media-timing-review-original-label {
position: absolute;
z-index: 7;
padding: 3px 5px;
border: 1px solid color-mix(in srgb, var(--ctp-crust) 35%, transparent);
border-radius: 4px;
background: var(--ctp-peach);
box-shadow: 0 2px 6px color-mix(in srgb, var(--ctp-crust) 55%, transparent);
color: var(--ctp-crust);
font-size: 8px;
font-weight: 800;
letter-spacing: 0.06em;
line-height: 1;
pointer-events: none;
white-space: nowrap;
text-transform: uppercase;
}
.media-timing-review-original-label.is-start {
top: 5px;
left: calc(var(--original-start) + 7px);
}
.media-timing-review-original-label.is-end {
right: calc(100% - var(--original-end) + 7px);
bottom: 5px;
}
.media-timing-review-track.is-loading::after {
position: absolute;
z-index: 2;
inset: 0;
content: '';
background: linear-gradient(
105deg,
transparent 20%,
rgba(138, 173, 244, 0.16) 44%,
rgba(183, 189, 248, 0.22) 50%,
transparent 76%
);
transform: translateX(-100%);
animation: media-timing-waveform-loading 1.1s ease-in-out infinite;
pointer-events: none;
}
.media-timing-review-track.is-waveform-unavailable .media-timing-review-waveform {
opacity: 0;
}
@keyframes media-timing-waveform-loading {
to {
transform: translateX(100%);
}
}
/* The oversized shadow spread dims everything outside the clip; the track clips it. */
.media-timing-review-selected-range {
position: absolute;
z-index: 3;
inset: 0 calc(100% - var(--selection-end)) 0 var(--selection-start);
border-inline: 1px solid color-mix(in srgb, var(--ctp-teal) 65%, transparent);
background: color-mix(in srgb, var(--ctp-teal) 9%, transparent);
box-shadow: 0 0 0 2000px color-mix(in srgb, var(--ctp-crust) 62%, transparent);
cursor: grab;
}
.media-timing-review-playhead {
position: absolute;
z-index: 5;
top: 0;
bottom: 0;
left: var(--selection-start);
width: 2px;
background: var(--ctp-yellow);
box-shadow: 0 0 12px color-mix(in srgb, var(--ctp-yellow) 75%, transparent);
opacity: 0;
pointer-events: none;
}
.media-timing-review-track.is-previewing .media-timing-review-playhead {
opacity: 1;
animation: media-timing-review-playhead var(--playhead-duration) linear forwards;
}
@keyframes media-timing-review-playhead {
from {
left: var(--selection-start);
}
to {
left: var(--selection-end);
}
}
.media-timing-review-handle {
position: absolute;
z-index: 6;
top: 0;
bottom: 0;
width: 14px;
border: 1px solid var(--ctp-teal);
background: linear-gradient(
180deg,
var(--ctp-teal),
color-mix(in srgb, var(--ctp-teal) 74%, var(--ctp-crust))
);
cursor: ew-resize;
touch-action: none;
transition: box-shadow 130ms ease;
}
/* Widens the grab area past the visible bracket without moving the clip edge. */
.media-timing-review-handle::before {
position: absolute;
inset: 0 -7px;
content: '';
}
.media-timing-review-handle::after {
position: absolute;
top: 50%;
left: 50%;
width: 6px;
height: 18px;
content: '';
border-inline: 1px solid color-mix(in srgb, var(--ctp-crust) 55%, transparent);
transform: translate(-50%, -50%);
}
.media-timing-review-handle-start {
left: var(--selection-start);
border-radius: 5px 0 0 5px;
}
.media-timing-review-handle-end {
left: var(--selection-end);
border-radius: 0 5px 5px 0;
transform: translateX(-100%);
}
.media-timing-review-handle:hover {
box-shadow: 0 0 16px color-mix(in srgb, var(--ctp-teal) 60%, transparent);
}
.media-timing-review-handle:focus-visible {
outline: 2px solid transparent;
box-shadow:
inset 0 0 0 2px var(--ctp-yellow),
0 0 16px color-mix(in srgb, var(--ctp-yellow) 55%, transparent);
}
.media-timing-review-expand-row span {
text-align: center;
}
.media-timing-review-expand-button,
.media-timing-review-quiet-button,
.media-timing-review-original-button,
.media-timing-review-skip-button,
.media-timing-review-discard-button,
.media-timing-review-play-button,
.media-timing-review-confirm-button,
.media-timing-review-fine-control button {
border: 1px solid var(--ctp-surface2);
border-radius: 9px;
background: var(--ctp-surface0);
color: var(--ctp-text);
font: inherit;
font-size: 12px;
font-weight: 720;
cursor: pointer;
transition:
background 130ms ease,
border-color 130ms ease,
transform 130ms ease;
}
.media-timing-review-expand-button {
min-width: 82px;
padding: 5px 9px;
color: var(--ctp-sky);
font-variant-numeric: tabular-nums;
}
.media-timing-review-expand-button:disabled,
.media-timing-review-content button:disabled {
cursor: default;
opacity: 0.42;
}
.media-timing-review-content button:not(:disabled):hover {
border-color: var(--ctp-lavender);
background: var(--ctp-surface1);
}
.media-timing-review-content button:not(:disabled):active {
transform: translateY(1px);
}
.media-timing-review-content button:focus-visible {
outline: 2px solid var(--ctp-yellow);
outline-offset: 2px;
}
.media-timing-review-fine-grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 10px;
margin-top: 10px;
}
.media-timing-review-fine-control {
display: grid;
grid-template-columns: 1fr auto auto;
align-items: center;
gap: 6px;
padding: 8px 10px;
border: 1px solid var(--ctp-surface0);
border-radius: 10px;
background: var(--ctp-mantle);
color: var(--ctp-subtext1);
}
.media-timing-review-fine-control span {
font-size: 11px;
font-weight: 750;
letter-spacing: 0.08em;
text-transform: uppercase;
}
.media-timing-review-fine-control button {
min-width: 58px;
padding: 5px 8px;
font-variant-numeric: tabular-nums;
}
.media-timing-review-status {
min-height: 21px;
padding-top: 8px;
color: var(--ctp-teal);
font-size: 11px;
font-weight: 620;
}
.media-timing-review-status.is-error {
color: var(--ctp-red);
}
.media-timing-review-footer,
.media-timing-review-preview-actions {
display: flex;
align-items: center;
gap: 9px;
}
.media-timing-review-footer {
justify-content: space-between;
padding-top: 4px;
}
.media-timing-review-play-button,
.media-timing-review-confirm-button,
.media-timing-review-quiet-button,
.media-timing-review-original-button,
.media-timing-review-skip-button,
.media-timing-review-discard-button {
padding: 9px 15px;
}
.media-timing-review-play-button {
display: inline-flex;
align-items: center;
gap: 9px;
border-color: var(--ctp-teal);
background: var(--ctp-teal);
color: var(--ctp-crust);
}
.media-timing-review-play-glyph {
width: 0;
height: 0;
border-block: 5px solid transparent;
border-left: 9px solid currentcolor;
}
.media-timing-review-play-button.is-playing {
border-color: var(--ctp-yellow);
background: var(--ctp-yellow);
}
.media-timing-review-play-button.is-playing .media-timing-review-play-glyph {
width: 9px;
height: 10px;
border: 0;
border-radius: 2px;
background: currentcolor;
}
.media-timing-review-confirm-button {
border-color: var(--ctp-blue);
background: var(--ctp-blue);
color: var(--ctp-crust);
}
.media-timing-review-original-button {
border-color: var(--ctp-yellow);
color: var(--ctp-yellow);
}
.media-timing-review-skip-button {
border-color: var(--ctp-sky);
color: var(--ctp-sky);
}
.media-timing-review-discard-button {
border-color: var(--ctp-red);
background: var(--ctp-red);
color: var(--ctp-crust);
}
.media-timing-review-hints {
display: flex;
flex-wrap: wrap;
gap: 4px 16px;
padding-top: 14px;
color: var(--ctp-overlay1);
font-size: 10px;
font-weight: 650;
letter-spacing: 0.04em;
}
.media-timing-review-hints kbd {
display: inline-block;
min-width: 15px;
margin-right: 3px;
padding: 1px 5px;
border: 1px solid var(--ctp-surface1);
border-bottom-width: 2px;
border-radius: 5px;
background: var(--ctp-mantle);
color: var(--ctp-subtext0);
font-family: inherit;
font-size: 9px;
font-weight: 750;
text-align: center;
}
.media-timing-review-cancel-step {
min-height: 300px;
padding: 44px 38px 34px;
text-align: center;
}
.media-timing-review-cancel-mark {
display: grid;
width: 52px;
height: 52px;
margin: 0 auto 16px;
place-items: center;
border: 2px solid var(--ctp-yellow);
border-radius: 50%;
background: color-mix(in srgb, var(--ctp-yellow) 12%, transparent);
color: var(--ctp-yellow);
font-size: 26px;
font-weight: 760;
}
.media-timing-review-cancel-step h2 {
color: var(--ctp-text);
font-size: 21px;
font-weight: 760;
}
.media-timing-review-cancel-step p {
max-width: 520px;
margin: 8px auto 24px;
color: var(--ctp-subtext0);
font-size: 13px;
line-height: 1.5;
}
.media-timing-review-cancel-actions {
display: flex;
flex-wrap: wrap;
justify-content: center;
gap: 10px;
}
@media (prefers-reduced-motion: reduce) {
.media-timing-review-content {
animation: none;
}
.media-timing-review-track.is-loading::after,
.media-timing-review-track.is-previewing .media-timing-review-playhead {
animation: none;
}
}
@media (max-width: 640px) {
.media-timing-review-content {
width: calc(100vw - 18px);
max-height: calc(100vh - 18px);
}
.media-timing-review-header,
.media-timing-review-editor {
padding-right: 15px;
padding-left: 15px;
}
.media-timing-review-readout {
grid-template-columns: 1fr;
}
.media-timing-review-readout > div + div {
border-top: 1px solid var(--ctp-surface0);
border-left: 0;
}
.media-timing-review-fine-grid {
grid-template-columns: 1fr;
}
.media-timing-review-footer {
align-items: stretch;
flex-direction: column;
}
.media-timing-review-preview-actions > *,
.media-timing-review-confirm-button {
flex: 1;
}
}
body.subtitle-sidebar-embedded-open #subtitleContainer {
max-width: min(80%, calc(100vw - var(--subtitle-sidebar-reserved-width) - 24px));
transform: translateX(calc(var(--subtitle-sidebar-reserved-width) * -0.5));
+112 -2
View File
@@ -44,6 +44,46 @@ export type RendererDom = {
youtubePickerStatus: HTMLDivElement;
youtubePickerTracks: HTMLUListElement;
mediaTimingReviewModal: HTMLDivElement;
mediaTimingReviewKind: HTMLDivElement;
mediaTimingReviewText: HTMLElement;
mediaTimingReviewLineCount: HTMLElement;
mediaTimingReviewLineControls: HTMLDivElement;
mediaTimingReviewPrevAdd: HTMLButtonElement;
mediaTimingReviewPrevRemove: HTMLButtonElement;
mediaTimingReviewNextAdd: HTMLButtonElement;
mediaTimingReviewNextRemove: HTMLButtonElement;
mediaTimingReviewStartValue: HTMLElement;
mediaTimingReviewEndValue: HTMLElement;
mediaTimingReviewDuration: HTMLElement;
mediaTimingReviewTimelineStart: HTMLElement;
mediaTimingReviewTimelineEnd: HTMLElement;
mediaTimingReviewSelectionTrack: HTMLDivElement;
mediaTimingReviewWaveformLabel: HTMLElement;
mediaTimingReviewWaveformPath: SVGPathElement;
mediaTimingReviewSelectedRange: HTMLDivElement;
mediaTimingReviewStartHandle: HTMLDivElement;
mediaTimingReviewEndHandle: HTMLDivElement;
mediaTimingReviewShowEarlier: HTMLButtonElement;
mediaTimingReviewShowLater: HTMLButtonElement;
mediaTimingReviewStartBack: HTMLButtonElement;
mediaTimingReviewStartForward: HTMLButtonElement;
mediaTimingReviewEndBack: HTMLButtonElement;
mediaTimingReviewEndForward: HTMLButtonElement;
mediaTimingReviewPlay: HTMLButtonElement;
mediaTimingReviewPlayLabel: HTMLElement;
mediaTimingReviewReset: HTMLButtonElement;
mediaTimingReviewCancel: HTMLButtonElement;
mediaTimingReviewConfirm: HTMLButtonElement;
mediaTimingReviewStatus: HTMLDivElement;
mediaTimingReviewEditor: HTMLDivElement;
mediaTimingReviewCancelStep: HTMLDivElement;
mediaTimingReviewCancelMessage: HTMLParagraphElement;
mediaTimingReviewCancelBack: HTMLButtonElement;
mediaTimingReviewUseOriginal: HTMLButtonElement;
mediaTimingReviewSkipMedia: HTMLButtonElement;
mediaTimingReviewDiscard: HTMLButtonElement;
kikuModal: HTMLDivElement;
kikuCard1: HTMLDivElement;
kikuCard2: HTMLDivElement;
@@ -145,8 +185,8 @@ export type RendererDom = {
playlistBrowserClose: HTMLButtonElement;
};
function getRequiredElement<T extends HTMLElement>(id: string): T {
const element = document.getElementById(id);
function getRequiredElement<T extends Element>(id: string): T {
const element = document.querySelector(`#${id}`);
if (!element) {
throw new Error(`Missing required DOM element #${id}`);
}
@@ -204,6 +244,76 @@ export function resolveRendererDom(): RendererDom {
youtubePickerStatus: getRequiredElement<HTMLDivElement>('youtubePickerStatus'),
youtubePickerTracks: getRequiredElement<HTMLUListElement>('youtubePickerTracks'),
mediaTimingReviewModal: getRequiredElement<HTMLDivElement>('mediaTimingReviewModal'),
mediaTimingReviewKind: getRequiredElement<HTMLDivElement>('mediaTimingReviewKind'),
mediaTimingReviewText: getRequiredElement<HTMLElement>('mediaTimingReviewText'),
mediaTimingReviewLineCount: getRequiredElement<HTMLElement>('mediaTimingReviewLineCount'),
mediaTimingReviewLineControls: getRequiredElement<HTMLDivElement>(
'mediaTimingReviewLineControls',
),
mediaTimingReviewPrevAdd: getRequiredElement<HTMLButtonElement>('mediaTimingReviewPrevAdd'),
mediaTimingReviewPrevRemove: getRequiredElement<HTMLButtonElement>(
'mediaTimingReviewPrevRemove',
),
mediaTimingReviewNextAdd: getRequiredElement<HTMLButtonElement>('mediaTimingReviewNextAdd'),
mediaTimingReviewNextRemove: getRequiredElement<HTMLButtonElement>(
'mediaTimingReviewNextRemove',
),
mediaTimingReviewStartValue: getRequiredElement<HTMLElement>('mediaTimingReviewStartValue'),
mediaTimingReviewEndValue: getRequiredElement<HTMLElement>('mediaTimingReviewEndValue'),
mediaTimingReviewDuration: getRequiredElement<HTMLElement>('mediaTimingReviewDuration'),
mediaTimingReviewTimelineStart: getRequiredElement<HTMLElement>(
'mediaTimingReviewTimelineStart',
),
mediaTimingReviewTimelineEnd: getRequiredElement<HTMLElement>('mediaTimingReviewTimelineEnd'),
mediaTimingReviewSelectionTrack: getRequiredElement<HTMLDivElement>(
'mediaTimingReviewSelectionTrack',
),
mediaTimingReviewWaveformLabel: getRequiredElement<HTMLElement>(
'mediaTimingReviewWaveformLabel',
),
mediaTimingReviewWaveformPath: getRequiredElement<SVGPathElement>(
'mediaTimingReviewWaveformPath',
),
mediaTimingReviewSelectedRange: getRequiredElement<HTMLDivElement>(
'mediaTimingReviewSelectedRange',
),
mediaTimingReviewStartHandle: getRequiredElement<HTMLDivElement>(
'mediaTimingReviewStartHandle',
),
mediaTimingReviewEndHandle: getRequiredElement<HTMLDivElement>('mediaTimingReviewEndHandle'),
mediaTimingReviewShowEarlier: getRequiredElement<HTMLButtonElement>(
'mediaTimingReviewShowEarlier',
),
mediaTimingReviewShowLater: getRequiredElement<HTMLButtonElement>('mediaTimingReviewShowLater'),
mediaTimingReviewStartBack: getRequiredElement<HTMLButtonElement>('mediaTimingReviewStartBack'),
mediaTimingReviewStartForward: getRequiredElement<HTMLButtonElement>(
'mediaTimingReviewStartForward',
),
mediaTimingReviewEndBack: getRequiredElement<HTMLButtonElement>('mediaTimingReviewEndBack'),
mediaTimingReviewEndForward: getRequiredElement<HTMLButtonElement>(
'mediaTimingReviewEndForward',
),
mediaTimingReviewPlay: getRequiredElement<HTMLButtonElement>('mediaTimingReviewPlay'),
mediaTimingReviewPlayLabel: getRequiredElement<HTMLElement>('mediaTimingReviewPlayLabel'),
mediaTimingReviewReset: getRequiredElement<HTMLButtonElement>('mediaTimingReviewReset'),
mediaTimingReviewCancel: getRequiredElement<HTMLButtonElement>('mediaTimingReviewCancel'),
mediaTimingReviewConfirm: getRequiredElement<HTMLButtonElement>('mediaTimingReviewConfirm'),
mediaTimingReviewStatus: getRequiredElement<HTMLDivElement>('mediaTimingReviewStatus'),
mediaTimingReviewEditor: getRequiredElement<HTMLDivElement>('mediaTimingReviewEditor'),
mediaTimingReviewCancelStep: getRequiredElement<HTMLDivElement>('mediaTimingReviewCancelStep'),
mediaTimingReviewCancelMessage: getRequiredElement<HTMLParagraphElement>(
'mediaTimingReviewCancelMessage',
),
mediaTimingReviewCancelBack: getRequiredElement<HTMLButtonElement>(
'mediaTimingReviewCancelBack',
),
mediaTimingReviewUseOriginal: getRequiredElement<HTMLButtonElement>(
'mediaTimingReviewUseOriginal',
),
mediaTimingReviewSkipMedia: getRequiredElement<HTMLButtonElement>('mediaTimingReviewSkipMedia'),
mediaTimingReviewDiscard: getRequiredElement<HTMLButtonElement>('mediaTimingReviewDiscard'),
kikuModal: getRequiredElement<HTMLDivElement>('kikuFieldGroupingModal'),
kikuCard1: getRequiredElement<HTMLDivElement>('kikuCard1'),
kikuCard2: getRequiredElement<HTMLDivElement>('kikuCard2'),
+16
View File
@@ -82,3 +82,19 @@ test('RuntimeOptionsManager keeps known-word and n+1 annotation toggles separate
assert.equal(effective.nPlusOne?.enabled, true);
assert.deepEqual(patches, []);
});
test('RuntimeOptionsManager applies media timing review to the live Anki config', () => {
const baseConfig = structuredClone(DEFAULT_CONFIG.ankiConnect);
const patches: unknown[] = [];
const manager = new RuntimeOptionsManager(() => structuredClone(baseConfig), {
applyAnkiPatch: (patch) => {
patches.push(patch);
},
onOptionsChanged: () => undefined,
});
assert.equal(manager.getOptionValue('anki.mediaReviewTiming'), false);
assert.equal(manager.setOptionValue('anki.mediaReviewTiming', true).ok, true);
assert.equal(manager.getEffectiveAnkiConnectConfig().media?.reviewTiming, true);
assert.deepEqual(patches, [{ media: { reviewTiming: true } }]);
});
+7
View File
@@ -7,6 +7,7 @@ export const OVERLAY_HOSTED_MODALS = [
'jimaku',
'tsukihime',
'youtube-track-picker',
'media-timing-review',
'playlist-browser',
'kiku',
'controller-select',
@@ -125,6 +126,10 @@ export const IPC_CHANNELS = {
syncUiRevealSnapshot: 'sync-ui:reveal-snapshot',
syncUiPickSnapshotFile: 'sync-ui:pick-snapshot-file',
getChangelogSnapshot: 'changelog:get-snapshot',
mediaTimingReviewPreview: 'media-timing-review:preview',
mediaTimingReviewWaveform: 'media-timing-review:waveform',
mediaTimingReviewStopPreview: 'media-timing-review:stop-preview',
mediaTimingReviewResolve: 'media-timing-review:resolve',
},
event: {
subtitleSet: 'subtitle:set',
@@ -142,6 +147,8 @@ export const IPC_CHANNELS = {
jimakuOpen: 'jimaku:open',
tsukihimeOpen: 'tsukihime:open',
youtubePickerOpen: 'youtube:picker-open',
mediaTimingReviewOpen: 'media-timing-review:open',
mediaTimingReviewPreviewEnded: 'media-timing-review:preview-ended',
youtubePickerCancel: 'youtube:picker-cancel',
playlistBrowserOpen: 'playlist-browser:open',
sessionNumericSelectionStart: 'session:numeric-selection-start',
+1
View File
@@ -53,6 +53,7 @@ const SESSION_ACTION_IDS: SessionActionId[] = [
const RUNTIME_OPTION_IDS: RuntimeOptionId[] = [
'anki.autoUpdateNewCards',
'anki.mediaReviewTiming',
'subtitle.annotation.knownWords.highlightEnabled',
'subtitle.annotation.knownWords.maturityEnabled',
'subtitle.annotation.nPlusOne',
+73
View File
@@ -11,6 +11,78 @@ export type CardKind = 'sentence' | 'audio' | 'word-and-sentence' | 'click';
/** Card kind SubMiner flags on word cards; 'none' leaves the flag fields untouched. */
export type WordCardKind = CardKind | 'none';
export type MediaTimingReviewKind = 'word' | 'sentence' | 'audio';
export interface MediaTimingReviewRequest {
kind: MediaTimingReviewKind;
text: string;
startTime: number;
endTime: number;
noteId?: number;
audioPadding: number;
maxMediaDuration: number;
}
/** A subtitle line adjacent to the mined one that the review can pull onto the card. */
export interface MediaTimingReviewContextLine {
text: string;
startTime: number;
endTime: number;
}
export type MediaTimingReviewDecision =
/** `text` is set when the review combined adjacent lines into the card sentence. */
| { action: 'confirm'; startTime: number; endTime: number; text?: string }
| { action: 'use-original' }
| { action: 'skip-media' }
| { action: 'discard' };
export interface MediaTimingReviewOpenPayload {
reviewId: string;
kind: MediaTimingReviewKind;
text: string;
/** Lines before/after the mined one, both chronological: nearest previous line is last, nearest next line is first. */
previousLines: MediaTimingReviewContextLine[];
nextLines: MediaTimingReviewContextLine[];
noteId?: number;
originalStartTime: number;
originalEndTime: number;
selectionStartTime: number;
selectionEndTime: number;
timelineStartTime: number;
timelineEndTime: number;
mediaDuration?: number;
maxMediaDuration: number;
}
export interface MediaTimingReviewPreviewRequest {
reviewId: string;
startTime: number;
endTime: number;
}
export interface MediaTimingReviewWaveformRequest {
reviewId: string;
startTime: number;
endTime: number;
}
export interface MediaTimingReviewWaveformResult extends MediaTimingReviewActionResult {
peaks?: number[];
}
export interface MediaTimingReviewResolveRequest {
reviewId: string;
decision: MediaTimingReviewDecision;
}
export interface MediaTimingReviewActionResult {
ok: boolean;
message?: string;
/** The review this request targeted has already ended; the renderer should close. */
stale?: boolean;
}
export interface NotificationOptions {
body?: string;
icon?: string;
@@ -85,6 +157,7 @@ export interface AnkiConnectConfig {
syncAnimatedImageToWordAudio?: boolean;
normalizeAudio?: boolean;
mirrorMpvVolume?: boolean;
reviewTiming?: boolean;
audioPadding?: number;
fallbackDuration?: number;
maxMediaDuration?: number;
+1
View File
@@ -248,6 +248,7 @@ export interface ResolvedConfig {
syncAnimatedImageToWordAudio: boolean;
normalizeAudio: boolean;
mirrorMpvVolume: boolean;
reviewTiming: boolean;
audioPadding: number;
fallbackDuration: number;
maxMediaDuration: number;
+1
View File
@@ -1,5 +1,6 @@
export type RuntimeOptionId =
| 'anki.autoUpdateNewCards'
| 'anki.mediaReviewTiming'
| 'subtitle.annotation.knownWords.highlightEnabled'
| 'subtitle.annotation.knownWords.maturityEnabled'
| 'subtitle.annotation.nPlusOne'
+20
View File
@@ -3,6 +3,12 @@ import type {
KikuFieldGroupingRequestData,
KikuMergePreviewRequest,
KikuMergePreviewResponse,
MediaTimingReviewActionResult,
MediaTimingReviewOpenPayload,
MediaTimingReviewPreviewRequest,
MediaTimingReviewResolveRequest,
MediaTimingReviewWaveformRequest,
MediaTimingReviewWaveformResult,
} from './anki';
import type { ChangelogSnapshot } from './changelog';
import type { ResolvedConfig, ShortcutsConfig } from './config';
@@ -516,6 +522,18 @@ export interface ElectronAPI {
onOpenJimaku: (callback: () => void) => void;
onOpenTsukihime: (callback: () => void) => void;
onOpenYoutubeTrackPicker: (callback: (payload: YoutubePickerOpenPayload) => void) => void;
onOpenMediaTimingReview: (callback: (payload: MediaTimingReviewOpenPayload) => void) => void;
onMediaTimingReviewPreviewEnded: (callback: (reviewId: string) => void) => void;
previewMediaTimingReview: (
request: MediaTimingReviewPreviewRequest,
) => Promise<MediaTimingReviewActionResult>;
getMediaTimingReviewWaveform: (
request: MediaTimingReviewWaveformRequest,
) => Promise<MediaTimingReviewWaveformResult>;
stopMediaTimingReviewPreview: (reviewId: string) => Promise<MediaTimingReviewActionResult>;
resolveMediaTimingReview: (
request: MediaTimingReviewResolveRequest,
) => Promise<MediaTimingReviewActionResult>;
onOpenPlaylistBrowser: (callback: () => void) => void;
onOpenCharacterDictionaryManager: (callback: () => void) => void;
onSubtitleSidebarToggle: (callback: () => void) => void;
@@ -561,6 +579,7 @@ export interface ElectronAPI {
| 'jimaku'
| 'tsukihime'
| 'youtube-track-picker'
| 'media-timing-review'
| 'playlist-browser'
| 'kiku'
| 'controller-select'
@@ -577,6 +596,7 @@ export interface ElectronAPI {
| 'jimaku'
| 'tsukihime'
| 'youtube-track-picker'
| 'media-timing-review'
| 'playlist-browser'
| 'kiku'
| 'controller-select'
+2
View File
@@ -243,6 +243,8 @@ export interface SubtitleMiningContext {
startTime: number;
endTime: number;
capturedAtMs?: number;
/** Explicit generator padding. Confirmed timing-review ranges set this to zero. */
mediaPaddingSeconds?: number;
}
export interface SubtitleHoverTokenPayload {