diff --git a/changes/media-timing-review.md b/changes/media-timing-review.md new file mode 100644 index 00000000..9b409efd --- /dev/null +++ b/changes/media-timing-review.md @@ -0,0 +1,4 @@ +type: added +area: mining + +- Added optional pre-generation timing review for word, sentence, and audio cards with draggable clip bounds, audio preview, exact screenshot and AVIF timing, and explicit cancellation choices. diff --git a/config.example.jsonc b/config.example.jsonc index 1408eab7..fff1907b 100644 --- a/config.example.jsonc +++ b/config.example.jsonc @@ -523,7 +523,7 @@ // ========================================== // AnkiConnect Integration // Automatic Anki updates and media generation options. - // Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running. + // Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume/reviewTiming, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running. // Shared AI provider transport settings are read from top-level ai and typically require restart. // Most other AnkiConnect settings still require restart. // ========================================== @@ -569,6 +569,7 @@ "syncAnimatedImageToWordAudio": true, // For animated AVIF images, prepend a frozen first frame matching the existing word-audio duration so motion starts with sentence audio. Values: true | false "normalizeAudio": true, // Normalize generated sentence audio loudness during media extraction. Changes apply live. Values: true | false "mirrorMpvVolume": true, // Apply mpv's current software volume curve to generated sentence audio. Changes apply live. Values: true | false + "reviewTiming": false, // Review and preview subtitle media timing before SubMiner creates or enriches a mined card. Values: true | false "audioPadding": 0, // Seconds of padding appended to both ends of generated sentence audio and animated AVIF clips. "fallbackDuration": 3, // Fallback clip duration in seconds when subtitle timing data is unavailable. "maxMediaDuration": 30 // Maximum allowed media clip duration in seconds. diff --git a/docs-site/anki-integration.md b/docs-site/anki-integration.md index d64c8c02..0e9ddc20 100644 --- a/docs-site/anki-integration.md +++ b/docs-site/anki-integration.md @@ -166,6 +166,7 @@ Audio is extracted from the video file using the subtitle's start and end timest "generateAudio": true, "normalizeAudio": true, // normalize generated clip loudness "mirrorMpvVolume": true, // apply the current mpv volume level + "reviewTiming": false, // review and adjust timing before media generation "audioPadding": 0, // optional seconds before and after subtitle timing "maxMediaDuration": 30 // cap total duration in seconds } @@ -178,6 +179,10 @@ Output format: MP3 at 44100 Hz. If the video has multiple audio streams, SubMine The audio is uploaded to Anki's media folder and inserted as `[sound:audio_.mp3]`. +Set `media.reviewTiming` to `true` to pause playback and review each word, sentence, or audio card before its media is generated. The review opens with the subtitle range plus configured audio padding, provides draggable start and end handles, 100 ms adjustments, audio preview, and controls to reveal another five seconds before or after the visible timeline. The confirmed range is exact: SubMiner does not apply audio padding a second time. Static screenshots use its midpoint, and animated AVIF clips use the full confirmed range. + +Canceling the review lets you keep editing, finish with the original timing, or discard the card. Discard deletes an existing Yomitan or audio card and skips creation for a direct sentence card. Clipboard updates and stats-dashboard mining do not open timing review. Audio preview failure does not block confirmation or card creation. The option is disabled by default and hot-reloads. + ### Screenshots (Static) A single frame is captured at the current playback position. diff --git a/docs-site/configuration.md b/docs-site/configuration.md index 6726c5b3..5653aa0a 100644 --- a/docs-site/configuration.md +++ b/docs-site/configuration.md @@ -967,6 +967,7 @@ Enable automatic Anki card creation and updates with media generation: "animatedCrf": 35, "normalizeAudio": true, "mirrorMpvVolume": true, + "reviewTiming": false, "audioPadding": 0, "fallbackDuration": 3, "maxMediaDuration": 30 @@ -1019,6 +1020,7 @@ This example is intentionally compact. The option table below documents availabl | `media.generateAudio` | `true`, `false` | Generate audio clips from video (default: `true`) | | `media.normalizeAudio` | `true`, `false` | Normalize generated sentence-audio loudness during media extraction (default: `true`). Set to `false` to keep raw source loudness. Changes apply live. | | `media.mirrorMpvVolume` | `true`, `false` | Apply mpv's cubic software-volume curve to each generated sentence-audio clip (default: `true`). This ignores mpv's separate mute state, falls back to unity scaling if volume cannot be read, and applies changes live. | +| `media.reviewTiming` | `true`, `false` | Pause playback and review word, sentence, and audio card timing before media generation (default: `false`). Clipboard updates and stats-dashboard mining do not open the review. | | `media.generateImage` | `true`, `false` | Generate image/animation screenshots (default: `true`) | | `media.imageType` | `"static"`, `"avif"` | Image type: static screenshot or animated AVIF (default: `"static"`) | | `media.imageFormat` | `"jpg"`, `"png"`, `"webp"` | Image format (default: `"jpg"`) | diff --git a/docs-site/public/config.example.jsonc b/docs-site/public/config.example.jsonc index 1408eab7..fff1907b 100644 --- a/docs-site/public/config.example.jsonc +++ b/docs-site/public/config.example.jsonc @@ -523,7 +523,7 @@ // ========================================== // AnkiConnect Integration // Automatic Anki updates and media generation options. - // Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running. + // Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume/reviewTiming, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running. // Shared AI provider transport settings are read from top-level ai and typically require restart. // Most other AnkiConnect settings still require restart. // ========================================== @@ -569,6 +569,7 @@ "syncAnimatedImageToWordAudio": true, // For animated AVIF images, prepend a frozen first frame matching the existing word-audio duration so motion starts with sentence audio. Values: true | false "normalizeAudio": true, // Normalize generated sentence audio loudness during media extraction. Changes apply live. Values: true | false "mirrorMpvVolume": true, // Apply mpv's current software volume curve to generated sentence audio. Changes apply live. Values: true | false + "reviewTiming": false, // Review and preview subtitle media timing before SubMiner creates or enriches a mined card. Values: true | false "audioPadding": 0, // Seconds of padding appended to both ends of generated sentence audio and animated AVIF clips. "fallbackDuration": 3, // Fallback clip duration in seconds when subtitle timing data is unavailable. "maxMediaDuration": 30 // Maximum allowed media clip duration in seconds. diff --git a/src/anki-integration.ts b/src/anki-integration.ts index fcc118e1..33038d9b 100644 --- a/src/anki-integration.ts +++ b/src/anki-integration.ts @@ -28,6 +28,8 @@ import { KikuMergePreviewResponse, NotificationOptions, type WordCardKind, + type MediaTimingReviewDecision, + type MediaTimingReviewRequest, } from './types/anki'; import { AiConfig } from './types/integrations'; import type { KnownWordMaturityTier } from './types/subtitle'; @@ -238,6 +240,9 @@ export class AnkiIntegration { private recordCardsMinedCallback: ((count: number, noteIds?: number[]) => void) | null = null; private knownWordCacheUpdatedCallback: (() => void) | null = null; private consumeSubtitleMiningContextCallback: (() => SubtitleMiningContext | null) | null = null; + private mediaTimingReviewCallback: + | ((request: MediaTimingReviewRequest) => Promise) + | null = null; private noteIdRedirects = new Map(); private trackedDuplicateNoteIds = new Map(); private getCachedMediaPath: MediaGenerationInputResolverOptions['getCachedMediaPath'] | null = @@ -509,6 +514,7 @@ export class AnkiIntegration { findNotes: async (query, options) => (await this.client.findNotes(query, options)) as number[], retrieveMediaFile: (filename) => this.client.retrieveMediaFile(filename), + deleteNotes: (noteIds) => this.client.deleteNotes(noteIds), }, mediaGenerator: { generateAudio: ( @@ -566,6 +572,7 @@ export class AnkiIntegration { getEffectiveSentenceCardConfig: () => this.getEffectiveSentenceCardConfig(), getFallbackDurationSeconds: () => this.getFallbackDurationSeconds(), appendKnownWordsFromNoteInfo: (noteInfo) => this.appendKnownWordsFromNoteInfo(noteInfo), + removeKnownWordNote: (noteId) => this.removeKnownWordNote(noteId), isUpdateInProgress: () => this.updateInProgress, setUpdateInProgress: (value) => { this.updateInProgress = value; @@ -581,6 +588,7 @@ export class AnkiIntegration { recordCardsMinedCallback: (count, noteIds) => { this.recordCardsMinedSafely(count, noteIds, 'card creation'); }, + reviewMediaTiming: (request) => this.reviewMediaTiming(request), }); } @@ -637,12 +645,14 @@ export class AnkiIntegration { notesInfo: async (noteIds) => (await this.client.notesInfo(noteIds)) as unknown, updateNoteFields: (noteId, fields) => this.client.updateNoteFields(noteId, fields), storeMediaFile: (filename, data) => this.client.storeMediaFile(filename, data), + deleteNotes: (noteIds) => this.client.deleteNotes(noteIds), }, getConfig: () => this.config, getCurrentSubtitleText: () => this.mpvClient.currentSubText, getCurrentSubtitleStart: () => this.mpvClient.currentSubStart, getEffectiveSentenceCardConfig: () => this.getEffectiveSentenceCardConfig(), appendKnownWordsFromNoteInfo: (noteInfo) => this.appendKnownWordsFromNoteInfo(noteInfo), + removeKnownWordNote: (noteId) => this.removeKnownWordNote(noteId), extractFields: (fields) => this.extractFields(fields), findDuplicateNote: (expression, excludeNoteId, noteInfo) => this.findDuplicateNote(expression, excludeNoteId, noteInfo), @@ -680,6 +690,7 @@ export class AnkiIntegration { logWarn: (...args) => log.warn(args[0] as string, ...args.slice(1)), logInfo: (...args) => log.info(args[0] as string, ...args.slice(1)), logError: (...args) => log.error(args[0] as string, ...args.slice(1)), + reviewMediaTiming: (request) => this.reviewMediaTiming(request), }); } @@ -799,6 +810,12 @@ export class AnkiIntegration { } } + private removeKnownWordNote(noteId: number): void { + if (this.knownWordCache.removeNote(noteId)) { + this.notifyKnownWordCacheUpdated(); + } + } + private notifyKnownWordCacheUpdated(): void { if (!this.knownWordCacheUpdatedCallback) { return; @@ -1039,7 +1056,7 @@ export class AnkiIntegration { videoPath, startTime, endTime, - this.config.media?.audioPadding, + context?.mediaPaddingSeconds ?? this.config.media?.audioPadding, resolveAudioStreamIndexForMediaGeneration(videoPath, this.mpvClient.currentAudioStreamIndex), this.config.media?.normalizeAudio !== false, await this.getMpvVolumeScale(), @@ -1072,7 +1089,7 @@ export class AnkiIntegration { videoPath, mediaRange.startTime, mediaRange.endTime, - this.config.media?.audioPadding, + context?.mediaPaddingSeconds ?? this.config.media?.audioPadding, { fps: this.config.media?.animatedFps, maxWidth: this.config.media?.animatedMaxWidth, @@ -1723,6 +1740,25 @@ export class AnkiIntegration { this.consumeSubtitleMiningContextCallback = callback; } + setMediaTimingReviewCallback( + callback: ((request: MediaTimingReviewRequest) => Promise) | null, + ): void { + this.mediaTimingReviewCallback = callback; + } + + private async reviewMediaTiming( + request: Omit, + ): Promise { + if (this.config.media?.reviewTiming !== true || !this.mediaTimingReviewCallback) { + return { action: 'use-original' }; + } + return await this.mediaTimingReviewCallback({ + ...request, + audioPadding: Math.max(0, this.config.media.audioPadding ?? 0), + maxMediaDuration: Math.max(0, this.config.media.maxMediaDuration ?? 30), + }); + } + resolveCurrentNoteId(noteId: number): number { let resolved = noteId; const seen = new Set(); diff --git a/src/anki-integration/card-creation-manual-update.test.ts b/src/anki-integration/card-creation-manual-update.test.ts index 59cbe267..836b92eb 100644 --- a/src/anki-integration/card-creation-manual-update.test.ts +++ b/src/anki-integration/card-creation-manual-update.test.ts @@ -85,6 +85,7 @@ function createManualUpdateService(overrides: Partial = {}): { }, findNotes: async () => [42], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, mediaGenerator: { generateAudio: async () => Buffer.from('audio'), @@ -129,6 +130,7 @@ function createManualUpdateService(overrides: Partial = {}): { }), getFallbackDurationSeconds: () => 10, appendKnownWordsFromNoteInfo: () => undefined, + removeKnownWordNote: () => undefined, isUpdateInProgress: () => false, setUpdateInProgress: () => undefined, trackLastAddedNoteId: () => undefined, @@ -201,6 +203,7 @@ test('manual clipboard subtitle update marks Kiku word cards as word-and-sentenc storeMediaFile: async () => undefined, findNotes: async () => [42], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, getEffectiveSentenceCardConfig: () => ({ model: 'Sentence', @@ -248,6 +251,7 @@ test('manual clipboard subtitle update skips audio when sentence audio field is }, findNotes: async () => [42], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, }); @@ -335,6 +339,7 @@ test('manual clipboard subtitle update uses resolved mpv stream URLs for remote }, findNotes: async () => [42], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, mediaGenerator: { generateAudio: async (path) => { @@ -383,3 +388,47 @@ test('createSentenceCard relies on Anki progress notification without standalone assert.deepEqual(progressMessages, ['Creating sentence card']); assert.deepEqual(statusMessages, []); }); + +test('discarding an audio-card timing review deletes the note before evicting its cache entry', async () => { + const events: string[] = []; + const statusMessages: string[] = []; + const { service } = createManualUpdateService({ + getMpvClient: () => + ({ + currentVideoPath: '/video.mp4', + currentSubText: '字幕', + currentSubStart: 4, + currentSubEnd: 6, + currentTimePos: 5, + }) as never, + client: { + addNote: async () => 0, + addTags: async () => undefined, + notesInfo: async () => [ + { + noteId: 42, + fields: { Expression: { value: '単語' } }, + }, + ], + updateNoteFields: async () => undefined, + storeMediaFile: async () => undefined, + findNotes: async () => [42], + retrieveMediaFile: async () => '', + deleteNotes: async (noteIds) => { + events.push(`delete:${noteIds.join(',')}`); + }, + }, + reviewMediaTiming: async () => ({ action: 'discard' }), + removeKnownWordNote: (noteId) => { + events.push(`cache:${noteId}`); + }, + showStatusNotification: (message) => { + statusMessages.push(message); + }, + }); + + await service.markLastCardAsAudioCard(); + + assert.deepEqual(events, ['delete:42', 'cache:42']); + assert.deepEqual(statusMessages, ['Card deleted.']); +}); diff --git a/src/anki-integration/card-creation-sentence-media.test.ts b/src/anki-integration/card-creation-sentence-media.test.ts index d8bb27cc..00f93804 100644 --- a/src/anki-integration/card-creation-sentence-media.test.ts +++ b/src/anki-integration/card-creation-sentence-media.test.ts @@ -12,6 +12,7 @@ test('sentence card writes generated audio only to sentence audio field', async const storedMedia: string[] = []; const requestedProperties: string[] = []; const audioVolumeScales: Array = []; + const audioRanges: Array<{ start: number; end: number; padding: number | undefined }> = []; const deps: CardCreationDeps = { getConfig: () => @@ -73,17 +74,19 @@ test('sentence card writes generated audio only to sentence audio field', async }, findNotes: async () => [], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, mediaGenerator: { generateAudio: async ( _path, - _startTime, - _endTime, - _audioPadding, + startTime, + endTime, + audioPadding, _audioStreamIndex, _normalizeAudio, volumeScale, ) => { + audioRanges.push({ start: startTime, end: endTime, padding: audioPadding }); audioVolumeScales.push(volumeScale); return Buffer.from('audio'); }, @@ -122,17 +125,15 @@ test('sentence card writes generated audio only to sentence audio field', async }), getFallbackDurationSeconds: () => 10, appendKnownWordsFromNoteInfo: () => undefined, + removeKnownWordNote: () => undefined, isUpdateInProgress: () => false, setUpdateInProgress: () => undefined, trackLastAddedNoteId: () => undefined, + reviewMediaTiming: async () => ({ action: 'confirm', startTime: 11.4, endTime: 14.2 }), }; - const created = await new CardCreationService(deps).createSentenceCard( - '字幕', - 12, - 14, - 'Subtitle', - ); + const service = new CardCreationService(deps); + const created = await service.createSentenceCard('字幕', 12, 14, 'Subtitle'); assert.equal(created, true); assert.deepEqual(addedFields[0], { @@ -144,7 +145,12 @@ test('sentence card writes generated audio only to sentence audio field', async assert.equal(storedMedia.length, 1); assert.deepEqual(requestedProperties, ['volume']); assert.deepEqual(audioVolumeScales, [0.4 ** 3]); + assert.deepEqual(audioRanges, [{ start: 11.4, end: 14.2, padding: 0 }]); const mediaUpdate = updatedFields.find((fields) => 'SentenceAudio' in fields); assert.equal(mediaUpdate?.SentenceAudio, `[sound:${storedMedia[0]}]`); assert.equal('ExpressionAudio' in mediaUpdate!, false); + + deps.reviewMediaTiming = async () => ({ action: 'discard' }); + assert.equal(await service.createSentenceCard('作らない', 20, 22), false); + assert.equal(addedFields.length, 1); }); diff --git a/src/anki-integration/card-creation.test.ts b/src/anki-integration/card-creation.test.ts index 19159887..2c1a2bd0 100644 --- a/src/anki-integration/card-creation.test.ts +++ b/src/anki-integration/card-creation.test.ts @@ -42,6 +42,7 @@ test('CardCreationService counts locally created sentence cards', async () => { storeMediaFile: async () => undefined, findNotes: async () => [], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, mediaGenerator: { generateAudio: async () => null, @@ -74,6 +75,7 @@ test('CardCreationService counts locally created sentence cards', async () => { }), getFallbackDurationSeconds: () => 10, appendKnownWordsFromNoteInfo: () => undefined, + removeKnownWordNote: () => undefined, isUpdateInProgress: () => false, setUpdateInProgress: () => undefined, trackLastAddedNoteId: () => undefined, @@ -139,6 +141,7 @@ test('CardCreationService keeps updating after trackLastAddedNoteId throws', asy storeMediaFile: async () => undefined, findNotes: async () => [], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, mediaGenerator: { generateAudio: async () => null, @@ -173,6 +176,7 @@ test('CardCreationService keeps updating after trackLastAddedNoteId throws', asy }), getFallbackDurationSeconds: () => 10, appendKnownWordsFromNoteInfo: () => undefined, + removeKnownWordNote: () => undefined, isUpdateInProgress: () => false, setUpdateInProgress: () => undefined, trackLastAddedNoteId: () => { @@ -238,6 +242,7 @@ test('CardCreationService keeps updating after recordCardsMinedCallback throws', storeMediaFile: async () => undefined, findNotes: async () => [], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, mediaGenerator: { generateAudio: async () => null, @@ -272,6 +277,7 @@ test('CardCreationService keeps updating after recordCardsMinedCallback throws', }), getFallbackDurationSeconds: () => 10, appendKnownWordsFromNoteInfo: () => undefined, + removeKnownWordNote: () => undefined, isUpdateInProgress: () => false, setUpdateInProgress: () => undefined, recordCardsMinedCallback: () => { @@ -348,6 +354,7 @@ test('CardCreationService uses stream-open-filename for remote media generation' storeMediaFile: async () => undefined, findNotes: async () => [], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, mediaGenerator: { generateAudio: async (path) => { @@ -392,6 +399,7 @@ test('CardCreationService uses stream-open-filename for remote media generation' }), getFallbackDurationSeconds: () => 10, appendKnownWordsFromNoteInfo: () => undefined, + removeKnownWordNote: () => undefined, isUpdateInProgress: () => false, setUpdateInProgress: () => undefined, trackLastAddedNoteId: () => undefined, @@ -454,6 +462,7 @@ test('CardCreationService does not use mpv stream indexes for ready cached YouTu storeMediaFile: async () => undefined, findNotes: async () => [], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, mediaGenerator: { generateAudio: async (path, _startTime, _endTime, _padding, audioStreamIndex) => { @@ -495,6 +504,7 @@ test('CardCreationService does not use mpv stream indexes for ready cached YouTu }), getFallbackDurationSeconds: () => 10, appendKnownWordsFromNoteInfo: () => undefined, + removeKnownWordNote: () => undefined, isUpdateInProgress: () => false, setUpdateInProgress: () => undefined, trackLastAddedNoteId: () => undefined, @@ -590,6 +600,7 @@ test('CardCreationService queues YouTube media when required cache is not ready' storeMediaFile: async () => undefined, findNotes: async () => [], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, mediaGenerator: { generateAudio: async () => { @@ -634,6 +645,7 @@ test('CardCreationService queues YouTube media when required cache is not ready' }), getFallbackDurationSeconds: () => 10, appendKnownWordsFromNoteInfo: () => undefined, + removeKnownWordNote: () => undefined, isUpdateInProgress: () => false, setUpdateInProgress: () => undefined, trackLastAddedNoteId: () => undefined, @@ -701,6 +713,7 @@ test('CardCreationService tracks pre-add duplicate note ids for kiku sentence ca storeMediaFile: async () => undefined, findNotes: async () => [], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, mediaGenerator: { generateAudio: async () => null, @@ -733,6 +746,7 @@ test('CardCreationService tracks pre-add duplicate note ids for kiku sentence ca }), getFallbackDurationSeconds: () => 10, appendKnownWordsFromNoteInfo: () => undefined, + removeKnownWordNote: () => undefined, isUpdateInProgress: () => false, setUpdateInProgress: () => undefined, trackLastAddedNoteId: () => undefined, @@ -790,6 +804,7 @@ test('CardCreationService does not track duplicate ids when pre-add lookup retur storeMediaFile: async () => undefined, findNotes: async () => [], retrieveMediaFile: async () => '', + deleteNotes: async () => undefined, }, mediaGenerator: { generateAudio: async () => null, @@ -822,6 +837,7 @@ test('CardCreationService does not track duplicate ids when pre-add lookup retur }), getFallbackDurationSeconds: () => 10, appendKnownWordsFromNoteInfo: () => undefined, + removeKnownWordNote: () => undefined, isUpdateInProgress: () => false, setUpdateInProgress: () => undefined, trackLastAddedNoteId: () => undefined, diff --git a/src/anki-integration/card-creation.ts b/src/anki-integration/card-creation.ts index abfe41f3..51a28e76 100644 --- a/src/anki-integration/card-creation.ts +++ b/src/anki-integration/card-creation.ts @@ -3,7 +3,13 @@ import { getConfiguredWordFieldName, getPreferredWordValueFromExtractedFields, } from '../anki-field-config'; -import { AnkiConnectConfig, type CardKind, type WordCardKind } from '../types/anki'; +import { + AnkiConnectConfig, + type CardKind, + type MediaTimingReviewDecision, + type MediaTimingReviewRequest, + type WordCardKind, +} from '../types/anki'; import { createLogger } from '../logger'; import type { MediaInput } from '../media-input'; import { SubtitleTimingTracker } from '../subtitle-timing-tracker'; @@ -55,6 +61,7 @@ interface CardCreationClient { storeMediaFile(filename: string, data: Buffer): Promise; findNotes(query: string, options?: { maxRetries?: number }): Promise; retrieveMediaFile(filename: string): Promise; + deleteNotes(noteIds: number[]): Promise; } interface CardCreationMediaGenerator { @@ -138,12 +145,16 @@ interface CardCreationDeps { }; getFallbackDurationSeconds: () => number; appendKnownWordsFromNoteInfo: (noteInfo: CardCreationNoteInfo) => void; + removeKnownWordNote: (noteId: number) => void; isUpdateInProgress: () => boolean; setUpdateInProgress: (value: boolean) => void; trackLastAddedNoteId?: (noteId: number) => void; trackLastAddedDuplicateNoteIds?: (noteId: number, duplicateNoteIds: number[]) => void; findDuplicateNoteIds?: (expression: string, noteInfo: CardCreationNoteInfo) => Promise; recordCardsMinedCallback?: (count: number, noteIds?: number[]) => void; + reviewMediaTiming?: ( + request: Omit, + ) => Promise; } export class CardCreationService { @@ -260,6 +271,7 @@ export class CardCreationService { fields, this.deps.getConfig(), ); + const sentenceAudioField = this.getResolvedSentenceOnlyAudioFieldName(noteInfo); const sentenceCardConfig = this.deps.getEffectiveSentenceCardConfig(); const sentenceField = sentenceCardConfig.sentenceField; @@ -451,6 +463,27 @@ export class CardCreationService { this.deps.getConfig(), ); + const timingDecision = this.deps.reviewMediaTiming + ? await this.deps.reviewMediaTiming({ + kind: 'audio', + text: mpvClient.currentSubText, + startTime, + endTime, + noteId, + }) + : ({ action: 'use-original' } as const); + if (timingDecision.action === 'discard') { + await this.deps.client.deleteNotes([noteId]); + this.deps.removeKnownWordNote(noteId); + this.deps.showStatusNotification('Card deleted.'); + return; + } + const exactReviewedRange = timingDecision.action === 'confirm'; + if (timingDecision.action === 'confirm') { + startTime = timingDecision.startTime; + endTime = timingDecision.endTime; + } + const updatedFields: Record = {}; const errors: string[] = []; let miscInfoFilename: string | null = null; @@ -471,6 +504,7 @@ export class CardCreationService { mpvClient.currentVideoPath, startTime, endTime, + exactReviewedRange ? 0 : undefined, ); if (audioBuffer) { @@ -492,6 +526,7 @@ export class CardCreationService { startTime, endTime, animatedLeadInSeconds, + exactReviewedRange, ); const imageField = this.deps.getConfig().fields?.image; @@ -564,6 +599,24 @@ export class CardCreationService { try { return await this.deps.withUpdateProgress('Creating sentence card', async () => { + const timingDecision = this.deps.reviewMediaTiming + ? await this.deps.reviewMediaTiming({ + kind: 'sentence', + text: sentence, + startTime, + endTime, + }) + : ({ action: 'use-original' } as const); + if (timingDecision.action === 'discard') { + this.deps.showStatusNotification('Card creation cancelled.'); + return false; + } + const exactReviewedRange = timingDecision.action === 'confirm'; + if (timingDecision.action === 'confirm') { + startTime = timingDecision.startTime; + endTime = timingDecision.endTime; + } + const config = this.deps.getConfig(); const generateAudio = shouldGenerateAudio(config); const generateImage = shouldGenerateImage(config); @@ -732,6 +785,7 @@ export class CardCreationService { generateAudio, generateImage, volumeScale, + ...(exactReviewedRange ? { mediaPaddingSeconds: 0 } : {}), }); await this.deps.showNotification(noteId, label, 'media queued'); return true; @@ -747,7 +801,12 @@ export class CardCreationService { try { const audioFilename = this.generateAudioFilename(); const audioBuffer = audioSourcePath - ? await this.mediaGenerateAudio(audioSourcePath, startTime, endTime) + ? await this.mediaGenerateAudio( + audioSourcePath, + startTime, + endTime, + exactReviewedRange ? 0 : undefined, + ) : null; if (audioBuffer) { @@ -765,7 +824,13 @@ export class CardCreationService { if (generateImage) { try { const imageFilename = this.generateImageFilename(); - const imageBuffer = await this.generateImageBuffer(videoPath!, startTime, endTime); + const imageBuffer = await this.generateImageBuffer( + videoPath!, + startTime, + endTime, + 0, + exactReviewedRange, + ); const imageField = config.fields?.image; if (imageBuffer && imageField) { @@ -833,6 +898,7 @@ export class CardCreationService { videoPath: MediaInput, startTime: number, endTime: number, + audioPaddingOverride?: number, ): Promise { const mpvClient = this.deps.getMpvClient(); if (!mpvClient) { @@ -843,7 +909,7 @@ export class CardCreationService { videoPath, startTime, endTime, - this.deps.getConfig().media?.audioPadding, + audioPaddingOverride ?? this.deps.getConfig().media?.audioPadding, resolveAudioStreamIndexForMediaGeneration( videoPath, mpvClient.currentAudioStreamIndex ?? undefined, @@ -861,13 +927,16 @@ export class CardCreationService { startTime: number, endTime: number, animatedLeadInSeconds = 0, + exactReviewedRange = false, ): Promise { const mpvClient = this.deps.getMpvClient(); if (!mpvClient) { return null; } - const timestamp = mpvClient.currentTimePos || 0; + const timestamp = exactReviewedRange + ? startTime + (endTime - startTime) / 2 + : mpvClient.currentTimePos || 0; if (this.deps.getConfig().media?.imageType === 'avif') { let imageStart = startTime; @@ -883,7 +952,7 @@ export class CardCreationService { videoPath, imageStart, imageEnd, - this.deps.getConfig().media?.audioPadding, + exactReviewedRange ? 0 : this.deps.getConfig().media?.audioPadding, { fps: this.deps.getConfig().media?.animatedFps, maxWidth: this.deps.getConfig().media?.animatedMaxWidth, diff --git a/src/anki-integration/known-word-cache.test.ts b/src/anki-integration/known-word-cache.test.ts index 3d54036b..306fdba8 100644 --- a/src/anki-integration/known-word-cache.test.ts +++ b/src/anki-integration/known-word-cache.test.ts @@ -261,6 +261,32 @@ test('KnownWordCacheManager invalidates persisted cache when fields.word changes } }); +test('KnownWordCacheManager removes a deleted note from memory and persisted state', () => { + const config: AnkiConnectConfig = { + fields: { word: 'Word' }, + knownWords: { highlightEnabled: true }, + }; + const { manager, statePath, cleanup } = createKnownWordCacheHarness(config); + + try { + manager.appendFromNoteInfo({ + noteId: 42, + fields: { Word: { value: '猫' } }, + }); + + assert.equal(manager.removeNote(42), true); + assert.equal(manager.removeNote(42), false); + assert.equal(manager.isKnownWord('猫'), false); + + const persisted = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as { + notes?: Record; + }; + assert.deepEqual(persisted.notes, {}); + } finally { + cleanup(); + } +}); + test('KnownWordCacheManager refresh incrementally reconciles deleted and edited note words', async () => { const config: AnkiConnectConfig = { fields: { diff --git a/src/anki-integration/known-word-cache.ts b/src/anki-integration/known-word-cache.ts index f1fc16b7..436d3d71 100644 --- a/src/anki-integration/known-word-cache.ts +++ b/src/anki-integration/known-word-cache.ts @@ -350,6 +350,17 @@ export class KnownWordCacheManager { return true; } + removeNote(noteId: number): boolean { + if (!this.noteEntriesById.has(noteId)) { + return false; + } + + this.removeNoteSnapshot(noteId); + this.persistKnownWordCacheState(); + log.info('Known-word cache removed deleted note', `noteId=${noteId}`); + return true; + } + clearKnownWordCacheState(): void { this.clearInMemoryState(); this.knownWordsStateKey = this.getKnownWordCacheStateKey(); diff --git a/src/anki-integration/note-update-workflow.test.ts b/src/anki-integration/note-update-workflow.test.ts index 2138b538..2a08624f 100644 --- a/src/anki-integration/note-update-workflow.test.ts +++ b/src/anki-integration/note-update-workflow.test.ts @@ -44,6 +44,7 @@ function createWorkflowHarness() { updates.push({ noteId, fields }); }, storeMediaFile: async () => undefined, + deleteNotes: async () => undefined, }, getConfig: () => ({ fields: { @@ -61,6 +62,7 @@ function createWorkflowHarness() { kikuFieldGrouping: 'disabled' as const, }), appendKnownWordsFromNoteInfo: (_noteInfo: NoteUpdateWorkflowNoteInfo) => undefined, + removeKnownWordNote: (_noteId: number) => undefined, extractFields: (fields: Record) => { const out: Record = {}; for (const [key, value] of Object.entries(fields)) { @@ -592,3 +594,62 @@ test('NoteUpdateWorkflow queues media updates when YouTube cache is pending', as assert.equal(queuedUpdates[0]?.context, undefined); assert.deepEqual(harness.updates, [{ noteId: 42, fields: { Sentence: 'subtitle-text' } }]); }); + +test('NoteUpdateWorkflow deletes an existing word card when timing review discards it', async () => { + const harness = createWorkflowHarness(); + const deletedNoteIds: number[][] = []; + const removedKnownWordNoteIds: number[] = []; + let appendedKnownWords = false; + harness.deps.captureSubtitleMediaContext = () => ({ + source: 'overlay', + text: 'subtitle-text', + startTime: 4, + endTime: 6, + }); + harness.deps.client.deleteNotes = async (noteIds) => { + deletedNoteIds.push(noteIds); + }; + harness.deps.appendKnownWordsFromNoteInfo = () => { + appendedKnownWords = true; + }; + harness.deps.removeKnownWordNote = (noteId) => { + removedKnownWordNoteIds.push(noteId); + }; + harness.deps.reviewMediaTiming = async () => ({ action: 'discard' }); + + await harness.workflow.execute(42); + + assert.deepEqual(deletedNoteIds, [[42]]); + assert.deepEqual(removedKnownWordNoteIds, [42]); + assert.equal(appendedKnownWords, false); + assert.deepEqual(harness.updates, []); + assert.deepEqual(harness.notifications, []); +}); + +test('NoteUpdateWorkflow keeps cache unchanged and reports when deletion fails', async () => { + const harness = createWorkflowHarness(); + const statusMessages: string[] = []; + let removedKnownWord = false; + harness.deps.captureSubtitleMediaContext = () => ({ + source: 'overlay', + text: 'subtitle-text', + startTime: 4, + endTime: 6, + }); + harness.deps.client.deleteNotes = async () => { + throw new Error('delete failed'); + }; + harness.deps.removeKnownWordNote = () => { + removedKnownWord = true; + }; + harness.deps.showOsdNotification = (message) => { + statusMessages.push(message); + }; + harness.deps.reviewMediaTiming = async () => ({ action: 'discard' }); + + await harness.workflow.execute(42); + + assert.equal(removedKnownWord, false); + assert.deepEqual(statusMessages, ['Card deletion failed: delete failed']); + assert.ok(harness.warnings.length === 0); +}); diff --git a/src/anki-integration/note-update-workflow.ts b/src/anki-integration/note-update-workflow.ts index 0348bdeb..9aee0e0c 100644 --- a/src/anki-integration/note-update-workflow.ts +++ b/src/anki-integration/note-update-workflow.ts @@ -1,7 +1,12 @@ import { DEFAULT_ANKI_CONNECT_CONFIG } from '../config'; import { getPreferredWordValueFromExtractedFields } from '../anki-field-config'; import type { SubtitleMiningContext } from '../types/subtitle'; -import type { CardKind, WordCardKind } from '../types/anki'; +import type { + CardKind, + MediaTimingReviewDecision, + MediaTimingReviewRequest, + WordCardKind, +} from '../types/anki'; import { resolveWordCardKind } from './note-field-utils'; export interface NoteUpdateWorkflowNoteInfo { @@ -14,6 +19,7 @@ export interface NoteUpdateWorkflowDeps { notesInfo(noteIds: number[]): Promise; updateNoteFields(noteId: number, fields: Record): Promise; storeMediaFile(filename: string, data: Buffer): Promise; + deleteNotes(noteIds: number[]): Promise; }; getConfig: () => { fields?: { @@ -43,6 +49,7 @@ export interface NoteUpdateWorkflowDeps { wordCardKind?: WordCardKind; }; appendKnownWordsFromNoteInfo: (noteInfo: NoteUpdateWorkflowNoteInfo) => void; + removeKnownWordNote: (noteId: number) => void; extractFields: (fields: Record) => Record; findDuplicateNote: ( expression: string, @@ -102,6 +109,9 @@ export interface NoteUpdateWorkflowDeps { logWarn: (message: string, ...args: unknown[]) => void; logInfo: (message: string, ...args: unknown[]) => void; logError: (message: string, ...args: unknown[]) => void; + reviewMediaTiming?: ( + request: Omit, + ) => Promise; } function normalizeSubtitleContextText(text: string): string { @@ -171,7 +181,6 @@ export class NoteUpdateWorkflow { } const noteInfo = notesInfo[0]!; - this.deps.appendKnownWordsFromNoteInfo(noteInfo); const fields = this.deps.extractFields(noteInfo.fields); const config = this.deps.getConfig(); @@ -207,10 +216,45 @@ export class NoteUpdateWorkflow { // Audio and image generation run sequentially and audio extraction can take tens of // seconds, so resolve the clip range exactly once up front; reading live mpv sub // timings per generator clips whichever line is on screen when each one starts. - const mediaTimingContext = + let mediaTimingContext = subtitleMiningContext ?? this.deps.captureSubtitleMediaContext?.() ?? null; const noteLabel = hasExpressionText ? expressionText : noteId; + if (mediaTimingContext) { + const timingDecision = this.deps.reviewMediaTiming + ? await this.deps.reviewMediaTiming({ + kind: 'word', + text: mediaTimingContext.text, + startTime: mediaTimingContext.startTime, + endTime: mediaTimingContext.endTime, + noteId, + }) + : ({ action: 'use-original' } as const); + if (timingDecision.action === 'discard') { + try { + await this.deps.client.deleteNotes([noteId]); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.deps.logError('Failed to delete discarded card:', message); + this.deps.showOsdNotification(`Card deletion failed: ${message}`); + return; + } + this.deps.removeKnownWordNote(noteId); + this.deps.showOsdNotification('Card deleted.'); + return; + } + if (timingDecision.action === 'confirm') { + mediaTimingContext = { + ...mediaTimingContext, + startTime: timingDecision.startTime, + endTime: timingDecision.endTime, + mediaPaddingSeconds: 0, + }; + } + } + + this.deps.appendKnownWordsFromNoteInfo(noteInfo); + const currentSubtitleText = subtitleMiningContext?.text ?? this.deps.getCurrentSubtitleText(); if (sentenceField && currentSubtitleText) { const processedSentence = this.deps.processSentence(currentSubtitleText, fields); diff --git a/src/anki-integration/pending-youtube-media-queue.ts b/src/anki-integration/pending-youtube-media-queue.ts index 68abd2c9..969e6275 100644 --- a/src/anki-integration/pending-youtube-media-queue.ts +++ b/src/anki-integration/pending-youtube-media-queue.ts @@ -148,6 +148,9 @@ export class PendingYoutubeMediaQueue { generateAudio: shouldGenerateAudio(config), generateImage: shouldGenerateImage(config), volumeScale, + ...(job.context?.mediaPaddingSeconds !== undefined + ? { mediaPaddingSeconds: job.context.mediaPaddingSeconds } + : {}), }); return true; } @@ -275,7 +278,7 @@ export class PendingYoutubeMediaQueue { cachedMediaInput, job.startTime, job.endTime, - config.media?.audioPadding, + job.mediaPaddingSeconds ?? config.media?.audioPadding, undefined, config.media?.normalizeAudio !== false, job.volumeScale, @@ -309,6 +312,7 @@ export class PendingYoutubeMediaQueue { job.startTime, job.endTime, animatedLeadInSeconds, + job.mediaPaddingSeconds, ); if (imageBuffer) { await this.deps.client.storeMediaFile(imageFilename, imageBuffer); @@ -369,6 +373,7 @@ export class PendingYoutubeMediaQueue { startTime: number, endTime: number, animatedLeadInSeconds = 0, + mediaPaddingSeconds?: number, ): Promise { const config = this.deps.getConfig(); if (config.media?.imageType === 'avif') { @@ -376,7 +381,7 @@ export class PendingYoutubeMediaQueue { videoPath, startTime, endTime, - config.media?.audioPadding, + mediaPaddingSeconds ?? config.media?.audioPadding, { fps: config.media?.animatedFps, maxWidth: config.media?.animatedMaxWidth, diff --git a/src/anki-integration/pending-youtube-media.ts b/src/anki-integration/pending-youtube-media.ts index 52d65af8..4e0abfb0 100644 --- a/src/anki-integration/pending-youtube-media.ts +++ b/src/anki-integration/pending-youtube-media.ts @@ -10,6 +10,7 @@ export interface PendingYoutubeMediaUpdate { generateAudio: boolean; generateImage: boolean; volumeScale?: number; + mediaPaddingSeconds?: number; } function trimToNonEmptyString(value: unknown): string | null { diff --git a/src/config/definitions/defaults-integrations.ts b/src/config/definitions/defaults-integrations.ts index 788564be..11e4ccf1 100644 --- a/src/config/definitions/defaults-integrations.ts +++ b/src/config/definitions/defaults-integrations.ts @@ -54,6 +54,7 @@ export const INTEGRATIONS_DEFAULT_CONFIG: Pick< syncAnimatedImageToWordAudio: true, normalizeAudio: true, mirrorMpvVolume: true, + reviewTiming: false, audioPadding: 0, fallbackDuration: 3.0, maxMediaDuration: 30, diff --git a/src/config/definitions/options-integrations.ts b/src/config/definitions/options-integrations.ts index 9f8c3e32..678c7580 100644 --- a/src/config/definitions/options-integrations.ts +++ b/src/config/definitions/options-integrations.ts @@ -196,6 +196,13 @@ export function buildIntegrationConfigOptionRegistry( description: "Apply mpv's current software volume curve to generated sentence audio. Changes apply live.", }, + { + path: 'ankiConnect.media.reviewTiming', + kind: 'boolean', + defaultValue: defaultConfig.ankiConnect.media.reviewTiming, + description: + 'Review and preview subtitle media timing before SubMiner creates or enriches a mined card.', + }, { path: 'ankiConnect.media.generateImage', kind: 'boolean', diff --git a/src/config/definitions/template-sections.ts b/src/config/definitions/template-sections.ts index 43b22c60..08a2f750 100644 --- a/src/config/definitions/template-sections.ts +++ b/src/config/definitions/template-sections.ts @@ -135,7 +135,7 @@ const INTEGRATION_TEMPLATE_SECTIONS: ConfigTemplateSection[] = [ title: 'AnkiConnect Integration', description: ['Automatic Anki updates and media generation options.'], notes: [ - 'Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running.', + 'Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume/reviewTiming, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running.', 'Shared AI provider transport settings are read from top-level ai and typically require restart.', 'Most other AnkiConnect settings still require restart.', ], diff --git a/src/config/resolve/anki-connect.test.ts b/src/config/resolve/anki-connect.test.ts index 6b469208..5246adf3 100644 --- a/src/config/resolve/anki-connect.test.ts +++ b/src/config/resolve/anki-connect.test.ts @@ -21,6 +21,34 @@ function makeContext(ankiConnect: unknown): { return { context, warnings }; } +test('media timing review is disabled by default and accepts a boolean override', () => { + const defaultContext = makeContext({}); + applyAnkiConnectResolution(defaultContext.context); + assert.equal(defaultContext.context.resolved.ankiConnect.media.reviewTiming, false); + + const enabledContext = makeContext({ media: { reviewTiming: true } }); + applyAnkiConnectResolution(enabledContext.context); + assert.equal(enabledContext.context.resolved.ankiConnect.media.reviewTiming, true); + assert.deepEqual(enabledContext.warnings, []); +}); + +test('modern media duration accepts zero as the disabled cap sentinel', () => { + const disabledCap = makeContext({ media: { maxMediaDuration: 0 } }); + applyAnkiConnectResolution(disabledCap.context); + assert.equal(disabledCap.context.resolved.ankiConnect.media.maxMediaDuration, 0); + assert.deepEqual(disabledCap.warnings, []); + + const invalidCap = makeContext({ media: { maxMediaDuration: -1 } }); + applyAnkiConnectResolution(invalidCap.context); + assert.equal( + invalidCap.context.resolved.ankiConnect.media.maxMediaDuration, + DEFAULT_CONFIG.ankiConnect.media.maxMediaDuration, + ); + assert.ok( + invalidCap.warnings.some((warning) => warning.path === 'ankiConnect.media.maxMediaDuration'), + ); +}); + test('modern invalid knownWords.highlightEnabled warns modern key and does not fallback to legacy', () => { const { context, warnings } = makeContext({ nPlusOne: { highlightEnabled: true }, diff --git a/src/config/resolve/anki-connect/modern-media.ts b/src/config/resolve/anki-connect/modern-media.ts index d391f1c0..0babef6f 100644 --- a/src/config/resolve/anki-connect/modern-media.ts +++ b/src/config/resolve/anki-connect/modern-media.ts @@ -19,6 +19,7 @@ export function applyModernMediaResolution( 'syncAnimatedImageToWordAudio', 'normalizeAudio', 'mirrorMpvVolume', + 'reviewTiming', ] as const) { applyModernValue( context, @@ -128,18 +129,28 @@ export function applyModernMediaResolution( 'Expected non-negative number.', ); - for (const key of ['fallbackDuration', 'maxMediaDuration'] as const) { - applyModernValue( - context, - media, - key, - `ankiConnect.media.${key}`, - asPositiveNumber, - DEFAULT_CONFIG.ankiConnect.media[key], - (value) => { - context.resolved.ankiConnect.media[key] = value; - }, - 'Expected positive number.', - ); - } + applyModernValue( + context, + media, + 'fallbackDuration', + 'ankiConnect.media.fallbackDuration', + asPositiveNumber, + DEFAULT_CONFIG.ankiConnect.media.fallbackDuration, + (value) => { + context.resolved.ankiConnect.media.fallbackDuration = value; + }, + 'Expected positive number.', + ); + applyModernValue( + context, + media, + 'maxMediaDuration', + 'ankiConnect.media.maxMediaDuration', + asNonNegativeNumber, + DEFAULT_CONFIG.ankiConnect.media.maxMediaDuration, + (value) => { + context.resolved.ankiConnect.media.maxMediaDuration = value; + }, + 'Expected non-negative number.', + ); } diff --git a/src/config/settings/registry.ts b/src/config/settings/registry.ts index 9230da0c..56b205cf 100644 --- a/src/config/settings/registry.ts +++ b/src/config/settings/registry.ts @@ -244,6 +244,7 @@ const LABEL_OVERRIDES: Record = { 'mpv.aniskipEnabled': 'Enable AniSkip', 'mpv.aniskipButtonKey': 'AniSkip Button Key', 'ankiConnect.media.mirrorMpvVolume': 'Mirror mpv Volume', + 'ankiConnect.media.reviewTiming': 'Review Media Timing', 'discordPresence.updateIntervalMs': 'Update Interval (ms)', }; @@ -694,6 +695,7 @@ function restartBehaviorForPath(path: string): ConfigSettingsRestartBehavior { path === 'ankiConnect.ai.enabled' || path === 'ankiConnect.media.normalizeAudio' || path === 'ankiConnect.media.mirrorMpvVolume' || + path === 'ankiConnect.media.reviewTiming' || path === 'ankiConnect.behavior.autoUpdateNewCards' || path === 'ankiConnect.knownWords.highlightEnabled' || path === 'ankiConnect.knownWords.refreshMinutes' || diff --git a/src/core/services/ipc.ts b/src/core/services/ipc.ts index 032ed2b2..64a6899c 100644 --- a/src/core/services/ipc.ts +++ b/src/core/services/ipc.ts @@ -19,6 +19,11 @@ import type { YoutubePickerResolveRequest, YoutubePickerResolveResult, } from '../../types'; +import type { + MediaTimingReviewActionResult, + MediaTimingReviewPreviewRequest, + MediaTimingReviewResolveRequest, +} from '../../types/anki'; import { IPC_CHANNELS, type OverlayHostedModal } from '../../shared/ipc/contracts'; import { parseMpvCommand, @@ -98,6 +103,13 @@ export interface IpcServiceDeps { onYoutubePickerResolve: ( request: YoutubePickerResolveRequest, ) => Promise; + previewMediaTimingReview?: ( + request: MediaTimingReviewPreviewRequest, + ) => Promise; + stopMediaTimingReviewPreview?: (reviewId: string) => Promise; + resolveMediaTimingReview?: ( + request: MediaTimingReviewResolveRequest, + ) => MediaTimingReviewActionResult | Promise; getAnkiConnectStatus: () => boolean; getRuntimeOptions: () => unknown; setRuntimeOption: (id: RuntimeOptionId, value: RuntimeOptionValue) => unknown; @@ -221,6 +233,59 @@ function parseOverlayNotificationActionPayload( return { notificationId, actionId, ...(typeof noteId === 'number' ? { noteId } : {}) }; } +function parseMediaTimingReviewPreviewRequest( + payload: unknown, +): MediaTimingReviewPreviewRequest | null { + if (!payload || typeof payload !== 'object') return null; + const record = payload as Record; + if ( + typeof record.reviewId !== 'string' || + !record.reviewId || + typeof record.startTime !== 'number' || + !Number.isFinite(record.startTime) || + typeof record.endTime !== 'number' || + !Number.isFinite(record.endTime) + ) { + return null; + } + return { + reviewId: record.reviewId, + startTime: record.startTime, + endTime: record.endTime, + }; +} + +function parseMediaTimingReviewResolveRequest( + payload: unknown, +): MediaTimingReviewResolveRequest | null { + if (!payload || typeof payload !== 'object') return null; + const record = payload as Record; + if (typeof record.reviewId !== 'string' || !record.reviewId) return null; + const decision = record.decision; + if (!decision || typeof decision !== 'object') return null; + const decisionRecord = decision as Record; + if (decisionRecord.action === 'use-original' || decisionRecord.action === 'discard') { + return { reviewId: record.reviewId, decision: { action: decisionRecord.action } }; + } + if ( + decisionRecord.action === 'confirm' && + typeof decisionRecord.startTime === 'number' && + Number.isFinite(decisionRecord.startTime) && + typeof decisionRecord.endTime === 'number' && + Number.isFinite(decisionRecord.endTime) + ) { + return { + reviewId: record.reviewId, + decision: { + action: 'confirm', + startTime: decisionRecord.startTime, + endTime: decisionRecord.endTime, + }, + }; + } + return null; +} + export interface IpcDepsRuntimeOptions { getMainWindow: () => WindowLike | null; getVisibleOverlayVisibility: () => boolean; @@ -277,6 +342,9 @@ export interface IpcDepsRuntimeOptions { onYoutubePickerResolve: ( request: YoutubePickerResolveRequest, ) => Promise; + previewMediaTimingReview?: IpcServiceDeps['previewMediaTimingReview']; + stopMediaTimingReviewPreview?: IpcServiceDeps['stopMediaTimingReviewPreview']; + resolveMediaTimingReview?: IpcServiceDeps['resolveMediaTimingReview']; getAnkiConnectStatus: () => boolean; getRuntimeOptions: () => unknown; setRuntimeOption: (id: RuntimeOptionId, value: RuntimeOptionValue) => unknown; @@ -370,6 +438,9 @@ export function createIpcDepsRuntime(options: IpcDepsRuntimeOptions): IpcService options.activatePlaybackWindowForOverlayInteraction ?? (() => false), runSubsyncManual: options.runSubsyncManual, onYoutubePickerResolve: options.onYoutubePickerResolve, + previewMediaTimingReview: options.previewMediaTimingReview, + stopMediaTimingReviewPreview: options.stopMediaTimingReviewPreview, + resolveMediaTimingReview: options.resolveMediaTimingReview, getAnkiConnectStatus: options.getAnkiConnectStatus, getRuntimeOptions: options.getRuntimeOptions, setRuntimeOption: options.setRuntimeOption, @@ -491,6 +562,36 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar }, ); + ipc.handle( + IPC_CHANNELS.request.mediaTimingReviewPreview, + async (_event: unknown, payload: unknown) => { + const request = parseMediaTimingReviewPreviewRequest(payload); + if (!request || !deps.previewMediaTimingReview) { + return { ok: false, message: 'Timing preview is unavailable.' }; + } + return await deps.previewMediaTimingReview(request); + }, + ); + ipc.handle( + IPC_CHANNELS.request.mediaTimingReviewStopPreview, + async (_event: unknown, reviewId: unknown) => { + if (typeof reviewId !== 'string' || !reviewId || !deps.stopMediaTimingReviewPreview) { + return { ok: false, message: 'Timing preview is unavailable.' }; + } + return await deps.stopMediaTimingReviewPreview(reviewId); + }, + ); + ipc.handle( + IPC_CHANNELS.request.mediaTimingReviewResolve, + async (_event: unknown, payload: unknown) => { + const request = parseMediaTimingReviewResolveRequest(payload); + if (!request || !deps.resolveMediaTimingReview) { + return { ok: false, message: 'Timing review is unavailable.' }; + } + return await deps.resolveMediaTimingReview(request); + }, + ); + ipc.on(IPC_CHANNELS.command.openYomitanSettings, () => { deps.openYomitanSettings(); }); diff --git a/src/core/services/media-timing-preview.test.ts b/src/core/services/media-timing-preview.test.ts new file mode 100644 index 00000000..02569d12 --- /dev/null +++ b/src/core/services/media-timing-preview.test.ts @@ -0,0 +1,144 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import net from 'node:net'; +import { describe, test } from 'node:test'; +import { buildMediaTimingPreviewArgs, MediaTimingPreviewSession } from './media-timing-preview'; + +describe('buildMediaTimingPreviewArgs', () => { + test('creates a hidden audio-only reusable mpv session', () => { + const args = buildMediaTimingPreviewArgs('/tmp/review.sock', { + mediaPath: '/video/show.mkv', + audioTrackId: 3, + volume: 55, + }); + + assert.ok(args.includes('--no-video')); + assert.ok(args.includes('--force-window=no')); + assert.ok(args.includes('--idle=yes')); + assert.ok(args.includes('--pause=yes')); + assert.ok(args.includes('--input-ipc-server=/tmp/review.sock')); + assert.ok(args.includes('--aid=3')); + assert.ok(args.includes('--volume=55')); + assert.equal(args.at(-2), '--'); + assert.equal(args.at(-1), '/video/show.mkv'); + }); +}); + +test('preview session handles socket errors after connecting', async () => { + const socket = new net.Socket(); + const child = new EventEmitter() as EventEmitter & { kill: () => boolean }; + child.kill = () => true; + const session = new MediaTimingPreviewSession({ + platform: 'linux', + spawnProcess: () => child as never, + connectSocket: () => { + queueMicrotask(() => socket.emit('connect')); + return socket; + }, + removeSocketFile: () => undefined, + createSocketPath: () => '/tmp/review.sock', + }); + + await session.start({ mediaPath: '/video/show.mkv' }); + assert.doesNotThrow(() => socket.emit('error', new Error('pipe closed'))); + await assert.rejects(session.play(1, 2), /not ready/); + session.dispose(); +}); + +test('preview session rejects a connection that finishes after disposal', async () => { + const socket = new net.Socket(); + const child = new EventEmitter() as EventEmitter & { kill: () => boolean }; + child.kill = () => true; + const session = new MediaTimingPreviewSession({ + platform: 'linux', + spawnProcess: () => child as never, + connectSocket: () => socket, + removeSocketFile: () => undefined, + createSocketPath: () => '/tmp/review.sock', + }); + + const pendingStart = session.start({ mediaPath: '-playlist' }); + session.dispose(); + socket.emit('connect'); + + await assert.rejects(pendingStart, /closed/); + assert.equal(socket.destroyed, true); +}); + +test('preview session shares one startup across concurrent start calls', async () => { + const socket = new net.Socket(); + const child = new EventEmitter() as EventEmitter & { kill: () => boolean }; + child.kill = () => true; + let spawnCount = 0; + const session = new MediaTimingPreviewSession({ + platform: 'linux', + spawnProcess: () => { + spawnCount += 1; + return child as never; + }, + connectSocket: () => socket, + removeSocketFile: () => undefined, + createSocketPath: () => '/tmp/review.sock', + }); + + const firstStart = session.start({ mediaPath: '/video/show.mkv' }); + const secondStart = session.start({ mediaPath: '/video/show.mkv' }); + socket.emit('connect'); + + await Promise.all([firstStart, secondStart]); + assert.equal(spawnCount, 1); + session.dispose(); +}); + +test('preview session can start again after a startup failure', async () => { + const socket = new net.Socket(); + const child = new EventEmitter() as EventEmitter & { kill: () => boolean }; + child.kill = () => true; + let spawnCount = 0; + const session = new MediaTimingPreviewSession({ + platform: 'linux', + spawnProcess: () => { + spawnCount += 1; + if (spawnCount === 1) throw new Error('spawn failed'); + return child as never; + }, + connectSocket: () => { + queueMicrotask(() => socket.emit('connect')); + return socket; + }, + removeSocketFile: () => undefined, + createSocketPath: () => '/tmp/review.sock', + }); + + await assert.rejects(session.start({ mediaPath: '/video/show.mkv' }), /spawn failed/); + await session.start({ mediaPath: '/video/show.mkv' }); + assert.equal(spawnCount, 2); + session.dispose(); +}); + +test('preview session bounds a connection attempt that never settles', async () => { + const child = new EventEmitter() as EventEmitter & { kill: () => boolean }; + child.kill = () => true; + let nowMs = 0; + let connectAttempts = 0; + const session = new MediaTimingPreviewSession({ + platform: 'linux', + spawnProcess: () => child as never, + connectSocket: () => { + connectAttempts += 1; + return new net.Socket(); + }, + now: () => { + const current = nowMs; + nowMs += 1_000; + return current; + }, + schedule: (callback) => setTimeout(callback, 0), + cancelSchedule: (timeout) => clearTimeout(timeout), + removeSocketFile: () => undefined, + createSocketPath: () => '/tmp/review.sock', + }); + + await assert.rejects(session.start({ mediaPath: '/video/show.mkv' }), /Timed out starting/); + assert.equal(connectAttempts, 1); +}); diff --git a/src/core/services/media-timing-preview.ts b/src/core/services/media-timing-preview.ts new file mode 100644 index 00000000..8563a956 --- /dev/null +++ b/src/core/services/media-timing-preview.ts @@ -0,0 +1,309 @@ +import { spawn, type ChildProcess } from 'child_process'; +import fs from 'fs'; +import net, { type Socket } from 'net'; +import os from 'os'; +import path from 'path'; +import { randomUUID } from 'crypto'; + +const CONNECT_TIMEOUT_MS = 5_000; +const CONNECT_ATTEMPT_TIMEOUT_MS = 500; +const CONNECT_RETRY_MS = 40; + +export interface MediaTimingPreviewStartOptions { + mediaPath: string; + executablePath?: string; + audioTrackId?: number; + volume?: number; +} + +type PreviewProcess = Pick; + +interface MediaTimingPreviewDeps { + platform: NodeJS.Platform; + spawnProcess: (command: string, args: string[]) => PreviewProcess; + connectSocket: (socketPath: string) => Socket; + now: () => number; + schedule: (callback: () => void, delayMs: number) => ReturnType; + cancelSchedule: (timeout: ReturnType) => void; + removeSocketFile: (socketPath: string) => void; + createSocketPath: () => string; +} + +export function buildMediaTimingPreviewArgs( + socketPath: string, + options: MediaTimingPreviewStartOptions, +): string[] { + const args = [ + '--no-config', + '--no-video', + '--audio-display=no', + '--force-window=no', + '--idle=yes', + '--keep-open=yes', + '--pause=yes', + '--terminal=no', + '--msg-level=all=warn', + `--input-ipc-server=${socketPath}`, + ]; + if (typeof options.audioTrackId === 'number' && Number.isInteger(options.audioTrackId)) { + args.push(`--aid=${options.audioTrackId}`); + } + if (typeof options.volume === 'number' && Number.isFinite(options.volume)) { + args.push(`--volume=${Math.max(0, options.volume)}`); + } + args.push('--', options.mediaPath); + return args; +} + +function createDefaultSocketPath(): string { + const suffix = `${process.pid}-${randomUUID()}`; + return process.platform === 'win32' + ? `\\\\.\\pipe\\subminer-timing-preview-${suffix}` + : path.join(os.tmpdir(), `subminer-timing-preview-${suffix}.sock`); +} + +function removePosixSocketFile(socketPath: string): void { + if (process.platform === 'win32') return; + try { + fs.unlinkSync(socketPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } +} + +export class MediaTimingPreviewSession { + private readonly deps: MediaTimingPreviewDeps; + private socketPath: string | null = null; + private socket: Socket | null = null; + private process: PreviewProcess | null = null; + private startupError: Error | null = null; + private startPromise: Promise | null = null; + private retryWait: { + timeout: ReturnType; + resolve: () => void; + } | null = null; + private disposed = false; + + constructor(deps: Partial = {}) { + this.deps = { + platform: process.platform, + spawnProcess: (command, args) => spawn(command, args, { stdio: 'ignore' }), + connectSocket: (socketPath) => net.createConnection(socketPath), + now: Date.now, + schedule: (callback, delayMs) => setTimeout(callback, delayMs), + cancelSchedule: (timeout) => clearTimeout(timeout), + removeSocketFile: removePosixSocketFile, + createSocketPath: createDefaultSocketPath, + ...deps, + }; + } + + async start(options: MediaTimingPreviewStartOptions): Promise { + if (this.disposed) throw new Error('Preview session is closed'); + if (this.socket) return; + if (this.startPromise) return await this.startPromise; + + const startPromise = this.startOnce(options); + this.startPromise = startPromise; + try { + await startPromise; + } catch (error) { + this.releaseResources(); + throw error; + } finally { + if (this.startPromise === startPromise) this.startPromise = null; + } + } + + private async startOnce(options: MediaTimingPreviewStartOptions): Promise { + const mediaPath = options.mediaPath.trim(); + if (!mediaPath) throw new Error('No media source is available for preview'); + + const socketPath = this.deps.createSocketPath(); + this.socketPath = socketPath; + if (this.deps.platform !== 'win32') { + this.deps.removeSocketFile(socketPath); + } + + const command = options.executablePath?.trim() || 'mpv'; + this.startupError = null; + const child = this.deps.spawnProcess( + command, + buildMediaTimingPreviewArgs(socketPath, { ...options, mediaPath }), + ); + this.process = child; + child.once('error', (error) => { + if (this.process !== child) return; + this.startupError = error; + }); + child.once('exit', () => { + if (this.process !== child) return; + if (!this.socket && !this.disposed && !this.startupError) { + this.startupError = new Error('The hidden mpv preview player exited during startup'); + } + this.socket?.destroy(); + this.socket = null; + this.process = null; + }); + + await this.connectWithRetry(socketPath); + } + + async play(startTime: number, endTime: number): Promise { + if (!this.socket || this.socket.destroyed) { + throw new Error('Preview player is not ready'); + } + if (!Number.isFinite(startTime) || !Number.isFinite(endTime) || endTime <= startTime) { + throw new Error('Preview timing is invalid'); + } + + this.send(['set_property', 'pause', true]); + this.send(['set_property', 'ab-loop-a', startTime]); + this.send(['set_property', 'ab-loop-b', endTime]); + this.send(['seek', startTime, 'absolute+exact']); + this.send(['set_property', 'pause', false]); + } + + async stop(): Promise { + if (!this.socket || this.socket.destroyed) return; + this.send(['set_property', 'pause', true]); + } + + dispose(): void { + if (this.disposed) return; + this.disposed = true; + this.releaseResources(); + } + + private releaseResources(): void { + this.cancelRetryWait(); + try { + this.send(['quit']); + } catch { + // The process may already have exited. + } + this.socket?.end(); + this.socket?.destroy(); + this.socket = null; + const child = this.process; + this.process = null; + child?.kill(); + if (this.socketPath && this.deps.platform !== 'win32') { + try { + this.deps.removeSocketFile(this.socketPath); + } catch { + // mpv may still be releasing the socket. The OS temp directory owns cleanup. + } + } + this.socketPath = null; + } + + private send(command: Array): void { + if (!this.socket || this.socket.destroyed) { + throw new Error('Preview player is not connected'); + } + this.socket.write(`${JSON.stringify({ command })}\n`); + } + + private async connectWithRetry(socketPath: string): Promise { + const deadline = this.deps.now() + CONNECT_TIMEOUT_MS; + while (!this.disposed && this.deps.now() < deadline) { + if (this.startupError) { + throw this.startupError; + } + try { + const remainingMs = deadline - this.deps.now(); + if (remainingMs <= 0) break; + const socket = await this.connectOnce( + socketPath, + Math.min(CONNECT_ATTEMPT_TIMEOUT_MS, remainingMs), + ); + if (this.disposed) { + socket.destroy(); + throw new Error('Preview session is closed'); + } + this.socket = socket; + return; + } catch { + if (this.disposed) { + throw new Error('Preview session is closed'); + } + const remainingMs = deadline - this.deps.now(); + if (remainingMs <= 0) break; + await this.waitForRetry(Math.min(CONNECT_RETRY_MS, remainingMs)); + } + } + if (this.startupError) { + throw this.startupError; + } + if (this.disposed) { + throw new Error('Preview session is closed'); + } + throw new Error('Timed out starting the hidden mpv preview player'); + } + + private waitForRetry(delayMs: number): Promise { + return new Promise((resolve) => { + const timeout = this.deps.schedule(() => { + if (this.retryWait?.timeout === timeout) this.retryWait = null; + resolve(); + }, delayMs); + this.retryWait = { timeout, resolve }; + }); + } + + private cancelRetryWait(): void { + const pending = this.retryWait; + this.retryWait = null; + if (!pending) return; + this.deps.cancelSchedule(pending.timeout); + pending.resolve(); + } + + private connectOnce(socketPath: string, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + let timeout: ReturnType | null = null; + let settled = false; + const clearAttemptTimeout = (): void => { + if (timeout !== null) this.deps.cancelSchedule(timeout); + timeout = null; + }; + const socket = this.deps.connectSocket(socketPath); + const onConnect = (): void => { + if (settled) return; + settled = true; + clearAttemptTimeout(); + socket.off('error', onError); + socket.on('error', () => { + socket.destroy(); + if (this.socket === socket) this.socket = null; + }); + socket.once('close', () => { + if (this.socket === socket) this.socket = null; + }); + resolve(socket); + }; + const onError = (error: Error): void => { + if (settled) return; + settled = true; + clearAttemptTimeout(); + socket.off('connect', onConnect); + socket.destroy(); + reject(error); + }; + socket.once('connect', onConnect); + socket.once('error', onError); + timeout = this.deps.schedule(() => { + if (settled) return; + settled = true; + timeout = null; + socket.off('connect', onConnect); + socket.off('error', onError); + socket.destroy(); + reject(new Error('Timed out connecting to the hidden mpv preview player')); + }, timeoutMs); + }); + } +} diff --git a/src/main.ts b/src/main.ts index 8ca54a75..feaf49e0 100644 --- a/src/main.ts +++ b/src/main.ts @@ -462,6 +462,9 @@ import { createMainBootServices, type MainBootServicesResult } from './main/boot import { handleCliCommandRuntimeServiceWithContext } from './main/cli-runtime'; import { createOverlayModalRuntimeService } from './main/overlay-runtime'; import { createOverlayModalInputState } from './main/runtime/overlay-modal-input-state'; +import { MediaTimingPreviewSession } from './core/services/media-timing-preview'; +import { createMediaTimingReviewRuntime } from './main/runtime/media-timing-review'; +import { openMediaTimingReviewModal } from './main/runtime/media-timing-review-open'; import { openYoutubeTrackPicker } from './main/runtime/youtube-picker-open'; import { openRuntimeOptionsModal as openRuntimeOptionsModalRuntime } from './main/runtime/runtime-options-open'; import { openJimakuModal as openJimakuModalRuntime } from './main/runtime/jimaku-open'; @@ -2797,6 +2800,18 @@ function createOverlayHostedModalOpenDeps(): { }; } +const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({ + getMpvClient: () => appState.mpvClient, + getCurrentMediaPath: () => + appState.currentMediaPath?.trim() || appState.mpvClient?.currentVideoPath?.trim() || null, + getMpvExecutablePath: () => + configService.getConfig().mpv.executablePath || process.env.SUBMINER_MPV_PATH?.trim() || '', + createPreviewSession: () => new MediaTimingPreviewSession(), + openModal: (payload) => openMediaTimingReviewModal(createOverlayHostedModalOpenDeps(), payload), + showStatus: (message) => + overlayNotificationsRuntime.showConfiguredStatusNotification(message, { variant: 'warning' }), +}); + function openOverlayHostedModalWithOsd( openModal: (deps: ReturnType) => Promise, unavailableMessage: string, @@ -5014,6 +5029,7 @@ function initializeOverlayRuntime(): void { refreshCurrentSubtitleAfterKnownWordUpdate, ); appState.ankiIntegration?.setSubtitleMiningContextConsumer(consumePendingSubtitleMiningContext); + appState.ankiIntegration?.setMediaTimingReviewCallback(mediaTimingReviewRuntime.requestReview); syncOverlayMpvSubtitleSuppression(); } @@ -5423,6 +5439,9 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({ showMpvOsd: (text: string) => showConfiguredPlaybackFeedback(text), }, mainDeps: { + previewMediaTimingReview: (request) => mediaTimingReviewRuntime.previewRange(request), + stopMediaTimingReviewPreview: (reviewId) => mediaTimingReviewRuntime.stopPreview(reviewId), + resolveMediaTimingReview: (request) => mediaTimingReviewRuntime.resolveReview(request), getMainWindow: () => overlayManager.getMainWindow(), getVisibleOverlayVisibility: () => overlayManager.getVisibleOverlayVisible(), focusMainWindow: () => { @@ -5456,6 +5475,9 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({ } }, onOverlayModalClosed: (modal, senderWindow) => { + if (modal === 'media-timing-review') { + void mediaTimingReviewRuntime.dispose(); + } if (modal === 'subtitle-sidebar' && senderWindow === overlayManager.getMainWindow()) { subtitleSidebarRequestedOpen = false; } @@ -5811,6 +5833,9 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({ appState.ankiIntegration?.setSubtitleMiningContextConsumer( consumePendingSubtitleMiningContext, ); + appState.ankiIntegration?.setMediaTimingReviewCallback( + mediaTimingReviewRuntime.requestReview, + ); }, getKnownWordCacheStatePath: () => path.join(USER_DATA_PATH, 'known-words-cache.json'), getCachedMediaPath: (currentVideoPath, kind) => @@ -6133,6 +6158,7 @@ const { createMainWindow: createMainWindowHandler, createModalWindow: createModa if (overlayManager.getModalWindow() !== window) { return; } + void mediaTimingReviewRuntime.dispose(); overlayManager.setModalWindow(null); } }, diff --git a/src/main/dependencies.ts b/src/main/dependencies.ts index cbbe4beb..183a5025 100644 --- a/src/main/dependencies.ts +++ b/src/main/dependencies.ts @@ -62,6 +62,9 @@ export interface MainIpcRuntimeServiceDepsParams { onOverlayInteractiveHint?: IpcDepsRuntimeOptions['onOverlayInteractiveHint']; handleOverlayNotificationAction?: IpcDepsRuntimeOptions['handleOverlayNotificationAction']; onYoutubePickerResolve: IpcDepsRuntimeOptions['onYoutubePickerResolve']; + previewMediaTimingReview?: IpcDepsRuntimeOptions['previewMediaTimingReview']; + stopMediaTimingReviewPreview?: IpcDepsRuntimeOptions['stopMediaTimingReviewPreview']; + resolveMediaTimingReview?: IpcDepsRuntimeOptions['resolveMediaTimingReview']; openYomitanSettings: IpcDepsRuntimeOptions['openYomitanSettings']; quitApp: IpcDepsRuntimeOptions['quitApp']; toggleVisibleOverlay: IpcDepsRuntimeOptions['toggleVisibleOverlay']; @@ -256,6 +259,9 @@ export function createMainIpcRuntimeServiceDeps( onOverlayInteractiveHint: params.onOverlayInteractiveHint, handleOverlayNotificationAction: params.handleOverlayNotificationAction, onYoutubePickerResolve: params.onYoutubePickerResolve, + previewMediaTimingReview: params.previewMediaTimingReview, + stopMediaTimingReviewPreview: params.stopMediaTimingReviewPreview, + resolveMediaTimingReview: params.resolveMediaTimingReview, openYomitanSettings: params.openYomitanSettings, quitApp: params.quitApp, toggleVisibleOverlay: params.toggleVisibleOverlay, diff --git a/src/main/runtime/media-timing-review-open.ts b/src/main/runtime/media-timing-review-open.ts new file mode 100644 index 00000000..9307a527 --- /dev/null +++ b/src/main/runtime/media-timing-review-open.ts @@ -0,0 +1,40 @@ +import type { OverlayHostedModal } from '../../shared/ipc/contracts'; +import type { MediaTimingReviewOpenPayload } from '../../types/anki'; +import { openOverlayHostedModal, retryOverlayModalOpen } from './overlay-hosted-modal-open'; + +const MODAL: OverlayHostedModal = 'media-timing-review'; + +export async function openMediaTimingReviewModal( + deps: { + ensureOverlayStartupPrereqs: () => void; + ensureOverlayWindowsReadyForVisibilityActions: () => void; + sendToActiveOverlayWindow: ( + channel: string, + payload?: unknown, + runtimeOptions?: { + restoreOnModalClose?: OverlayHostedModal; + preferModalWindow?: boolean; + }, + ) => boolean; + waitForModalOpen: (modal: OverlayHostedModal, timeoutMs: number) => Promise; + logWarn: (message: string) => void; + }, + payload: MediaTimingReviewOpenPayload, +): Promise { + return await retryOverlayModalOpen( + { waitForModalOpen: deps.waitForModalOpen, logWarn: deps.logWarn }, + { + modal: MODAL, + timeoutMs: 1_500, + retryWarning: + 'Media timing review did not acknowledge modal open; retrying the dedicated modal window.', + sendOpen: () => + openOverlayHostedModal(deps, { + channel: 'media-timing-review:open', + modal: MODAL, + payload, + preferModalWindow: true, + }), + }, + ); +} diff --git a/src/main/runtime/media-timing-review.test.ts b/src/main/runtime/media-timing-review.test.ts new file mode 100644 index 00000000..5568b701 --- /dev/null +++ b/src/main/runtime/media-timing-review.test.ts @@ -0,0 +1,246 @@ +import assert from 'node:assert/strict'; +import { describe, test } from 'node:test'; +import { + buildMediaTimingReviewPayload, + createMediaTimingReviewRuntime, +} from './media-timing-review'; + +describe('buildMediaTimingReviewPayload', () => { + test('starts from the padded range and leaves two seconds to drag on each side', () => { + const payload = buildMediaTimingReviewPayload( + { + kind: 'sentence', + text: '字幕', + startTime: 10, + endTime: 12, + audioPadding: 0.5, + maxMediaDuration: 30, + }, + { reviewId: 'review-1', mediaDuration: 100 }, + ); + + assert.equal(payload.selectionStartTime, 9.5); + assert.equal(payload.selectionEndTime, 12.5); + assert.equal(payload.timelineStartTime, 7.5); + assert.equal(payload.timelineEndTime, 14.5); + }); + + test('clamps the padded selection and timeline to media bounds', () => { + const payload = buildMediaTimingReviewPayload( + { + kind: 'word', + text: '字幕', + startTime: 0.2, + endTime: 9.8, + audioPadding: 1, + maxMediaDuration: 30, + }, + { reviewId: 'review-2', mediaDuration: 10 }, + ); + + assert.equal(payload.selectionStartTime, 0); + assert.equal(payload.selectionEndTime, 10); + assert.equal(payload.timelineStartTime, 0); + assert.equal(payload.timelineEndTime, 10); + }); + + test('keeps an uncapped selection when max media duration is disabled', () => { + const payload = buildMediaTimingReviewPayload( + { + kind: 'sentence', + text: '字幕', + startTime: 10, + endTime: 55, + audioPadding: 1, + maxMediaDuration: 0, + }, + { reviewId: 'review-unlimited', mediaDuration: 100 }, + ); + + assert.equal(payload.selectionStartTime, 9); + assert.equal(payload.selectionEndTime, 56); + assert.equal(payload.maxMediaDuration, 0); + }); +}); + +test('media timing review pauses playback, resolves exact timing, and restores playing state', async () => { + const commands: Array> = []; + const previewCalls: Array<[number, number]> = []; + let runtime: ReturnType; + runtime = createMediaTimingReviewRuntime({ + getMpvClient: () => ({ + connected: true, + currentVideoPath: '/video/show.mkv', + requestProperty: async (name) => + ({ pause: false, duration: 100, aid: 2, volume: 60 })[ + name as 'pause' | 'duration' | 'aid' | 'volume' + ], + send: ({ command }) => commands.push(command), + }), + getCurrentMediaPath: () => '/video/show.mkv', + getMpvExecutablePath: () => 'mpv', + createPreviewSession: () => ({ + start: async () => undefined, + play: async (startTime, endTime) => { + previewCalls.push([startTime, endTime]); + }, + stop: async () => undefined, + dispose: () => undefined, + }), + openModal: async (payload) => { + queueMicrotask(() => { + void runtime + .previewRange({ + reviewId: payload.reviewId, + startTime: 9.5, + endTime: 12.5, + }) + .then(() => { + runtime.resolveReview({ + reviewId: payload.reviewId, + decision: { action: 'confirm', startTime: 9.5, endTime: 12.5 }, + }); + }); + }); + return true; + }, + showStatus: () => undefined, + }); + + const decision = await runtime.requestReview({ + kind: 'word', + text: '字幕', + startTime: 10, + endTime: 12, + noteId: 42, + audioPadding: 0.5, + maxMediaDuration: 30, + }); + + assert.deepEqual(decision, { action: 'confirm', startTime: 9.5, endTime: 12.5 }); + assert.deepEqual(commands, [ + ['set_property', 'pause', 'yes'], + ['set_property', 'pause', 'no'], + ]); + assert.deepEqual(previewCalls, [[9.5, 12.5]]); +}); + +test('media timing review does not resume playback when the prior state is unavailable', async () => { + const commands: Array> = []; + let runtime: ReturnType; + runtime = createMediaTimingReviewRuntime({ + getMpvClient: () => ({ + connected: true, + currentVideoPath: '/video/show.mkv', + requestProperty: async () => null, + send: ({ command }) => commands.push(command), + }), + getCurrentMediaPath: () => '/video/show.mkv', + getMpvExecutablePath: () => '', + createPreviewSession: () => ({ + start: async () => { + throw new Error('preview unavailable'); + }, + play: async () => undefined, + stop: async () => undefined, + dispose: () => undefined, + }), + openModal: async (payload) => { + queueMicrotask(() => { + runtime.resolveReview({ + reviewId: payload.reviewId, + decision: { action: 'use-original' }, + }); + }); + return true; + }, + showStatus: () => undefined, + }); + + assert.deepEqual( + await runtime.requestReview({ + kind: 'sentence', + text: '字幕', + startTime: 10, + endTime: 12, + audioPadding: 0, + maxMediaDuration: 30, + }), + { action: 'use-original' }, + ); + assert.deepEqual(commands, [['set_property', 'pause', 'yes']]); +}); + +test('media timing review restores playback when setup fails after pausing', async () => { + const commands: Array> = []; + const runtime = createMediaTimingReviewRuntime({ + getMpvClient: () => ({ + connected: true, + currentVideoPath: '/video/show.mkv', + requestProperty: async (name) => (name === 'pause' ? false : null), + send: ({ command }) => commands.push(command), + }), + getCurrentMediaPath: () => '/video/show.mkv', + getMpvExecutablePath: () => 'mpv', + createPreviewSession: () => { + throw new Error('preview setup failed'); + }, + openModal: async () => true, + showStatus: () => undefined, + }); + + assert.deepEqual( + await runtime.requestReview({ + kind: 'word', + text: '字幕', + startTime: 10, + endTime: 12, + audioPadding: 0, + maxMediaDuration: 30, + }), + { action: 'use-original' }, + ); + assert.deepEqual(commands, [ + ['set_property', 'pause', 'yes'], + ['set_property', 'pause', 'no'], + ]); +}); + +test('disposing an open review settles it with original timing and restores playback', async () => { + const commands: Array> = []; + const runtime = createMediaTimingReviewRuntime({ + getMpvClient: () => ({ + connected: true, + currentVideoPath: '/video/show.mkv', + requestProperty: async (name) => (name === 'pause' ? false : null), + send: ({ command }) => commands.push(command), + }), + getCurrentMediaPath: () => '/video/show.mkv', + getMpvExecutablePath: () => 'mpv', + createPreviewSession: () => ({ + start: async () => undefined, + play: async () => undefined, + stop: async () => undefined, + dispose: () => undefined, + }), + openModal: async () => true, + showStatus: () => undefined, + }); + + const pending = runtime.requestReview({ + kind: 'word', + text: '字幕', + startTime: 10, + endTime: 12, + audioPadding: 0, + maxMediaDuration: 30, + }); + await new Promise((resolve) => setImmediate(resolve)); + await runtime.dispose(); + + assert.deepEqual(await pending, { action: 'use-original' }); + assert.deepEqual(commands, [ + ['set_property', 'pause', 'yes'], + ['set_property', 'pause', 'no'], + ]); +}); diff --git a/src/main/runtime/media-timing-review.ts b/src/main/runtime/media-timing-review.ts new file mode 100644 index 00000000..989ba1a9 --- /dev/null +++ b/src/main/runtime/media-timing-review.ts @@ -0,0 +1,289 @@ +import { randomUUID } from 'crypto'; +import type { + MediaTimingReviewActionResult, + MediaTimingReviewDecision, + MediaTimingReviewOpenPayload, + MediaTimingReviewPreviewRequest, + MediaTimingReviewRequest, + MediaTimingReviewResolveRequest, +} from '../../types/anki'; + +const INITIAL_TIMELINE_MARGIN_SECONDS = 2; + +interface ReviewMpvClient { + connected: boolean; + currentVideoPath: string; + requestProperty?: (name: string) => Promise; + send: (payload: { command: Array }) => void; +} + +interface PreviewSession { + start(options: { + mediaPath: string; + executablePath?: string; + audioTrackId?: number; + volume?: number; + }): Promise; + play(startTime: number, endTime: number): Promise; + stop(): Promise; + dispose(): void; +} + +interface ActiveReview { + payload: MediaTimingReviewOpenPayload; + mpvClient: ReviewMpvClient; + restorePlayback: boolean; + preview: Promise; + resolve: (decision: MediaTimingReviewDecision) => void; +} + +export interface MediaTimingReviewRuntimeDeps { + getMpvClient: () => ReviewMpvClient | null; + getCurrentMediaPath: () => string | null; + getMpvExecutablePath: () => string; + createPreviewSession: () => PreviewSession; + openModal: (payload: MediaTimingReviewOpenPayload) => Promise; + showStatus: (message: string) => void; +} + +function finiteNumber(value: unknown): number | null { + return typeof value === 'number' && Number.isFinite(value) ? value : null; +} + +function booleanProperty(value: unknown): boolean | null { + if (typeof value === 'boolean') return value; + if (value === 'yes' || value === 1) return true; + if (value === 'no' || value === 0) return false; + return null; +} + +export function buildMediaTimingReviewPayload( + request: MediaTimingReviewRequest, + options: { reviewId: string; mediaDuration?: number }, +): MediaTimingReviewOpenPayload { + const duration = finiteNumber(options.mediaDuration); + const maxTime = duration !== null && duration > 0 ? duration : Number.POSITIVE_INFINITY; + const paddedStart = Math.max(0, request.startTime - request.audioPadding); + let paddedEnd = Math.min(maxTime, request.endTime + request.audioPadding); + const maxMediaDuration = Math.max(0, request.maxMediaDuration); + if (maxMediaDuration > 0 && paddedEnd - paddedStart > maxMediaDuration) { + paddedEnd = paddedStart + maxMediaDuration; + } + if (paddedEnd <= paddedStart) { + paddedEnd = Math.min(maxTime, paddedStart + 0.1); + } + + const timelineStartTime = Math.max(0, paddedStart - INITIAL_TIMELINE_MARGIN_SECONDS); + const timelineEndTime = Math.max( + paddedEnd, + Math.min(maxTime, paddedEnd + INITIAL_TIMELINE_MARGIN_SECONDS), + ); + + return { + reviewId: options.reviewId, + kind: request.kind, + text: request.text, + ...(request.noteId !== undefined ? { noteId: request.noteId } : {}), + originalStartTime: request.startTime, + originalEndTime: request.endTime, + selectionStartTime: paddedStart, + selectionEndTime: paddedEnd, + timelineStartTime, + timelineEndTime, + ...(duration !== null && duration > 0 ? { mediaDuration: duration } : {}), + maxMediaDuration, + }; +} + +export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDeps) { + let active: ActiveReview | null = null; + let reviewInProgress = false; + let pendingPauseRestore: ReviewMpvClient | null = null; + + function restorePendingPlayback(): void { + const mpvClient = pendingPauseRestore; + pendingPauseRestore = null; + if (mpvClient?.connected) { + mpvClient.send({ command: ['set_property', 'pause', 'no'] }); + } + } + + async function runReview(request: MediaTimingReviewRequest): Promise { + const mpvClient = deps.getMpvClient(); + const mediaPath = + deps.getCurrentMediaPath()?.trim() || mpvClient?.currentVideoPath?.trim() || ''; + if (!mpvClient?.connected || !mediaPath) { + deps.showStatus('Timing review unavailable. Using the original subtitle timing.'); + return { action: 'use-original' }; + } + + const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw] = await Promise.all([ + mpvClient.requestProperty?.('pause').catch(() => null) ?? null, + mpvClient.requestProperty?.('duration').catch(() => null) ?? null, + mpvClient.requestProperty?.('aid').catch(() => null) ?? null, + mpvClient.requestProperty?.('volume').catch(() => null) ?? null, + ]); + const pauseState = booleanProperty(pauseRaw); + mpvClient.send({ command: ['set_property', 'pause', 'yes'] }); + pendingPauseRestore = pauseState === false ? mpvClient : null; + + const payload = buildMediaTimingReviewPayload(request, { + reviewId: randomUUID(), + mediaDuration: finiteNumber(durationRaw) ?? undefined, + }); + const previewSession = deps.createPreviewSession(); + const preview = previewSession + .start({ + mediaPath, + executablePath: deps.getMpvExecutablePath(), + audioTrackId: finiteNumber(audioTrackRaw) ?? undefined, + volume: finiteNumber(volumeRaw) ?? undefined, + }) + .then(() => previewSession) + .catch((error) => { + previewSession.dispose(); + throw error; + }); + void preview.catch(() => {}); + + let resolveDecision!: (decision: MediaTimingReviewDecision) => void; + const decisionPromise = new Promise((resolve) => { + resolveDecision = resolve; + }); + active = { + payload, + mpvClient, + restorePlayback: pendingPauseRestore === mpvClient, + preview, + resolve: resolveDecision, + }; + pendingPauseRestore = null; + + const opened = await deps.openModal(payload).catch(() => false); + if (!opened) { + await cleanupActiveReview(); + deps.showStatus('Timing review could not open. Using the original subtitle timing.'); + return { action: 'use-original' }; + } + + const decision = await decisionPromise; + await cleanupActiveReview(); + return decision; + } + + async function requestReview( + request: MediaTimingReviewRequest, + ): Promise { + if (active || reviewInProgress) { + deps.showStatus('Finish the current timing review before mining another card.'); + return { action: 'use-original' }; + } + reviewInProgress = true; + try { + return await runReview(request); + } catch { + await cleanupActiveReview(); + restorePendingPlayback(); + deps.showStatus('Timing review failed. Using the original subtitle timing.'); + return { action: 'use-original' }; + } finally { + reviewInProgress = false; + } + } + + async function previewRange( + request: MediaTimingReviewPreviewRequest, + ): Promise { + const current = active; + if (!current || request.reviewId !== current.payload.reviewId) { + return { ok: false, message: 'This timing review is no longer active.' }; + } + if ( + !Number.isFinite(request.startTime) || + !Number.isFinite(request.endTime) || + request.startTime < 0 || + request.endTime <= request.startTime || + (current.payload.maxMediaDuration > 0 && + request.endTime - request.startTime > current.payload.maxMediaDuration + 0.001) || + (current.payload.mediaDuration !== undefined && + request.endTime > current.payload.mediaDuration + 0.001) + ) { + return { ok: false, message: 'The selected preview range is invalid.' }; + } + try { + const previewSession = await current.preview; + await previewSession.play(request.startTime, request.endTime); + return { ok: true }; + } catch (error) { + return { + ok: false, + message: `Audio preview unavailable: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } + + async function stopPreview(reviewId: string): Promise { + const current = active; + if (!current || reviewId !== current.payload.reviewId) { + return { ok: false, message: 'This timing review is no longer active.' }; + } + try { + const previewSession = await current.preview; + await previewSession.stop(); + return { ok: true }; + } catch (error) { + return { + ok: false, + message: `Could not stop preview: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } + + function resolveReview(request: MediaTimingReviewResolveRequest): MediaTimingReviewActionResult { + const current = active; + if (!current || request.reviewId !== current.payload.reviewId) { + return { ok: false, message: 'This timing review is no longer active.' }; + } + if (request.decision.action === 'confirm') { + const { startTime, endTime } = request.decision; + if ( + !Number.isFinite(startTime) || + !Number.isFinite(endTime) || + startTime < 0 || + endTime <= startTime || + (current.payload.maxMediaDuration > 0 && + endTime - startTime > current.payload.maxMediaDuration + 0.001) || + (current.payload.mediaDuration !== undefined && + endTime > current.payload.mediaDuration + 0.001) + ) { + return { ok: false, message: 'The selected timing range is invalid.' }; + } + } + current.resolve(request.decision); + return { ok: true }; + } + + async function cleanupActiveReview(): Promise { + const current = active; + active = null; + if (!current) return; + void current.preview.then((session) => session.dispose()).catch(() => {}); + if (current.restorePlayback && current.mpvClient.connected) { + current.mpvClient.send({ command: ['set_property', 'pause', 'no'] }); + } + } + + async function dispose(): Promise { + active?.resolve({ action: 'use-original' }); + await cleanupActiveReview(); + restorePendingPlayback(); + } + + return { + requestReview, + previewRange, + stopPreview, + resolveReview, + dispose, + }; +} diff --git a/src/preload.ts b/src/preload.ts index a48fdb0c..f36a5a3a 100644 --- a/src/preload.ts +++ b/src/preload.ts @@ -69,6 +69,9 @@ import type { OverlayNotificationEventPayload, OverlayNotificationPosition, ChangelogSnapshot, + MediaTimingReviewOpenPayload, + MediaTimingReviewPreviewRequest, + MediaTimingReviewResolveRequest, } from './types'; import { IPC_CHANNELS } from './shared/ipc/contracts'; @@ -181,6 +184,11 @@ const onOpenYoutubeTrackPickerEvent = createQueuedIpcListenerWithPayload payload as YoutubePickerOpenPayload, ); +const onOpenMediaTimingReviewEvent = + createQueuedIpcListenerWithPayload( + IPC_CHANNELS.event.mediaTimingReviewOpen, + (payload) => payload as MediaTimingReviewOpenPayload, + ); const onOpenPlaylistBrowserEvent = createQueuedIpcListener(IPC_CHANNELS.event.playlistBrowserOpen); const onCancelYoutubeTrackPickerEvent = createQueuedIpcListener( IPC_CHANNELS.event.youtubePickerCancel, @@ -458,6 +466,13 @@ const electronAPI: ElectronAPI = { onOpenJimaku: onOpenJimakuEvent, onOpenTsukihime: onOpenTsukihimeEvent, onOpenYoutubeTrackPicker: onOpenYoutubeTrackPickerEvent, + onOpenMediaTimingReview: onOpenMediaTimingReviewEvent, + previewMediaTimingReview: (request: MediaTimingReviewPreviewRequest) => + ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewPreview, request), + stopMediaTimingReviewPreview: (reviewId: string) => + ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewStopPreview, reviewId), + resolveMediaTimingReview: (request: MediaTimingReviewResolveRequest) => + ipcRenderer.invoke(IPC_CHANNELS.request.mediaTimingReviewResolve, request), onOpenPlaylistBrowser: onOpenPlaylistBrowserEvent, onOpenCharacterDictionaryManager: onOpenCharacterDictionaryManagerEvent, onSubtitleSidebarToggle: onSubtitleSidebarToggleEvent, diff --git a/src/renderer/handlers/keyboard.ts b/src/renderer/handlers/keyboard.ts index 04f63381..95c16d5f 100644 --- a/src/renderer/handlers/keyboard.ts +++ b/src/renderer/handlers/keyboard.ts @@ -18,6 +18,7 @@ export function createKeyboardHandlers( handleJimakuKeydown: (e: KeyboardEvent) => boolean; handleTsukihimeKeydown: (e: KeyboardEvent) => boolean; handleYoutubePickerKeydown: (e: KeyboardEvent) => boolean; + handleMediaTimingReviewKeydown?: (e: KeyboardEvent) => boolean; handlePlaylistBrowserKeydown: (e: KeyboardEvent) => boolean; handleControllerSelectKeydown: (e: KeyboardEvent) => boolean; handleControllerDebugKeydown: (e: KeyboardEvent) => boolean; @@ -1078,6 +1079,11 @@ export function createKeyboardHandlers( ); document.addEventListener('keydown', (e: KeyboardEvent) => { + if (ctx.state.mediaTimingReviewModalOpen) { + options.handleMediaTimingReviewKeydown?.(e); + return; + } + if (isKeyboardDrivenModeToggle(e) && ctx.platform.isModalLayer) { e.preventDefault(); handleKeyboardModeToggleRequested(); diff --git a/src/renderer/index.html b/src/renderer/index.html index 49c09027..217195bf 100644 --- a/src/renderer/index.html +++ b/src/renderer/index.html @@ -196,6 +196,195 @@ +