import { randomUUID } from 'crypto'; import type { MediaTimingReviewActionResult, MediaTimingReviewContextLine, MediaTimingReviewDecision, MediaTimingReviewOpenPayload, MediaTimingReviewPreviewRequest, MediaTimingReviewRequest, MediaTimingReviewResolveRequest, MediaTimingReviewWaveformRequest, MediaTimingReviewWaveformResult, } from '../../types/anki'; import type { SpeechWaveformOptions } from '../../core/services/media-timing-waveform'; const INITIAL_TIMELINE_MARGIN_SECONDS = 2; const REVIEW_DECISION_TIMEOUT_MS = 5 * 60_000; const CONTEXT_LINE_LIMIT = 12; const CONTEXT_LINE_EPSILON_SECONDS = 0.05; interface ReviewMpvClient { connected: boolean; currentVideoPath: string; currentAudioStreamIndex?: number | null; requestProperty?: (name: string) => Promise; send: (payload: { command: Array }) => void; } interface PreviewSession { start(options: { mediaPath: string; executablePath?: string; audioTrackId?: number; volume?: number; }): Promise; play(startTime: number, endTime: number): Promise; stop(): Promise; dispose(): void; } interface ActiveReview { payload: MediaTimingReviewOpenPayload; mediaPath: string; audioStreamIndex?: number; mpvClient: ReviewMpvClient; restorePlayback: boolean; preview: Promise; resolve: (decision: MediaTimingReviewDecision) => void; } export interface MediaTimingReviewRuntimeDeps { getMpvClient: () => ReviewMpvClient | null; getCurrentMediaPath: () => string | null; getMpvExecutablePath: () => string; createPreviewSession: () => PreviewSession; generateWaveform: (options: SpeechWaveformOptions) => Promise; getSubtitleContextLines?: (range: { startTime: number; endTime: number }) => { previous: MediaTimingReviewContextLine[]; next: MediaTimingReviewContextLine[]; }; decisionTimeoutMs?: number; openModal: (payload: MediaTimingReviewOpenPayload) => Promise; showStatus: (message: string) => void; } function finiteNumber(value: unknown): number | null { return typeof value === 'number' && Number.isFinite(value) ? value : null; } function booleanProperty(value: unknown): boolean | null { if (typeof value === 'boolean') return value; if (value === 'yes' || value === 1) return true; if (value === 'no' || value === 0) return false; return null; } /** * Picks the subtitle lines adjacent to the mined range that the review modal can pull * onto the card. Parsed cues cover both directions; when none are loaded (e.g. the * active track was never parsed) the timing tracker's history still provides the * lines that already played, so only "next" is unavailable. */ export function collectMediaTimingContextLines(options: { cues: readonly { text: string; startTime: number; endTime: number }[]; fallbackPrevious?: readonly { displayText: string; startTime: number; endTime: number }[]; startTime: number; endTime: number; }): { previous: MediaTimingReviewContextLine[]; next: MediaTimingReviewContextLine[] } { const usable = options.cues .filter( (cue) => cue.text.trim().length > 0 && Number.isFinite(cue.startTime) && Number.isFinite(cue.endTime) && cue.endTime > cue.startTime, ) .sort((a, b) => a.startTime - b.startTime || a.endTime - b.endTime); let previous = usable .filter((cue) => cue.endTime <= options.startTime + CONTEXT_LINE_EPSILON_SECONDS) .slice(-CONTEXT_LINE_LIMIT) .map(({ text, startTime, endTime }) => ({ text: text.trim(), startTime, endTime })); const next = usable .filter((cue) => cue.startTime >= options.endTime - CONTEXT_LINE_EPSILON_SECONDS) .slice(0, CONTEXT_LINE_LIMIT) .map(({ text, startTime, endTime }) => ({ text: text.trim(), startTime, endTime })); if (previous.length === 0 && options.fallbackPrevious) { previous = options.fallbackPrevious .filter( (entry) => entry.displayText.trim().length > 0 && Number.isFinite(entry.startTime) && Number.isFinite(entry.endTime) && entry.endTime > entry.startTime && entry.endTime <= options.startTime + CONTEXT_LINE_EPSILON_SECONDS, ) .slice(-CONTEXT_LINE_LIMIT) .map((entry) => ({ text: entry.displayText.trim(), startTime: entry.startTime, endTime: entry.endTime, })); } return { previous, next }; } function isValidMediaTimingRange( payload: MediaTimingReviewOpenPayload, startTime: number, endTime: number, ): boolean { return ( Number.isFinite(startTime) && Number.isFinite(endTime) && startTime >= 0 && endTime > startTime && (payload.maxMediaDuration <= 0 || endTime - startTime <= payload.maxMediaDuration + 0.001) && (payload.mediaDuration === undefined || endTime <= payload.mediaDuration + 0.001) ); } export function buildMediaTimingReviewPayload( request: MediaTimingReviewRequest, options: { reviewId: string; mediaDuration?: number; contextLines?: { previous: MediaTimingReviewContextLine[]; next: MediaTimingReviewContextLine[]; }; }, ): MediaTimingReviewOpenPayload { const duration = finiteNumber(options.mediaDuration); const maxTime = duration !== null && duration > 0 ? duration : Number.POSITIVE_INFINITY; const paddedStart = Math.max(0, request.startTime - request.audioPadding); let paddedEnd = Math.min(maxTime, request.endTime + request.audioPadding); const maxMediaDuration = Math.max(0, request.maxMediaDuration); if (maxMediaDuration > 0 && paddedEnd - paddedStart > maxMediaDuration) { paddedEnd = paddedStart + maxMediaDuration; } if (paddedEnd <= paddedStart) { paddedEnd = Math.min(maxTime, paddedStart + 0.1); } const timelineStartTime = Math.max(0, paddedStart - INITIAL_TIMELINE_MARGIN_SECONDS); const timelineEndTime = Math.max( paddedEnd, Math.min(maxTime, paddedEnd + INITIAL_TIMELINE_MARGIN_SECONDS), ); return { reviewId: options.reviewId, kind: request.kind, text: request.text, previousLines: options.contextLines?.previous ?? [], nextLines: options.contextLines?.next ?? [], ...(request.noteId !== undefined ? { noteId: request.noteId } : {}), originalStartTime: request.startTime, originalEndTime: request.endTime, selectionStartTime: paddedStart, selectionEndTime: paddedEnd, timelineStartTime, timelineEndTime, ...(duration !== null && duration > 0 ? { mediaDuration: duration } : {}), maxMediaDuration, }; } export function createMediaTimingReviewRuntime(deps: MediaTimingReviewRuntimeDeps) { let active: ActiveReview | null = null; let reviewInProgress = false; let pendingPauseRestore: ReviewMpvClient | null = null; function restorePendingPlayback(): void { const mpvClient = pendingPauseRestore; pendingPauseRestore = null; if (mpvClient?.connected) { mpvClient.send({ command: ['set_property', 'pause', 'no'] }); } } async function runReview(request: MediaTimingReviewRequest): Promise { const mpvClient = deps.getMpvClient(); const mediaPath = deps.getCurrentMediaPath()?.trim() || mpvClient?.currentVideoPath?.trim() || ''; if (!mpvClient?.connected || !mediaPath) { deps.showStatus('Timing review unavailable. Using the original subtitle timing.'); return { action: 'use-original' }; } const [pauseRaw, durationRaw, audioTrackRaw, volumeRaw] = await Promise.all([ mpvClient.requestProperty?.('pause').catch(() => null) ?? null, mpvClient.requestProperty?.('duration').catch(() => null) ?? null, mpvClient.requestProperty?.('aid').catch(() => null) ?? null, mpvClient.requestProperty?.('volume').catch(() => null) ?? null, ]); const pauseState = booleanProperty(pauseRaw); mpvClient.send({ command: ['set_property', 'pause', 'yes'] }); pendingPauseRestore = pauseState === false ? mpvClient : null; let contextLines: ReturnType> | undefined; try { contextLines = deps.getSubtitleContextLines?.({ startTime: request.startTime, endTime: request.endTime, }); } catch { contextLines = undefined; } const payload = buildMediaTimingReviewPayload(request, { reviewId: randomUUID(), mediaDuration: finiteNumber(durationRaw) ?? undefined, ...(contextLines ? { contextLines } : {}), }); const previewSession = deps.createPreviewSession(); const preview = previewSession .start({ mediaPath, executablePath: deps.getMpvExecutablePath(), audioTrackId: finiteNumber(audioTrackRaw) ?? undefined, volume: finiteNumber(volumeRaw) ?? undefined, }) .then(() => previewSession) .catch((error) => { previewSession.dispose(); throw error; }); void preview.catch(() => {}); let resolveDecision!: (decision: MediaTimingReviewDecision) => void; const decisionPromise = new Promise((resolve) => { resolveDecision = resolve; }); active = { payload, mediaPath, ...(mpvClient.currentAudioStreamIndex !== null && mpvClient.currentAudioStreamIndex !== undefined ? { audioStreamIndex: mpvClient.currentAudioStreamIndex } : {}), mpvClient, restorePlayback: pendingPauseRestore === mpvClient, preview, resolve: resolveDecision, }; pendingPauseRestore = null; const opened = await deps.openModal(payload).catch(() => false); if (!opened) { await cleanupActiveReview(); deps.showStatus('Timing review could not open. Using the original subtitle timing.'); return { action: 'use-original' }; } const decisionWatchdog = setTimeout( () => resolveDecision({ action: 'use-original' }), Math.max(0, deps.decisionTimeoutMs ?? REVIEW_DECISION_TIMEOUT_MS), ); let decision: MediaTimingReviewDecision; try { decision = await decisionPromise; } finally { clearTimeout(decisionWatchdog); } await cleanupActiveReview(); return decision; } async function requestReview( request: MediaTimingReviewRequest, ): Promise { if (active || reviewInProgress) { deps.showStatus('Finish the current timing review before mining another card.'); return { action: 'use-original' }; } reviewInProgress = true; try { return await runReview(request); } catch { await cleanupActiveReview(); restorePendingPlayback(); deps.showStatus('Timing review failed. Using the original subtitle timing.'); return { action: 'use-original' }; } finally { reviewInProgress = false; } } async function previewRange( request: MediaTimingReviewPreviewRequest, ): Promise { const current = active; if (!current || request.reviewId !== current.payload.reviewId) { return { ok: false, message: 'This timing review is no longer active.' }; } if (!isValidMediaTimingRange(current.payload, request.startTime, request.endTime)) { return { ok: false, message: 'The selected preview range is invalid.' }; } try { const previewSession = await current.preview; await previewSession.play(request.startTime, request.endTime); return { ok: true }; } catch (error) { return { ok: false, message: `Audio preview unavailable: ${error instanceof Error ? error.message : String(error)}`, }; } } async function getWaveform( request: MediaTimingReviewWaveformRequest, ): Promise { const current = active; if (!current || request.reviewId !== current.payload.reviewId) { return { ok: false, message: 'This timing review is no longer active.' }; } if ( !Number.isFinite(request.startTime) || !Number.isFinite(request.endTime) || request.startTime < 0 || request.endTime <= request.startTime || (current.payload.mediaDuration !== undefined && request.endTime > current.payload.mediaDuration + 0.001) ) { return { ok: false, message: 'The waveform range is invalid.' }; } try { const peaks = await deps.generateWaveform({ mediaPath: current.mediaPath, startTime: request.startTime, endTime: request.endTime, ...(current.audioStreamIndex !== undefined ? { audioStreamIndex: current.audioStreamIndex } : {}), }); if (active !== current || peaks.length < 2 || peaks.some((peak) => !Number.isFinite(peak))) { return { ok: false, message: 'Timing waveform is unavailable.' }; } return { ok: true, peaks }; } catch { return { ok: false, message: 'Timing waveform is unavailable.' }; } } async function stopPreview(reviewId: string): Promise { const current = active; if (!current || reviewId !== current.payload.reviewId) { return { ok: false, message: 'This timing review is no longer active.' }; } try { const previewSession = await current.preview; await previewSession.stop(); return { ok: true }; } catch (error) { return { ok: false, message: `Could not stop preview: ${error instanceof Error ? error.message : String(error)}`, }; } } function resolveReview(request: MediaTimingReviewResolveRequest): MediaTimingReviewActionResult { const current = active; if (!current || request.reviewId !== current.payload.reviewId) { return { ok: false, message: 'This timing review is no longer active.' }; } if (request.decision.action === 'confirm') { const { startTime, endTime, text } = request.decision; if (!isValidMediaTimingRange(current.payload, startTime, endTime)) { return { ok: false, message: 'The selected timing range is invalid.' }; } if (text !== undefined && (typeof text !== 'string' || text.trim().length === 0)) { return { ok: false, message: 'The combined sentence text is invalid.' }; } } current.resolve(request.decision); return { ok: true }; } async function cleanupActiveReview(): Promise { const current = active; active = null; if (!current) return; void current.preview.then((session) => session.dispose()).catch(() => {}); if (current.restorePlayback && current.mpvClient.connected) { current.mpvClient.send({ command: ['set_property', 'pause', 'no'] }); } } async function dispose(): Promise { active?.resolve({ action: 'use-original' }); await cleanupActiveReview(); restorePendingPlayback(); } return { requestReview, previewRange, getWaveform, stopPreview, resolveReview, dispose, }; }