// SPDX-License-Identifier: GPL-3.0-or-later import { createAudioRing, createCapturePinStore, createFrameRing, encodeMonoWav, } from "./capture-buffer.js"; import { encodeCapturedAnimation } from "./capture-encoder-client.js"; import { MAX_ANIMATED_AVIF_BYTES } from "./avif-sequence.js"; import { createCaptureTimeline, resolveCaptureInterval } from "./capture-timeline.js"; const JOB_LIFETIME_MS = 2 * 60 * 1000; export const MEDIA_DRAIN_MS = 250; function safeAssetId(value = crypto.randomUUID()) { const id = value.toLowerCase().replace(/[^a-z0-9]/gu, ""); if (!id) throw new Error("Could not allocate a media asset identity."); return id; } function waitUntil(deadline, now, setTimer) { const remaining = Math.max(0, deadline - now()); if (remaining === 0) return Promise.resolve(); return new Promise(resolve => setTimer(resolve, remaining)); } function assertJobOwner(job, owner) { if (owner && (owner.tabId !== job.owner?.tabId || owner.documentId !== job.owner?.documentId)) { throw new Error("This reading document does not own the media export job."); } } export function createCaptureSession({ now = () => performance.timeOrigin + performance.now(), wallNow = Date.now, setTimer = setTimeout, clearTimer = clearTimeout, encodeAnimation = encodeCapturedAnimation, randomId = () => crypto.randomUUID(), } = {}) { let config = null; let captureSessionId = ""; let state = "disabled"; let statusError = ""; let mediaSource = null; let capturedAudioAvailable = false; let linkedPage = null; let texthookerStatus = "Disabled"; let texthookerActive = false; let texthookerSource = null; let videoDeliveredThroughMs = -Infinity; const timeline = createCaptureTimeline(); let frameRing = null; let audioRing = null; let pins = createCapturePinStore({ now: wallNow }); const jobs = new Map(); let activeJobId = null; function status() { const frameSize = frameRing?.size() ?? { count: 0, bytes: 0 }; const audioSize = audioRing?.size() ?? { blocks: 0, samples: 0 }; return { state, error: statusError, captureSessionId, mediaSource, linkedPage, texthookerStatus, texthookerActive, history: { frameCount: frameSize.count, frameBytes: frameSize.bytes, audioBlocks: audioSize.blocks, audioSamples: audioSize.samples, frameOldestMs: frameRing?.oldestTimestamp() ?? null, frameNewestMs: frameRing?.newestTimestamp() ?? null, audioOldestMs: audioRing?.oldestTimestamp() ?? null, audioNewestMs: audioRing?.newestTimestamp() ?? null, oldestMs: oldestRequiredTimestamp(), newestMs: (() => { const newest = Math.max(frameRing?.newestTimestamp() ?? -Infinity, audioRing?.newestTimestamp() ?? -Infinity); return Number.isFinite(newest) ? newest : null; })(), }, pinActive: pins.active() || activeJobId !== null, }; } function oldestRequiredTimestamp() { if (!config) return null; const values = []; if (config.includeAnimation) values.push(frameRing?.oldestTimestamp()); if (config.includeCapturedAudio && capturedAudioAvailable) values.push(audioRing?.oldestTimestamp()); const available = values.filter(Number.isFinite); return available.length ? Math.max(...available) : null; } function configure(value) { cancelOwnedCapture("Capture settings changed."); config = structuredClone(value); frameRing = createFrameRing({ maxAgeMs: config.historySeconds * 1000 }); audioRing = createAudioRing({ maxAgeMs: config.historySeconds * 1000 }); pins = createCapturePinStore({ now: wallNow }); timeline.reset(); jobs.clear(); activeJobId = null; texthookerActive = false; texthookerSource = null; videoDeliveredThroughMs = -Infinity; texthookerStatus = config.timingMode === "auto" && config.texthooker.enabled ? "Disconnected" : "Disabled"; state = config.enabled ? "stopped" : "disabled"; statusError = ""; } function start({ sourceName = "Shared tab", displaySurface = "browser", audioAvailable = true } = {}) { if (!config?.enabled) throw new Error("Enable media capture in Settings first."); captureSessionId = randomId(); capturedAudioAvailable = audioAvailable === true; mediaSource = { name: String(sourceName).slice(0, 200), displaySurface, audioAvailable: capturedAudioAvailable, }; state = "recording"; statusError = ""; return status(); } function stop(error = "") { cancelOwnedCapture(error || "Capture stopped."); state = config ? "stopped" : "disabled"; statusError = error; captureSessionId = ""; mediaSource = null; capturedAudioAvailable = false; linkedPage = null; texthookerActive = false; texthookerSource = null; videoDeliveredThroughMs = -Infinity; texthookerStatus = config?.timingMode === "auto" && config.texthooker.enabled ? "Disconnected" : "Disabled"; frameRing?.clear(); audioRing?.clear(); timeline.reset(); pins.clear(); jobs.clear(); activeJobId = null; } function requireRecording() { if (state !== "recording" || !captureSessionId) throw new Error("Start capture before looking up text."); } function addFrame(frame) { requireRecording(); const added = frameRing.append(frame); videoDelivered(added.timestampMs); return added; } function videoDelivered(timestampMs) { requireRecording(); videoDeliveredThroughMs = Math.max(videoDeliveredThroughMs, timestampMs); currentPin()?.checkDrain?.(); } function addAudio(block) { requireRecording(); const added = audioRing.append(block); currentPin()?.checkDrain?.(); return added; } function assertAudioCapture() { if (state !== "recording" || !captureSessionId) { throw new Error("Start media capture with shared audio before attaching browser text-to-speech to Anki."); } if (!config.includeCapturedAudio || !capturedAudioAvailable) { throw new Error("The active media capture has no shared audio. Start capture again and enable audio in Chrome's share picker."); } } function selectAudio(startMs, endMs) { assertAudioCapture(); const selected = audioRing.select(startMs, endMs); if (selected.partial) { throw new Error("The active media capture did not record all browser text-to-speech samples. Try adding the note again after shared audio resumes."); } return selected; } function setLinkedPage(page) { const previous = linkedPage; if (previous && (previous.tabId !== page?.tabId || previous.documentId !== page?.documentId) && activePin) releasePin(activePin.token); if (previous) { const sourceId = `tab:${previous.tabId}`; const endMs = now(); for (const sourceKind of ["cue", "dom"]) { for (const record of timeline.closeSource(sourceKind, sourceId, undefined, endMs)) { adjustOpenPin(record); } } } linkedPage = page ? { tabId: page.tabId, documentId: page.documentId, title: String(page.title || "").slice(0, 200), url: String(page.url || "").slice(0, 2048), } : null; } function textBegin(record) { requireRecording(); const begun = timeline.begin(record); if (record.sourceKind === "texthooker") { texthookerSource = { sourceId: record.sourceId, sourceEpoch: record.sourceEpoch }; } return begun; } function textClose(identity, endMs = now()) { const closed = timeline.close(identity, endMs); if (closed) adjustOpenPin(closed); return closed; } function closeTextSource(sourceKind, sourceId, sourceEpoch, endMs = now()) { const closed = timeline.closeSource(sourceKind, sourceId, sourceEpoch, endMs); for (const record of closed) adjustOpenPin(record); return closed; } function setTexthooker(nextStatus, active = false) { texthookerStatus = nextStatus; texthookerActive = active; if (!active) texthookerSource = null; } function adjustOpenPin(closed) { const active = currentPin(); if (!active || active.finalized || active.sourceKind !== closed.sourceKind || active.sourceId !== closed.sourceId || active.sourceEpoch !== closed.sourceEpoch || active.occurrenceId !== closed.occurrenceId) return; const offset = closed.sourceKind === "cue" ? 0 : config.estimatedOffsetMs; const closedEnd = closed.endMs + offset; if (closedEnd > active.startMs && closedEnd < active.endMs) { active.endMs = closedEnd; active.deadlineVersion += 1; active.finishDrain?.(); void finalizeAtDeadline(active, active.deadlineVersion); } } function currentPin() { return activePin ?? (activeJobId ? jobs.get(activeJobId)?.pin ?? null : null); } let activePin = null; function cancelOwnedCapture(message) { const error = new Error(message); if (activePin && !activePin.finalized) { activePin.finishDrain?.(); activePin.rejectReady(error); } activePin = null; for (const job of jobs.values()) { job.controller.abort(); if (!job.pin.finalized) { job.pin.finishDrain?.(); job.pin.rejectReady(error); } } } function mediaDelivered(pin) { return (!config.includeAnimation || videoDeliveredThroughMs >= pin.endMs) && (!config.includeCapturedAudio || !pin.audioAvailable || audioRing.covers(pin.startMs, pin.endMs)); } function drainMedia(pin) { if (mediaDelivered(pin)) return Promise.resolve(); return new Promise(resolve => { const timer = setTimer(finish, MEDIA_DRAIN_MS); function finish() { clearTimer(timer); pin.finishDrain = null; pin.checkDrain = null; resolve(); } pin.finishDrain = finish; pin.checkDrain = () => { if (mediaDelivered(pin)) finish(); }; }); } async function finalizeAtDeadline(pin, version) { await waitUntil(pin.endMs, now, setTimer); if (currentPin() !== pin || pin.deadlineVersion !== version || pin.finalized) return; try { // Delivery can trail its timestamp (JPEG encoding and worklet batches). // Wait for that delivery without moving the lookup's frozen interval. await drainMedia(pin); if (currentPin() !== pin || pin.deadlineVersion !== version || pin.finalized) return; pin.frames = config.includeAnimation ? frameRing.select(pin.startMs, pin.endMs) : []; pin.audio = config.includeCapturedAudio && pin.audioAvailable ? audioRing.select(pin.startMs, pin.endMs) : null; pin.mediaErrors = {}; // A display-capture track can emit only changed frames. Once the bounded // drain ends, its last pixels remain valid until source mute/loss stops us. if (pin.audio?.partial) { pin.mediaErrors.audio = "Captured audio is missing samples in this clip. Look up the text again."; } if (config.includeCapturedAudio && !pin.audioAvailable) pin.partial = true; pin.partial ||= pin.audio?.partial === true; pin.finalized = true; pin.resolveReady(pin); } catch (error) { pin.rejectReady(error); pins.release(pin.token); activePin = null; } } function pinLookup({ lookupText, occurrenceId = "", occurrenceSourceKind = "", lookupTimeMs = now() }) { requireRecording(); pruneJobs(); if (activeJobId !== null) throw new Error("Another captured clip is still exporting. Finish or cancel it first."); if (!config.includeAnimation && config.includeCapturedAudio && !capturedAudioAvailable) { throw new Error("The shared source did not provide audio for media capture."); } const availableStartMs = oldestRequiredTimestamp(); if (!Number.isFinite(availableStartMs)) throw new Error("Capture history is still warming up."); const interval = resolveCaptureInterval({ records: timeline.snapshot(), lookupText, occurrenceId, occurrenceSourceKind, lookupTimeMs, availableStartMs, timingMode: config.timingMode, clipSeconds: config.clipSeconds, estimatedOffsetMs: config.estimatedOffsetMs, texthookerActive, texthookerSource: texthookerActive ? texthookerSource : null, }); if (!interval) throw new Error("No retained capture interval is available for this lookup."); const assetId = safeAssetId(randomId()); let resolveReady, rejectReady; const ready = new Promise((resolve, reject) => { resolveReady = resolve; rejectReady = reject; }); void ready.catch(() => {}); const created = pins.create({ ...interval, captureSessionId, assetId, animationFilename: `hachidori-${assetId}.avif`, audioFilename: `hachidori-${assetId}.wav`, audioAvailable: capturedAudioAvailable, deadlineVersion: 1, finalized: false, frames: null, audio: null, mediaErrors: {}, ready, resolveReady, rejectReady, }); activePin = pins.get(created.token); void finalizeAtDeadline(activePin, activePin.deadlineVersion); return { token: created.token, captureSessionId, sourceKind: interval.sourceKind, sourceLabel: interval.sourceLabel, partial: interval.partial === true, animationFilename: created.animationFilename, audioFilename: created.audioFilename, readyAtMs: interval.endMs, }; } function releasePin(token) { let released = pins.release(token); // Status may have expired the token before the reader dismisses its clip. // The session still owns those media copies until this reference is retired. if (activePin !== null && activePin.token === token) { activePin.finishDrain?.(); activePin.rejectReady(new Error("The capture pin was released.")); activePin = null; released = true; } return released; } function pruneJobs() { const cutoff = wallNow() - JOB_LIFETIME_MS; for (const [id, job] of jobs) { if (job.updatedAt >= cutoff) continue; job.controller.abort(); if (!job.pin.finalized) { job.pin.finishDrain?.(); job.pin.rejectReady(new Error("The media export job expired.")); } jobs.delete(id); if (activeJobId === id) activeJobId = null; } } function beginExport(token, requirements, owner = linkedPage) { pruneJobs(); if (activeJobId !== null) { const existing = jobs.get(activeJobId); if (existing?.token === token) { assertJobOwner(existing, owner); return { jobId: existing.id, state: existing.state, sourceLabel: existing.sourceLabel, partial: existing.partial }; } throw new Error("Another captured clip is still exporting."); } const pin = pins.get(token); if (pin?.captureSessionId !== captureSessionId) throw new Error("The capture pin expired. Look up the text again."); const includeAnimation = requirements?.includeAnimation === true && config.includeAnimation; const includeAudio = requirements?.includeAudio === true && config.includeCapturedAudio && pin.audioAvailable; if (!includeAnimation && requirements?.includeAudio === true && config.includeCapturedAudio && !pin.audioAvailable) { throw new Error("The shared source did not provide audio for the mapped captured-audio field."); } if (!includeAnimation && !includeAudio) throw new Error("The selected Anki fields do not reference captured media."); const id = randomId(); const controller = new AbortController(); const job = { id, token, state: "finishing", error: "", progress: 0, total: 0, encoderHeapBytes: 0, sourceLabel: pin.sourceLabel, partial: pin.partial === true, assets: {}, updatedAt: wallNow(), controller, pin, owner: owner ? { tabId: owner.tabId, documentId: owner.documentId } : null, warnings: requirements?.includeAudio === true && !pin.audioAvailable ? ["The shared source did not provide audio; this note will use animation only."] : [] }; jobs.set(id, job); activeJobId = id; pins.release(token); if (activePin === pin) activePin = null; void (async () => { try { await pin.ready; if (controller.signal.aborted) throw new Error("Media encoding was cancelled."); if (includeAudio && pin.mediaErrors.audio) throw new Error(pin.mediaErrors.audio); job.state = "encoding"; job.updatedAt = wallNow(); if (includeAnimation) { job.assets.animation = { filename: pin.animationFilename, data: await encodeAnimation(pin.frames, { endMs: pin.endMs, videoPreset: config.videoPreset }, { signal: controller.signal, onProgress(completed, total, heapBytes) { job.progress = completed; job.total = total; if (Number.isSafeInteger(heapBytes) && heapBytes > job.encoderHeapBytes) { job.encoderHeapBytes = heapBytes; } job.updatedAt = wallNow(); }, }), }; if (job.assets.animation.data.byteLength > MAX_ANIMATED_AVIF_BYTES) { throw new Error("Animated AVIF exceeds its 4 MiB output limit."); } } if (includeAudio) { job.assets.audio = { filename: pin.audioFilename, data: encodeMonoWav(pin.audio.samples, pin.audio.sampleRate), }; } job.partial ||= pin.partial || pin.audio?.partial === true; job.state = "ready"; } catch (error) { job.state = "error"; job.error = error instanceof Error ? error.message : String(error); } job.updatedAt = wallNow(); })(); return { jobId: id, state: job.state, sourceLabel: job.sourceLabel, partial: job.partial }; } function jobStatus(id, owner = null) { pruneJobs(); const job = jobs.get(id); if (!job) throw new Error("The media export job expired."); assertJobOwner(job, owner); return { jobId: id, state: job.state, error: job.error, progress: job.progress, total: job.total, encoderHeapBytes: job.encoderHeapBytes, sourceLabel: job.sourceLabel, partial: job.partial, warnings: [...job.warnings], assets: Object.fromEntries(Object.entries(job.assets).map(([kind, asset]) => [kind, { filename: asset.filename, byteLength: asset.data.byteLength }])), }; } function jobAsset(id, kind) { const job = jobs.get(id); if (job?.state !== "ready" || !["animation", "audio"].includes(kind) || !job.assets[kind]) { throw new Error("The requested captured media asset is unavailable."); } const asset = job.assets[kind]; return { filename: asset.filename, data: asset.data.slice() }; } function completeExport(id) { const job = jobs.get(id); if (!job) return false; jobs.delete(id); if (activeJobId === id) activeJobId = null; return true; } function cancelExport(id, owner = null) { const job = jobs.get(id); if (!job) return false; assertJobOwner(job, owner); job.controller.abort(); if (!job.pin.finalized) { job.pin.finishDrain?.(); job.pin.rejectReady(new Error("The media export job was cancelled.")); } jobs.delete(id); if (activeJobId === id) activeJobId = null; return true; } return { configure, start, stop, status, addFrame, videoDelivered, addAudio, assertAudioCapture, selectAudio, setLinkedPage, textBegin, textClose, closeTextSource, setTexthooker, pinLookup, releasePin, beginExport, jobStatus, jobAsset, completeExport, cancelExport, }; }