From fb7c0d149776a1d933a06036849ef84877ac80a5 Mon Sep 17 00:00:00 2001 From: sudacode Date: Thu, 25 Jun 2026 22:03:21 -0700 Subject: [PATCH] feat(youtube): harden media cache and drain queued cards on failure - Add --force-ipv4 and retry flags to yt-dlp background downloads - Clean stale cache dirs on startup and before new downloads - Notify and drain pending card updates when cache download fails - Fix generateAudio/generateImage config checks (missing = enabled) --- .../youtube-media-generation-reliability.md | 2 + docs-site/configuration.md | 4 +- docs-site/youtube-integration.md | 7 + src/anki-integration.ts | 16 ++- src/anki-integration/card-creation.ts | 120 +++++++++-------- src/anki-integration/note-update-workflow.ts | 9 +- .../pending-youtube-media-queue.test.ts | 123 +++++++++++++++++- .../pending-youtube-media-queue.ts | 60 ++++++++- src/core/services/youtube/media-cache.test.ts | 117 +++++++++++++++++ src/core/services/youtube/media-cache.ts | 48 ++++++- src/main.ts | 17 +++ src/media-generator.test.ts | 2 + src/media-input.ts | 13 +- 13 files changed, 465 insertions(+), 73 deletions(-) diff --git a/changes/youtube-media-generation-reliability.md b/changes/youtube-media-generation-reliability.md index 73dccac1..1ade662c 100644 --- a/changes/youtube-media-generation-reliability.md +++ b/changes/youtube-media-generation-reliability.md @@ -3,3 +3,5 @@ area: youtube - Improved YouTube card media generation by sending safer ffmpeg request options for resolved streams and skipping stale stream maps, including cached YouTube files. - Added `youtube.mediaCache.mode` with `direct` and `background` modes so YouTube card audio/image extraction can optionally use a background yt-dlp media cache when direct stream extraction is unreliable; background mode now announces cache download start/readiness through queued overlay/OSD notifications, creates text-only cards while the cache downloads, queues media updates for the mined note IDs, fills audio/image fields once the cached file is ready, and caps background downloads at `youtube.mediaCache.maxHeight` 720p by default. +- Cleaned stale YouTube background media cache files on startup and before new background downloads so only the active cached media file remains. +- Hardened YouTube media cache downloads with IPv4/extractor retry flags and made failed background downloads notify users and clear queued media updates instead of leaving them pending silently. diff --git a/docs-site/configuration.md b/docs-site/configuration.md index 1c32ce98..b2a8f99f 100644 --- a/docs-site/configuration.md +++ b/docs-site/configuration.md @@ -1535,7 +1535,9 @@ Set defaults used by managed subtitle auto-selection and the `subminer` launcher | `mediaCache.mode` | `direct` \| `background` | YouTube card audio/image extraction mode (default `direct`) | | `mediaCache.maxHeight` | number | Maximum background cache download height. Set `0` for unlimited (default `720`) | -`mediaCache.mode: "direct"` extracts card media from the active YouTube stream URL. `mediaCache.mode: "background"` starts a separate yt-dlp media download after YouTube playback has loaded. Playback and subtitle loading do not wait for that download. Background cache downloads are capped by `mediaCache.maxHeight`, which defaults to 720p; set it to `0` to let yt-dlp choose the best available height. SubMiner announces when the background cache download starts and when the cache is ready, using the configured notification surface; overlay and OSD messages queue until the overlay or mpv is ready. If you mine cards before the cache is ready, SubMiner creates the text fields immediately, queues the audio/image work for those note IDs, shows a status notification, and fills the media fields once the cached file is ready. +`mediaCache.mode: "direct"` extracts card media from the active YouTube stream URL. `mediaCache.mode: "background"` starts a separate yt-dlp media download after YouTube playback has loaded. Playback and subtitle loading do not wait for that download. Use background mode if direct card media generation hits YouTube `403` errors from expiring stream URLs. + +Background cache downloads are capped by `mediaCache.maxHeight`, which defaults to 720p; set it to `0` to let yt-dlp choose the best available height. Downloads use IPv4 and yt-dlp retry flags to reduce YouTube throttling failures. SubMiner announces when the background cache download starts and when the cache is ready, using the configured notification surface; overlay and OSD messages queue until the overlay or mpv is ready. If you mine cards before the cache is ready, SubMiner creates the text fields immediately, queues the audio/image work for those note IDs, shows a status notification, and fills the media fields once the cached file is ready. If the cache download fails, SubMiner shows a failure notification, shows queued-card failure notifications, and clears the pending updates. Current launcher behavior: diff --git a/docs-site/youtube-integration.md b/docs-site/youtube-integration.md index 28d2e14a..a5ce157a 100644 --- a/docs-site/youtube-integration.md +++ b/docs-site/youtube-integration.md @@ -88,6 +88,12 @@ SubMiner handles several YouTube subtitle formats transparently: For auto-generated tracks, SubMiner prefers `srv3` > `srv2` > `srv1` > `vtt` (TimedText XML produces cleaner output). For manual tracks, `srt` > `vtt` is preferred. +## Card Media Cache + +By default, YouTube card audio and screenshots are extracted directly from mpv's active stream URLs. If generated card media fails with YouTube `403` errors, set `youtube.mediaCache.mode` to `"background"`. Background mode starts a separate `yt-dlp` media download after playback loads, creates text fields immediately, queues audio/image work for mined notes, and fills those fields once the local cache file is ready. + +Background cache downloads use IPv4 and retry flags to reduce YouTube throttling failures. If the background download still fails, SubMiner shows a cache failure notification, shows queued-card failure notifications, and clears those pending updates so cards are not left waiting silently. + ## Configuration Reference ### Primary Subtitle Languages @@ -131,6 +137,7 @@ Precedence: CLI flag > environment variable > `config.jsonc` > built-in default. - **No subtitles found**: The video may not have Japanese subtitles. Open the picker with `Ctrl+Alt+C` to see all available tracks. - **yt-dlp not found**: Install `yt-dlp` and ensure it is on `PATH`, or set `SUBMINER_YTDLP_BIN` to the binary path. - **Probe timeout**: `yt-dlp` has a 15-second timeout per operation. Slow connections or rate-limited IPs may hit this. Retry or update `yt-dlp`. +- **Card media `403` errors**: Switch `youtube.mediaCache.mode` from `"direct"` to `"background"` so card media is generated from a local `yt-dlp` cache instead of ffmpeg reading an expiring YouTube stream URL. - **Auto-caption quality**: YouTube auto-generated captions vary in quality. Manual subtitles (when available) are always preferred. - **`ytsearch:` targets**: `subminer ytsearch:"keyword"` plays the first search result. Subtitle availability depends on the matched video. - **Secondary subtitle fails**: Secondary track failures never block playback. The primary subtitle loads independently. diff --git a/src/anki-integration.ts b/src/anki-integration.ts index 0deb223e..9f5a1845 100644 --- a/src/anki-integration.ts +++ b/src/anki-integration.ts @@ -69,7 +69,10 @@ import { } from './anki-integration/media-source'; import type { PendingYoutubeMediaUpdate } from './anki-integration/pending-youtube-media'; import { PendingYoutubeMediaQueue } from './anki-integration/pending-youtube-media-queue'; -import type { PendingYoutubeMediaQueueReadyOptions } from './anki-integration/pending-youtube-media-queue'; +import type { + PendingYoutubeMediaQueueFailedOptions, + PendingYoutubeMediaQueueReadyOptions, +} from './anki-integration/pending-youtube-media-queue'; const log = createLogger('anki').child('integration'); @@ -950,6 +953,13 @@ export class AnkiIntegration { await this.pendingYoutubeMediaQueue.handleReady(sourceUrl, cachedPath, options); } + async handleYoutubeMediaCacheFailed( + sourceUrl: string, + options?: PendingYoutubeMediaQueueFailedOptions, + ): Promise { + await this.pendingYoutubeMediaQueue.handleFailed(sourceUrl, options); + } + private async generateAudio(context?: SubtitleMiningContext): Promise { const mpvClient = this.mpvClient; if (!mpvClient || !mpvClient.currentVideoPath) { @@ -1595,7 +1605,7 @@ export class AnkiIntegration { miscInfoValue?: string; } = {}; - if (this.config.media?.generateAudio && this.mpvClient?.currentVideoPath) { + if (this.config.media?.generateAudio !== false && this.mpvClient?.currentVideoPath) { try { const audioFilename = this.generateAudioFilename(); const audioBuffer = await this.generateAudio(); @@ -1615,7 +1625,7 @@ export class AnkiIntegration { } } - if (this.config.media?.generateImage && this.mpvClient?.currentVideoPath) { + if (this.config.media?.generateImage !== false && this.mpvClient?.currentVideoPath) { try { const animatedLeadInSeconds = noteInfo ? await this.getAnimatedImageLeadInSeconds(noteInfo) diff --git a/src/anki-integration/card-creation.ts b/src/anki-integration/card-creation.ts index 16545840..24e48785 100644 --- a/src/anki-integration/card-creation.ts +++ b/src/anki-integration/card-creation.ts @@ -20,6 +20,14 @@ import type { PendingYoutubeMediaUpdate } from './pending-youtube-media'; const log = createLogger('anki').child('integration.card-creation'); +function shouldGenerateAudio(config: AnkiConnectConfig): boolean { + return config.media?.generateAudio !== false; +} + +function shouldGenerateImage(config: AnkiConnectConfig): boolean { + return config.media?.generateImage !== false; +} + export interface CardCreationNoteInfo { noteId: number; fields: Record; @@ -268,15 +276,18 @@ export class CardCreationService { `Clipboard update: timing range ${rangeStart.toFixed(2)}s - ${rangeEnd.toFixed(2)}s`, ); + const config = this.deps.getConfig(); + const generateAudio = shouldGenerateAudio(config); + const generateImage = shouldGenerateImage(config); const mediaResolverOptions = this.getMediaResolverOptions(); - const audioSourcePath = this.deps.getConfig().media?.generateAudio + const audioSourcePath = generateAudio ? await resolveMediaGenerationInput(mpvClient, 'audio', mediaResolverOptions) : null; - const videoPath = this.deps.getConfig().media?.generateImage + const videoPath = generateImage ? await resolveMediaGenerationInput(mpvClient, 'video', mediaResolverOptions) : null; - if (this.deps.getConfig().media?.generateAudio) { + if (generateAudio) { try { const audioFilename = this.generateAudioFilename(); const audioBuffer = audioSourcePath @@ -304,7 +315,7 @@ export class CardCreationService { } } - if (this.deps.getConfig().media?.generateImage) { + if (generateImage) { try { const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo); const imageFilename = this.generateImageFilename(); @@ -464,7 +475,7 @@ export class CardCreationService { errors.push('audio'); } - if (this.deps.getConfig().media?.generateImage) { + if (shouldGenerateImage(this.deps.getConfig())) { try { const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo); const imageFilename = this.generateImageFilename(); @@ -545,22 +556,23 @@ export class CardCreationService { try { return await this.deps.withUpdateProgress('Creating sentence card', async () => { + const config = this.deps.getConfig(); + const generateAudio = shouldGenerateAudio(config); + const generateImage = shouldGenerateImage(config); const mediaResolverOptions = this.getMediaResolverOptions(); - const videoPath = await resolveMediaGenerationInput( - mpvClient, - 'video', - mediaResolverOptions, - ); - const audioSourcePath = await resolveMediaGenerationInput( - mpvClient, - 'audio', - mediaResolverOptions, - ); + const videoPath = generateImage + ? await resolveMediaGenerationInput(mpvClient, 'video', mediaResolverOptions) + : null; + const audioSourcePath = generateAudio + ? await resolveMediaGenerationInput(mpvClient, 'audio', mediaResolverOptions) + : null; + const missingRequestedMediaInput = + (generateImage && !videoPath) || (generateAudio && !audioSourcePath); const shouldQueuePendingYoutubeMedia = - !videoPath && + missingRequestedMediaInput && this.deps.shouldRequireRemoteMediaCache?.() === true && typeof this.deps.queuePendingYoutubeMediaUpdate === 'function'; - if (!videoPath && !shouldQueuePendingYoutubeMedia) { + if (missingRequestedMediaInput && !shouldQueuePendingYoutubeMedia) { this.deps.showOsdNotification('No video loaded'); return false; } @@ -570,13 +582,13 @@ export class CardCreationService { const sentenceField = sentenceCardConfig.sentenceField; const audioFieldName = sentenceCardConfig.audioField || 'SentenceAudio'; - const translationField = this.deps.getConfig().fields?.translation || 'SelectionText'; + const translationField = config.fields?.translation || 'SelectionText'; let resolvedMiscInfoField: string | null = null; let resolvedSentenceAudioField: string = audioFieldName; fields[sentenceField] = sentence; - const ankiAiConfig = this.deps.getConfig().ai; + const ankiAiConfig = config.ai; const ankiAiEnabled = typeof ankiAiConfig === 'object' && ankiAiConfig !== null ? ankiAiConfig.enabled === true @@ -599,7 +611,7 @@ export class CardCreationService { if (sentenceCardConfig.lapisEnabled || sentenceCardConfig.kikuEnabled) { fields.IsSentenceCard = 'x'; - fields[getConfiguredWordFieldName(this.deps.getConfig())] = sentence; + fields[getConfiguredWordFieldName(config)] = sentence; } const pendingNoteInfo = this.createPendingNoteInfo(fields); @@ -608,7 +620,7 @@ export class CardCreationService { ); const pendingExpressionText = getPreferredWordValueFromExtractedFields( pendingNoteFields, - this.deps.getConfig(), + config, ).trim(); let duplicateNoteIds: number[] = []; if ( @@ -626,7 +638,7 @@ export class CardCreationService { } } - const deck = this.deps.getConfig().deck || 'Default'; + const deck = config.deck || 'Default'; let noteId: number; try { noteId = await this.deps.client.addNote( @@ -672,7 +684,7 @@ export class CardCreationService { this.deps.resolveNoteFieldName(createdNoteInfo, audioFieldName) || audioFieldName; resolvedMiscInfoField = this.deps.resolveConfiguredFieldName( createdNoteInfo, - this.deps.getConfig().fields?.miscInfo, + config.fields?.miscInfo, ); const cardTypeFields: Record = {}; @@ -699,54 +711,58 @@ export class CardCreationService { endTime, label, audioFieldName: resolvedSentenceAudioField, - imageFieldName: this.deps.getConfig().fields?.image, + imageFieldName: config.fields?.image, miscInfoFieldName: resolvedMiscInfoField ?? undefined, - generateAudio: this.deps.getConfig().media?.generateAudio !== false, - generateImage: this.deps.getConfig().media?.generateImage !== false, + generateAudio, + generateImage, }); await this.deps.showNotification(noteId, label, 'media queued'); return true; } - if (!videoPath) { + if (missingRequestedMediaInput) { return false; } const mediaFields: Record = {}; - try { - const audioFilename = this.generateAudioFilename(); - const audioBuffer = audioSourcePath - ? await this.mediaGenerateAudio(audioSourcePath, startTime, endTime) - : null; + if (generateAudio) { + try { + const audioFilename = this.generateAudioFilename(); + const audioBuffer = audioSourcePath + ? await this.mediaGenerateAudio(audioSourcePath, startTime, endTime) + : null; - if (audioBuffer) { - await this.deps.client.storeMediaFile(audioFilename, audioBuffer); - const audioValue = `[sound:${audioFilename}]`; - mediaFields[resolvedSentenceAudioField] = audioValue; - miscInfoFilename = audioFilename; + if (audioBuffer) { + await this.deps.client.storeMediaFile(audioFilename, audioBuffer); + const audioValue = `[sound:${audioFilename}]`; + mediaFields[resolvedSentenceAudioField] = audioValue; + miscInfoFilename = audioFilename; + } + } catch (error) { + log.error('Failed to generate sentence audio:', (error as Error).message); + errors.push('audio'); } - } catch (error) { - log.error('Failed to generate sentence audio:', (error as Error).message); - errors.push('audio'); } - try { - const imageFilename = this.generateImageFilename(); - const imageBuffer = await this.generateImageBuffer(videoPath, startTime, endTime); + if (generateImage) { + try { + const imageFilename = this.generateImageFilename(); + const imageBuffer = await this.generateImageBuffer(videoPath!, startTime, endTime); - const imageField = this.deps.getConfig().fields?.image; - if (imageBuffer && imageField) { - await this.deps.client.storeMediaFile(imageFilename, imageBuffer); - mediaFields[imageField] = ``; - miscInfoFilename = imageFilename; + const imageField = config.fields?.image; + if (imageBuffer && imageField) { + await this.deps.client.storeMediaFile(imageFilename, imageBuffer); + mediaFields[imageField] = ``; + miscInfoFilename = imageFilename; + } + } catch (error) { + log.error('Failed to generate sentence image:', (error as Error).message); + errors.push('image'); } - } catch (error) { - log.error('Failed to generate sentence image:', (error as Error).message); - errors.push('image'); } - if (this.deps.getConfig().fields?.miscInfo) { + if (config.fields?.miscInfo) { const miscInfo = this.deps.formatMiscInfoPattern(miscInfoFilename || '', startTime); if (miscInfo && resolvedMiscInfoField) { mediaFields[resolvedMiscInfoField] = miscInfo; diff --git a/src/anki-integration/note-update-workflow.ts b/src/anki-integration/note-update-workflow.ts index 7b95cbb4..e8a762e7 100644 --- a/src/anki-integration/note-update-workflow.ts +++ b/src/anki-integration/note-update-workflow.ts @@ -234,9 +234,10 @@ export class NoteUpdateWorkflow { } } + const generateAudio = config.media?.generateAudio !== false; + const generateImage = config.media?.generateImage !== false; const mediaCacheQueued = - (config.media?.generateAudio || config.media?.generateImage) && - this.deps.queuePendingYoutubeMediaUpdate + (generateAudio || generateImage) && this.deps.queuePendingYoutubeMediaUpdate ? await this.deps.queuePendingYoutubeMediaUpdate({ noteId, noteInfo, @@ -245,7 +246,7 @@ export class NoteUpdateWorkflow { }) : false; - if (!mediaCacheQueued && config.media?.generateAudio) { + if (!mediaCacheQueued && generateAudio) { try { const audioFilename = this.deps.generateAudioFilename(); const audioBuffer = await this.deps.generateAudio(subtitleMiningContext ?? undefined); @@ -270,7 +271,7 @@ export class NoteUpdateWorkflow { } } - if (!mediaCacheQueued && config.media?.generateImage) { + if (!mediaCacheQueued && generateImage) { try { const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo); const imageFilename = this.deps.generateImageFilename(); diff --git a/src/anki-integration/pending-youtube-media-queue.test.ts b/src/anki-integration/pending-youtube-media-queue.test.ts index fb726f56..f2cca786 100644 --- a/src/anki-integration/pending-youtube-media-queue.test.ts +++ b/src/anki-integration/pending-youtube-media-queue.test.ts @@ -2,7 +2,10 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import type { AnkiConnectConfig } from '../types/anki'; -import { PendingYoutubeMediaQueue, type PendingYoutubeMediaQueueDeps } from './pending-youtube-media-queue'; +import { + PendingYoutubeMediaQueue, + type PendingYoutubeMediaQueueDeps, +} from './pending-youtube-media-queue'; function createDeps( overrides: Partial = {}, @@ -70,3 +73,121 @@ test('PendingYoutubeMediaQueue treats cache lookup failures as an immediate gene ], ]); }); + +test('PendingYoutubeMediaQueue drains matching queued jobs when the cache download fails', async () => { + const statusMessages: string[] = []; + const notifications: Array<{ noteId: number; label: string | number; suffix?: string }> = []; + const updatedNotes: number[] = []; + const deps = createDeps({ + client: { + notesInfo: async (noteIds) => + noteIds.map((noteId) => ({ + noteId, + fields: { + SentenceAudio: { value: '' }, + Picture: { value: '' }, + }, + })), + updateNoteFields: async (noteId) => { + updatedNotes.push(noteId); + }, + storeMediaFile: async () => {}, + }, + showStatusNotification: (message) => { + statusMessages.push(message); + }, + showNotification: async (noteId, label, suffix) => { + notifications.push({ noteId, label, suffix }); + }, + }); + const queue = new PendingYoutubeMediaQueue(deps); + + queue.enqueue({ + sourceUrl: 'https://www.youtube.com/watch?v=abc123', + noteId: 42, + startTime: 1, + endTime: 2, + label: 'queued', + generateAudio: true, + generateImage: true, + }); + + await queue.handleFailed('https://youtu.be/abc123'); + await queue.handleReady('https://youtu.be/abc123', '/tmp/media.mkv'); + + assert.deepEqual(updatedNotes, []); + assert.deepEqual(notifications, [{ noteId: 42, label: 'queued', suffix: 'media cache failed' }]); + assert.equal( + statusMessages.includes('YouTube media cache failed. Media was not added to 1 queued card.'), + true, + ); +}); + +test('PendingYoutubeMediaQueue defaults missing media flags to enabled when queuing from notes', async () => { + const updatedNotes: Array<{ noteId: number; fields: Record }> = []; + const storedMedia: string[] = []; + const deps = createDeps({ + client: { + notesInfo: async (noteIds) => + noteIds.map((noteId) => ({ + noteId, + fields: { + SentenceAudio: { value: '' }, + Picture: { value: '' }, + }, + })), + updateNoteFields: async (noteId, fields) => { + updatedNotes.push({ noteId, fields }); + }, + storeMediaFile: async (filename) => { + storedMedia.push(filename); + }, + }, + getConfig: () => ({ media: {}, fields: { image: 'Picture' } }) as AnkiConnectConfig, + }); + const queue = new PendingYoutubeMediaQueue(deps); + + const queued = await queue.queueFromNote({ + noteId: 42, + noteInfo: { noteId: 42, fields: {} }, + label: 'demo', + }); + await queue.handleReady('https://youtu.be/abc123', '/tmp/media.mkv'); + + assert.equal(queued, true); + assert.equal(updatedNotes.length, 1); + assert.equal(storedMedia.length, 2); + assert.match(updatedNotes[0]?.fields.SentenceAudio ?? '', /^\[sound:audio\.mp3\]$/); + assert.match(updatedNotes[0]?.fields.Picture ?? '', /^$/); +}); + +test('PendingYoutubeMediaQueue only announces a download once per source while jobs collect', () => { + const statusMessages: string[] = []; + const deps = createDeps({ + showStatusNotification: (message) => { + statusMessages.push(message); + }, + }); + const queue = new PendingYoutubeMediaQueue(deps); + + queue.enqueue({ + sourceUrl: 'https://youtu.be/abc123', + noteId: 1, + startTime: 1, + endTime: 2, + label: 'first', + generateAudio: true, + generateImage: false, + }); + queue.enqueue({ + sourceUrl: 'https://www.youtube.com/watch?v=abc123', + noteId: 2, + startTime: 3, + endTime: 4, + label: 'second', + generateAudio: false, + generateImage: true, + }); + + assert.equal(statusMessages.length, 1); +}); diff --git a/src/anki-integration/pending-youtube-media-queue.ts b/src/anki-integration/pending-youtube-media-queue.ts index 5ec3cb7b..40745d20 100644 --- a/src/anki-integration/pending-youtube-media-queue.ts +++ b/src/anki-integration/pending-youtube-media-queue.ts @@ -12,6 +12,10 @@ export interface PendingYoutubeMediaQueueReadyOptions { notifyNoQueued?: boolean; } +export interface PendingYoutubeMediaQueueFailedOptions { + notifyStatus?: boolean; +} + export interface PendingYoutubeMediaNoteInfo { noteId: number; fields: Record; @@ -65,6 +69,14 @@ function trimToNonEmptyString(value: unknown): string | null { return trimmed.length > 0 ? trimmed : null; } +function shouldGenerateAudio(config: AnkiConnectConfig): boolean { + return config.media?.generateAudio !== false; +} + +function shouldGenerateImage(config: AnkiConnectConfig): boolean { + return config.media?.generateImage !== false; +} + export class PendingYoutubeMediaQueue { private updates: PendingYoutubeMediaUpdate[] = []; @@ -74,11 +86,16 @@ export class PendingYoutubeMediaQueue { if (!job.generateAudio && !job.generateImage) { return; } + const isFirstQueuedForSource = !this.updates.some((existing) => + youtubeMediaUrlsMatch(existing.sourceUrl, job.sourceUrl), + ); this.updates.push(job); this.deps.logInfo('Queued YouTube media update for note:', job.noteId); - this.deps.showStatusNotification( - 'YouTube media cache is still downloading. Card media will be added when the cache is ready.', - ); + if (isFirstQueuedForSource) { + this.deps.showStatusNotification( + 'YouTube media cache is still downloading. Card media will be added when the cache is ready.', + ); + } } async queueFromNote(job: { @@ -124,8 +141,8 @@ export class PendingYoutubeMediaQueue { ) ?? undefined, miscInfoFieldName: this.deps.resolveConfiguredFieldName(job.noteInfo, config.fields?.miscInfo) ?? undefined, - generateAudio: config.media?.generateAudio === true, - generateImage: config.media?.generateImage === true, + generateAudio: shouldGenerateAudio(config), + generateImage: shouldGenerateImage(config), }); return true; } @@ -178,6 +195,39 @@ export class PendingYoutubeMediaQueue { } } + async handleFailed( + sourceUrl: string, + options: PendingYoutubeMediaQueueFailedOptions = {}, + ): Promise { + const jobs = this.takeMatchingUpdates(sourceUrl); + if (jobs.length === 0) { + if (options.notifyStatus !== false) { + this.deps.showStatusNotification('YouTube media cache failed.'); + } + return; + } + + if (options.notifyStatus !== false) { + this.deps.showStatusNotification( + `YouTube media cache failed. Media was not added to ${jobs.length} queued card${ + jobs.length === 1 ? '' : 's' + }.`, + ); + } + this.deps.logWarn('Discarding queued YouTube media updates after cache failure:', jobs.length); + + for (const job of jobs) { + try { + await this.deps.showNotification(job.noteId, job.label, 'media cache failed'); + } catch (error) { + this.deps.logWarn( + 'Failed to show queued YouTube media failure notification:', + error instanceof Error ? error.message : String(error), + ); + } + } + } + private takeMatchingUpdates(sourceUrl: string): PendingYoutubeMediaUpdate[] { const matched: PendingYoutubeMediaUpdate[] = []; const remaining: PendingYoutubeMediaUpdate[] = []; diff --git a/src/core/services/youtube/media-cache.test.ts b/src/core/services/youtube/media-cache.test.ts index 74593214..c9a4c700 100644 --- a/src/core/services/youtube/media-cache.test.ts +++ b/src/core/services/youtube/media-cache.test.ts @@ -82,6 +82,10 @@ test('YouTube media cache exposes the downloaded file after the background job c assert.equal(spawnCalls.length, 1); assert.equal(spawnCalls[0]?.command, 'yt-dlp'); assert.ok(spawnCalls[0]?.args.includes('--no-playlist')); + assert.ok(spawnCalls[0]?.args.includes('--force-ipv4')); + assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--retries') + 1], '5'); + assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--fragment-retries') + 1], '5'); + assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--extractor-retries') + 1], '5'); assert.ok(spawnCalls[0]?.args.includes('--merge-output-format')); assert.equal( spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-f') + 1], @@ -107,6 +111,68 @@ test('YouTube media cache exposes the downloaded file after the background job c } }); +test('YouTube media cache reports failed background downloads', async () => { + const cacheRoot = makeTempCacheRoot(); + const spawnedProcesses: FakeYtDlpProcess[] = []; + const failedEvents: Array<{ url: string }> = []; + + try { + const cache = createYoutubeMediaCacheService({ + cacheRoot, + getYtDlpCommand: () => 'yt-dlp', + onFailed: (event) => { + failedEvents.push(event); + }, + spawn: () => { + const proc = new FakeYtDlpProcess(); + spawnedProcesses.push(proc); + return proc; + }, + }); + + cache.start('https://youtu.be/demo', { mode: 'background' }); + spawnedProcesses[0]?.emit('close', 1); + + await new Promise((resolve) => setImmediate(resolve)); + + assert.deepEqual(failedEvents, [{ url: 'https://youtu.be/demo' }]); + assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), null); + } finally { + fs.rmSync(cacheRoot, { recursive: true, force: true }); + } +}); + +test('YouTube media cache only reports a failed download once when error is followed by close', async () => { + const cacheRoot = makeTempCacheRoot(); + const spawnedProcesses: FakeYtDlpProcess[] = []; + const failedEvents: Array<{ url: string }> = []; + + try { + const cache = createYoutubeMediaCacheService({ + cacheRoot, + getYtDlpCommand: () => 'yt-dlp', + onFailed: (event) => { + failedEvents.push(event); + }, + spawn: () => { + const proc = new FakeYtDlpProcess(); + spawnedProcesses.push(proc); + return proc; + }, + }); + + cache.start('https://youtu.be/demo', { mode: 'background' }); + spawnedProcesses[0]?.emit('error', new Error('spawn failed')); + spawnedProcesses[0]?.emit('close', 1); + + await new Promise((resolve) => setImmediate(resolve)); + + assert.deepEqual(failedEvents, [{ url: 'https://youtu.be/demo' }]); + } finally { + fs.rmSync(cacheRoot, { recursive: true, force: true }); + } +}); + test('YouTube media cache can disable the download height cap', () => { const cacheRoot = makeTempCacheRoot(); const spawnCalls: SpawnCall[] = []; @@ -159,6 +225,31 @@ test('YouTube media cache applies the configured download height cap', () => { } }); +test('YouTube media cache removes stale files from previous runs on startup', () => { + const cacheRoot = makeTempCacheRoot(); + const staleDir = path.join(cacheRoot, 'stale-session'); + const spawnCalls: SpawnCall[] = []; + + try { + fs.mkdirSync(staleDir, { recursive: true }); + fs.writeFileSync(path.join(staleDir, 'media.mkv'), 'stale cached media'); + + createYoutubeMediaCacheService({ + cacheRoot, + getYtDlpCommand: () => 'yt-dlp', + spawn: (command, args) => { + spawnCalls.push({ command, args }); + return new FakeYtDlpProcess(); + }, + }); + + assert.equal(fs.existsSync(staleDir), false); + assert.deepEqual(spawnCalls, []); + } finally { + fs.rmSync(cacheRoot, { recursive: true, force: true }); + } +}); + test('YouTube media cache restarts when a ready cached file was deleted externally', async () => { const cacheRoot = makeTempCacheRoot(); const spawnedProcesses: FakeYtDlpProcess[] = []; @@ -198,6 +289,32 @@ test('YouTube media cache restarts when a ready cached file was deleted external } }); +test('YouTube media cache removes stale disk siblings before starting a new cache', () => { + const cacheRoot = makeTempCacheRoot(); + const staleDir = path.join(cacheRoot, 'stale-sibling'); + const spawnCalls: Array<{ command: string; args: string[] }> = []; + + try { + const cache = createYoutubeMediaCacheService({ + cacheRoot, + getYtDlpCommand: () => 'yt-dlp', + spawn: (command, args) => { + spawnCalls.push({ command, args }); + return new FakeYtDlpProcess(); + }, + }); + fs.mkdirSync(staleDir, { recursive: true }); + fs.writeFileSync(path.join(staleDir, 'media.mkv'), 'stale cached media'); + + cache.start('https://youtu.be/demo', { mode: 'background' }); + + assert.equal(fs.existsSync(staleDir), false); + assert.equal(spawnCalls.length, 1); + } finally { + fs.rmSync(cacheRoot, { recursive: true, force: true }); + } +}); + test('YouTube media cache drops old sessions when a new background cache starts', async () => { const cacheRoot = makeTempCacheRoot(); const spawnedProcesses: FakeYtDlpProcess[] = []; diff --git a/src/core/services/youtube/media-cache.ts b/src/core/services/youtube/media-cache.ts index 57e37fa2..8453cee2 100644 --- a/src/core/services/youtube/media-cache.ts +++ b/src/core/services/youtube/media-cache.ts @@ -40,6 +40,7 @@ export interface YoutubeMediaCacheServiceDeps { spawn?: SpawnProcess; onDownloadStarted?: (event: { url: string }) => void; onReady?: (event: { url: string; path: string }) => void; + onFailed?: (event: { url: string }) => void; logInfo?: (message: string) => void; logWarn?: (message: string) => void; } @@ -88,6 +89,13 @@ function createYtDlpArgs(url: string, outputTemplate: string, maxHeight?: number return [ '--no-playlist', '--no-warnings', + '--force-ipv4', + '--retries', + '5', + '--fragment-retries', + '5', + '--extractor-retries', + '5', '-f', getFormatSelector(normalizeMaxHeight(maxHeight)), '--merge-output-format', @@ -109,6 +117,29 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep let activeKey: string | null = null; const getSessionDir = (url: string): string => path.join(cacheRoot, cacheKeyForUrl(url)); + const removeCacheDir = (dir: string): void => { + try { + fs.rmSync(dir, { recursive: true, force: true }); + } catch { + // Temp cache cleanup should not block shutdown or playback startup. + } + }; + const removeCacheRootEntriesExcept = (dirsToKeep: string[]): void => { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(cacheRoot, { withFileTypes: true }); + } catch { + return; + } + const keepDirs = new Set(dirsToKeep.map((dir) => path.resolve(dir))); + for (const entry of entries) { + const entryPath = path.join(cacheRoot, entry.name); + if (keepDirs.has(path.resolve(entryPath))) { + continue; + } + removeCacheDir(entryPath); + } + }; const removeSession = (key: string): void => { const session = sessions.get(key); if (!session) { @@ -121,11 +152,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep if (activeKey === key) { activeKey = null; } - try { - fs.rmSync(session.dir, { recursive: true, force: true }); - } catch { - // Temp cache cleanup should not block shutdown or playback startup. - } + removeCacheDir(session.dir); }; const removeInactiveSessions = (keyToKeep: string): void => { for (const key of [...sessions.keys()]) { @@ -134,6 +161,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep } } }; + removeCacheRootEntriesExcept([]); const getCachedMediaPath = async (url: string): Promise => { const key = cacheKeyForUrl(url); @@ -172,9 +200,11 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep const key = cacheKeyForUrl(url); activeKey = key; + const dir = getSessionDir(url); const existingSession = sessions.get(key); if (existingSession?.state === 'running') { removeInactiveSessions(key); + removeCacheRootEntriesExcept([existingSession.dir]); return; } if (existingSession) { @@ -184,14 +214,15 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep findReadyMediaPath(existingSession.dir)) ) { removeInactiveSessions(key); + removeCacheRootEntriesExcept([existingSession.dir]); return; } removeSession(key); activeKey = key; } removeInactiveSessions(key); + removeCacheRootEntriesExcept([dir]); - const dir = getSessionDir(url); fs.mkdirSync(dir, { recursive: true }); const outputTemplate = path.join(dir, 'media.%(ext)s'); const args = createYtDlpArgs(url, outputTemplate, options.maxHeight); @@ -219,6 +250,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep error instanceof Error ? error.message : String(error) }`, ); + deps.onFailed?.({ url }); }); child.once('close', (code) => { @@ -226,6 +258,9 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep if (currentSession !== session) { return; } + if (session.state === 'failed') { + return; + } session.process = null; if (code === 0) { const readyPath = findReadyMediaPath(dir); @@ -239,6 +274,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep } session.state = 'failed'; deps.logWarn?.(`YouTube media cache download exited without a usable media file.`); + deps.onFailed?.({ url }); }); }; diff --git a/src/main.ts b/src/main.ts index 1dcfe6fb..5fe7480e 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1200,6 +1200,23 @@ const youtubeMediaCache = createYoutubeMediaCacheService({ ); }); }, + onFailed: (event) => { + showConfiguredStatusNotification('YouTube media cache failed.', { + id: 'youtube-media-cache-status', + title: 'YouTube media cache', + variant: 'error', + persistent: false, + }); + void appState.ankiIntegration + ?.handleYoutubeMediaCacheFailed(event.url, { notifyStatus: false }) + .catch((error) => { + logger.warn( + `Failed to drain queued YouTube media updates after cache failure: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + }); + }, logInfo: (message) => logger.info(message), logWarn: (message) => logger.warn(message), }); diff --git a/src/media-generator.test.ts b/src/media-generator.test.ts index ebcbd60f..1bbe9116 100644 --- a/src/media-generator.test.ts +++ b/src/media-generator.test.ts @@ -212,6 +212,8 @@ test('generateAudio adds remote input options before the ffmpeg input', async () assert.ok(args.indexOf('-reconnect') < inputIndex); assert.equal(args[args.indexOf('-reconnect') + 1], '1'); assert.equal(args[args.indexOf('-reconnect_streamed') + 1], '1'); + assert.equal(args[args.indexOf('-reconnect_on_network_error') + 1], '1'); + assert.equal(args[args.indexOf('-reconnect_on_http_error') + 1], '403,5xx'); assert.equal(args[args.indexOf('-reconnect_delay_max') + 1], '5'); assert.equal(args[args.indexOf('-user_agent') + 1], 'Mozilla/5.0'); assert.equal( diff --git a/src/media-input.ts b/src/media-input.ts index 95916217..52349e19 100644 --- a/src/media-input.ts +++ b/src/media-input.ts @@ -65,7 +65,18 @@ export function normalizeMediaInput(input: MediaInput): NormalizedMediaInput { const inputArgs: string[] = []; if (input.inputOptions?.reconnect) { - inputArgs.push('-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5'); + inputArgs.push( + '-reconnect', + '1', + '-reconnect_streamed', + '1', + '-reconnect_on_network_error', + '1', + '-reconnect_on_http_error', + '403,5xx', + '-reconnect_delay_max', + '5', + ); } const userAgent = trimToNonEmptyString(input.inputOptions?.userAgent);