mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-27 16:49:51 -07:00
feat(youtube): harden media cache and drain queued cards on failure
- Add --force-ipv4 and retry flags to yt-dlp background downloads - Clean stale cache dirs on startup and before new downloads - Notify and drain pending card updates when cache download fails - Fix generateAudio/generateImage config checks (missing = enabled)
This commit is contained in:
@@ -3,3 +3,5 @@ area: youtube
|
||||
|
||||
- Improved YouTube card media generation by sending safer ffmpeg request options for resolved streams and skipping stale stream maps, including cached YouTube files.
|
||||
- Added `youtube.mediaCache.mode` with `direct` and `background` modes so YouTube card audio/image extraction can optionally use a background yt-dlp media cache when direct stream extraction is unreliable; background mode now announces cache download start/readiness through queued overlay/OSD notifications, creates text-only cards while the cache downloads, queues media updates for the mined note IDs, fills audio/image fields once the cached file is ready, and caps background downloads at `youtube.mediaCache.maxHeight` 720p by default.
|
||||
- Cleaned stale YouTube background media cache files on startup and before new background downloads so only the active cached media file remains.
|
||||
- Hardened YouTube media cache downloads with IPv4/extractor retry flags and made failed background downloads notify users and clear queued media updates instead of leaving them pending silently.
|
||||
|
||||
@@ -1535,7 +1535,9 @@ Set defaults used by managed subtitle auto-selection and the `subminer` launcher
|
||||
| `mediaCache.mode` | `direct` \| `background` | YouTube card audio/image extraction mode (default `direct`) |
|
||||
| `mediaCache.maxHeight` | number | Maximum background cache download height. Set `0` for unlimited (default `720`) |
|
||||
|
||||
`mediaCache.mode: "direct"` extracts card media from the active YouTube stream URL. `mediaCache.mode: "background"` starts a separate yt-dlp media download after YouTube playback has loaded. Playback and subtitle loading do not wait for that download. Background cache downloads are capped by `mediaCache.maxHeight`, which defaults to 720p; set it to `0` to let yt-dlp choose the best available height. SubMiner announces when the background cache download starts and when the cache is ready, using the configured notification surface; overlay and OSD messages queue until the overlay or mpv is ready. If you mine cards before the cache is ready, SubMiner creates the text fields immediately, queues the audio/image work for those note IDs, shows a status notification, and fills the media fields once the cached file is ready.
|
||||
`mediaCache.mode: "direct"` extracts card media from the active YouTube stream URL. `mediaCache.mode: "background"` starts a separate yt-dlp media download after YouTube playback has loaded. Playback and subtitle loading do not wait for that download. Use background mode if direct card media generation hits YouTube `403` errors from expiring stream URLs.
|
||||
|
||||
Background cache downloads are capped by `mediaCache.maxHeight`, which defaults to 720p; set it to `0` to let yt-dlp choose the best available height. Downloads use IPv4 and yt-dlp retry flags to reduce YouTube throttling failures. SubMiner announces when the background cache download starts and when the cache is ready, using the configured notification surface; overlay and OSD messages queue until the overlay or mpv is ready. If you mine cards before the cache is ready, SubMiner creates the text fields immediately, queues the audio/image work for those note IDs, shows a status notification, and fills the media fields once the cached file is ready. If the cache download fails, SubMiner shows a failure notification, shows queued-card failure notifications, and clears the pending updates.
|
||||
|
||||
Current launcher behavior:
|
||||
|
||||
|
||||
@@ -88,6 +88,12 @@ SubMiner handles several YouTube subtitle formats transparently:
|
||||
|
||||
For auto-generated tracks, SubMiner prefers `srv3` > `srv2` > `srv1` > `vtt` (TimedText XML produces cleaner output). For manual tracks, `srt` > `vtt` is preferred.
|
||||
|
||||
## Card Media Cache
|
||||
|
||||
By default, YouTube card audio and screenshots are extracted directly from mpv's active stream URLs. If generated card media fails with YouTube `403` errors, set `youtube.mediaCache.mode` to `"background"`. Background mode starts a separate `yt-dlp` media download after playback loads, creates text fields immediately, queues audio/image work for mined notes, and fills those fields once the local cache file is ready.
|
||||
|
||||
Background cache downloads use IPv4 and retry flags to reduce YouTube throttling failures. If the background download still fails, SubMiner shows a cache failure notification, shows queued-card failure notifications, and clears those pending updates so cards are not left waiting silently.
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
### Primary Subtitle Languages
|
||||
@@ -131,6 +137,7 @@ Precedence: CLI flag > environment variable > `config.jsonc` > built-in default.
|
||||
- **No subtitles found**: The video may not have Japanese subtitles. Open the picker with `Ctrl+Alt+C` to see all available tracks.
|
||||
- **yt-dlp not found**: Install `yt-dlp` and ensure it is on `PATH`, or set `SUBMINER_YTDLP_BIN` to the binary path.
|
||||
- **Probe timeout**: `yt-dlp` has a 15-second timeout per operation. Slow connections or rate-limited IPs may hit this. Retry or update `yt-dlp`.
|
||||
- **Card media `403` errors**: Switch `youtube.mediaCache.mode` from `"direct"` to `"background"` so card media is generated from a local `yt-dlp` cache instead of ffmpeg reading an expiring YouTube stream URL.
|
||||
- **Auto-caption quality**: YouTube auto-generated captions vary in quality. Manual subtitles (when available) are always preferred.
|
||||
- **`ytsearch:` targets**: `subminer ytsearch:"keyword"` plays the first search result. Subtitle availability depends on the matched video.
|
||||
- **Secondary subtitle fails**: Secondary track failures never block playback. The primary subtitle loads independently.
|
||||
|
||||
+13
-3
@@ -69,7 +69,10 @@ import {
|
||||
} from './anki-integration/media-source';
|
||||
import type { PendingYoutubeMediaUpdate } from './anki-integration/pending-youtube-media';
|
||||
import { PendingYoutubeMediaQueue } from './anki-integration/pending-youtube-media-queue';
|
||||
import type { PendingYoutubeMediaQueueReadyOptions } from './anki-integration/pending-youtube-media-queue';
|
||||
import type {
|
||||
PendingYoutubeMediaQueueFailedOptions,
|
||||
PendingYoutubeMediaQueueReadyOptions,
|
||||
} from './anki-integration/pending-youtube-media-queue';
|
||||
|
||||
const log = createLogger('anki').child('integration');
|
||||
|
||||
@@ -950,6 +953,13 @@ export class AnkiIntegration {
|
||||
await this.pendingYoutubeMediaQueue.handleReady(sourceUrl, cachedPath, options);
|
||||
}
|
||||
|
||||
async handleYoutubeMediaCacheFailed(
|
||||
sourceUrl: string,
|
||||
options?: PendingYoutubeMediaQueueFailedOptions,
|
||||
): Promise<void> {
|
||||
await this.pendingYoutubeMediaQueue.handleFailed(sourceUrl, options);
|
||||
}
|
||||
|
||||
private async generateAudio(context?: SubtitleMiningContext): Promise<Buffer | null> {
|
||||
const mpvClient = this.mpvClient;
|
||||
if (!mpvClient || !mpvClient.currentVideoPath) {
|
||||
@@ -1595,7 +1605,7 @@ export class AnkiIntegration {
|
||||
miscInfoValue?: string;
|
||||
} = {};
|
||||
|
||||
if (this.config.media?.generateAudio && this.mpvClient?.currentVideoPath) {
|
||||
if (this.config.media?.generateAudio !== false && this.mpvClient?.currentVideoPath) {
|
||||
try {
|
||||
const audioFilename = this.generateAudioFilename();
|
||||
const audioBuffer = await this.generateAudio();
|
||||
@@ -1615,7 +1625,7 @@ export class AnkiIntegration {
|
||||
}
|
||||
}
|
||||
|
||||
if (this.config.media?.generateImage && this.mpvClient?.currentVideoPath) {
|
||||
if (this.config.media?.generateImage !== false && this.mpvClient?.currentVideoPath) {
|
||||
try {
|
||||
const animatedLeadInSeconds = noteInfo
|
||||
? await this.getAnimatedImageLeadInSeconds(noteInfo)
|
||||
|
||||
@@ -20,6 +20,14 @@ import type { PendingYoutubeMediaUpdate } from './pending-youtube-media';
|
||||
|
||||
const log = createLogger('anki').child('integration.card-creation');
|
||||
|
||||
function shouldGenerateAudio(config: AnkiConnectConfig): boolean {
|
||||
return config.media?.generateAudio !== false;
|
||||
}
|
||||
|
||||
function shouldGenerateImage(config: AnkiConnectConfig): boolean {
|
||||
return config.media?.generateImage !== false;
|
||||
}
|
||||
|
||||
export interface CardCreationNoteInfo {
|
||||
noteId: number;
|
||||
fields: Record<string, { value: string }>;
|
||||
@@ -268,15 +276,18 @@ export class CardCreationService {
|
||||
`Clipboard update: timing range ${rangeStart.toFixed(2)}s - ${rangeEnd.toFixed(2)}s`,
|
||||
);
|
||||
|
||||
const config = this.deps.getConfig();
|
||||
const generateAudio = shouldGenerateAudio(config);
|
||||
const generateImage = shouldGenerateImage(config);
|
||||
const mediaResolverOptions = this.getMediaResolverOptions();
|
||||
const audioSourcePath = this.deps.getConfig().media?.generateAudio
|
||||
const audioSourcePath = generateAudio
|
||||
? await resolveMediaGenerationInput(mpvClient, 'audio', mediaResolverOptions)
|
||||
: null;
|
||||
const videoPath = this.deps.getConfig().media?.generateImage
|
||||
const videoPath = generateImage
|
||||
? await resolveMediaGenerationInput(mpvClient, 'video', mediaResolverOptions)
|
||||
: null;
|
||||
|
||||
if (this.deps.getConfig().media?.generateAudio) {
|
||||
if (generateAudio) {
|
||||
try {
|
||||
const audioFilename = this.generateAudioFilename();
|
||||
const audioBuffer = audioSourcePath
|
||||
@@ -304,7 +315,7 @@ export class CardCreationService {
|
||||
}
|
||||
}
|
||||
|
||||
if (this.deps.getConfig().media?.generateImage) {
|
||||
if (generateImage) {
|
||||
try {
|
||||
const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo);
|
||||
const imageFilename = this.generateImageFilename();
|
||||
@@ -464,7 +475,7 @@ export class CardCreationService {
|
||||
errors.push('audio');
|
||||
}
|
||||
|
||||
if (this.deps.getConfig().media?.generateImage) {
|
||||
if (shouldGenerateImage(this.deps.getConfig())) {
|
||||
try {
|
||||
const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo);
|
||||
const imageFilename = this.generateImageFilename();
|
||||
@@ -545,22 +556,23 @@ export class CardCreationService {
|
||||
|
||||
try {
|
||||
return await this.deps.withUpdateProgress('Creating sentence card', async () => {
|
||||
const config = this.deps.getConfig();
|
||||
const generateAudio = shouldGenerateAudio(config);
|
||||
const generateImage = shouldGenerateImage(config);
|
||||
const mediaResolverOptions = this.getMediaResolverOptions();
|
||||
const videoPath = await resolveMediaGenerationInput(
|
||||
mpvClient,
|
||||
'video',
|
||||
mediaResolverOptions,
|
||||
);
|
||||
const audioSourcePath = await resolveMediaGenerationInput(
|
||||
mpvClient,
|
||||
'audio',
|
||||
mediaResolverOptions,
|
||||
);
|
||||
const videoPath = generateImage
|
||||
? await resolveMediaGenerationInput(mpvClient, 'video', mediaResolverOptions)
|
||||
: null;
|
||||
const audioSourcePath = generateAudio
|
||||
? await resolveMediaGenerationInput(mpvClient, 'audio', mediaResolverOptions)
|
||||
: null;
|
||||
const missingRequestedMediaInput =
|
||||
(generateImage && !videoPath) || (generateAudio && !audioSourcePath);
|
||||
const shouldQueuePendingYoutubeMedia =
|
||||
!videoPath &&
|
||||
missingRequestedMediaInput &&
|
||||
this.deps.shouldRequireRemoteMediaCache?.() === true &&
|
||||
typeof this.deps.queuePendingYoutubeMediaUpdate === 'function';
|
||||
if (!videoPath && !shouldQueuePendingYoutubeMedia) {
|
||||
if (missingRequestedMediaInput && !shouldQueuePendingYoutubeMedia) {
|
||||
this.deps.showOsdNotification('No video loaded');
|
||||
return false;
|
||||
}
|
||||
@@ -570,13 +582,13 @@ export class CardCreationService {
|
||||
|
||||
const sentenceField = sentenceCardConfig.sentenceField;
|
||||
const audioFieldName = sentenceCardConfig.audioField || 'SentenceAudio';
|
||||
const translationField = this.deps.getConfig().fields?.translation || 'SelectionText';
|
||||
const translationField = config.fields?.translation || 'SelectionText';
|
||||
let resolvedMiscInfoField: string | null = null;
|
||||
let resolvedSentenceAudioField: string = audioFieldName;
|
||||
|
||||
fields[sentenceField] = sentence;
|
||||
|
||||
const ankiAiConfig = this.deps.getConfig().ai;
|
||||
const ankiAiConfig = config.ai;
|
||||
const ankiAiEnabled =
|
||||
typeof ankiAiConfig === 'object' && ankiAiConfig !== null
|
||||
? ankiAiConfig.enabled === true
|
||||
@@ -599,7 +611,7 @@ export class CardCreationService {
|
||||
|
||||
if (sentenceCardConfig.lapisEnabled || sentenceCardConfig.kikuEnabled) {
|
||||
fields.IsSentenceCard = 'x';
|
||||
fields[getConfiguredWordFieldName(this.deps.getConfig())] = sentence;
|
||||
fields[getConfiguredWordFieldName(config)] = sentence;
|
||||
}
|
||||
|
||||
const pendingNoteInfo = this.createPendingNoteInfo(fields);
|
||||
@@ -608,7 +620,7 @@ export class CardCreationService {
|
||||
);
|
||||
const pendingExpressionText = getPreferredWordValueFromExtractedFields(
|
||||
pendingNoteFields,
|
||||
this.deps.getConfig(),
|
||||
config,
|
||||
).trim();
|
||||
let duplicateNoteIds: number[] = [];
|
||||
if (
|
||||
@@ -626,7 +638,7 @@ export class CardCreationService {
|
||||
}
|
||||
}
|
||||
|
||||
const deck = this.deps.getConfig().deck || 'Default';
|
||||
const deck = config.deck || 'Default';
|
||||
let noteId: number;
|
||||
try {
|
||||
noteId = await this.deps.client.addNote(
|
||||
@@ -672,7 +684,7 @@ export class CardCreationService {
|
||||
this.deps.resolveNoteFieldName(createdNoteInfo, audioFieldName) || audioFieldName;
|
||||
resolvedMiscInfoField = this.deps.resolveConfiguredFieldName(
|
||||
createdNoteInfo,
|
||||
this.deps.getConfig().fields?.miscInfo,
|
||||
config.fields?.miscInfo,
|
||||
);
|
||||
|
||||
const cardTypeFields: Record<string, string> = {};
|
||||
@@ -699,54 +711,58 @@ export class CardCreationService {
|
||||
endTime,
|
||||
label,
|
||||
audioFieldName: resolvedSentenceAudioField,
|
||||
imageFieldName: this.deps.getConfig().fields?.image,
|
||||
imageFieldName: config.fields?.image,
|
||||
miscInfoFieldName: resolvedMiscInfoField ?? undefined,
|
||||
generateAudio: this.deps.getConfig().media?.generateAudio !== false,
|
||||
generateImage: this.deps.getConfig().media?.generateImage !== false,
|
||||
generateAudio,
|
||||
generateImage,
|
||||
});
|
||||
await this.deps.showNotification(noteId, label, 'media queued');
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!videoPath) {
|
||||
if (missingRequestedMediaInput) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const mediaFields: Record<string, string> = {};
|
||||
|
||||
try {
|
||||
const audioFilename = this.generateAudioFilename();
|
||||
const audioBuffer = audioSourcePath
|
||||
? await this.mediaGenerateAudio(audioSourcePath, startTime, endTime)
|
||||
: null;
|
||||
if (generateAudio) {
|
||||
try {
|
||||
const audioFilename = this.generateAudioFilename();
|
||||
const audioBuffer = audioSourcePath
|
||||
? await this.mediaGenerateAudio(audioSourcePath, startTime, endTime)
|
||||
: null;
|
||||
|
||||
if (audioBuffer) {
|
||||
await this.deps.client.storeMediaFile(audioFilename, audioBuffer);
|
||||
const audioValue = `[sound:${audioFilename}]`;
|
||||
mediaFields[resolvedSentenceAudioField] = audioValue;
|
||||
miscInfoFilename = audioFilename;
|
||||
if (audioBuffer) {
|
||||
await this.deps.client.storeMediaFile(audioFilename, audioBuffer);
|
||||
const audioValue = `[sound:${audioFilename}]`;
|
||||
mediaFields[resolvedSentenceAudioField] = audioValue;
|
||||
miscInfoFilename = audioFilename;
|
||||
}
|
||||
} catch (error) {
|
||||
log.error('Failed to generate sentence audio:', (error as Error).message);
|
||||
errors.push('audio');
|
||||
}
|
||||
} catch (error) {
|
||||
log.error('Failed to generate sentence audio:', (error as Error).message);
|
||||
errors.push('audio');
|
||||
}
|
||||
|
||||
try {
|
||||
const imageFilename = this.generateImageFilename();
|
||||
const imageBuffer = await this.generateImageBuffer(videoPath, startTime, endTime);
|
||||
if (generateImage) {
|
||||
try {
|
||||
const imageFilename = this.generateImageFilename();
|
||||
const imageBuffer = await this.generateImageBuffer(videoPath!, startTime, endTime);
|
||||
|
||||
const imageField = this.deps.getConfig().fields?.image;
|
||||
if (imageBuffer && imageField) {
|
||||
await this.deps.client.storeMediaFile(imageFilename, imageBuffer);
|
||||
mediaFields[imageField] = `<img src="${imageFilename}">`;
|
||||
miscInfoFilename = imageFilename;
|
||||
const imageField = config.fields?.image;
|
||||
if (imageBuffer && imageField) {
|
||||
await this.deps.client.storeMediaFile(imageFilename, imageBuffer);
|
||||
mediaFields[imageField] = `<img src="${imageFilename}">`;
|
||||
miscInfoFilename = imageFilename;
|
||||
}
|
||||
} catch (error) {
|
||||
log.error('Failed to generate sentence image:', (error as Error).message);
|
||||
errors.push('image');
|
||||
}
|
||||
} catch (error) {
|
||||
log.error('Failed to generate sentence image:', (error as Error).message);
|
||||
errors.push('image');
|
||||
}
|
||||
|
||||
if (this.deps.getConfig().fields?.miscInfo) {
|
||||
if (config.fields?.miscInfo) {
|
||||
const miscInfo = this.deps.formatMiscInfoPattern(miscInfoFilename || '', startTime);
|
||||
if (miscInfo && resolvedMiscInfoField) {
|
||||
mediaFields[resolvedMiscInfoField] = miscInfo;
|
||||
|
||||
@@ -234,9 +234,10 @@ export class NoteUpdateWorkflow {
|
||||
}
|
||||
}
|
||||
|
||||
const generateAudio = config.media?.generateAudio !== false;
|
||||
const generateImage = config.media?.generateImage !== false;
|
||||
const mediaCacheQueued =
|
||||
(config.media?.generateAudio || config.media?.generateImage) &&
|
||||
this.deps.queuePendingYoutubeMediaUpdate
|
||||
(generateAudio || generateImage) && this.deps.queuePendingYoutubeMediaUpdate
|
||||
? await this.deps.queuePendingYoutubeMediaUpdate({
|
||||
noteId,
|
||||
noteInfo,
|
||||
@@ -245,7 +246,7 @@ export class NoteUpdateWorkflow {
|
||||
})
|
||||
: false;
|
||||
|
||||
if (!mediaCacheQueued && config.media?.generateAudio) {
|
||||
if (!mediaCacheQueued && generateAudio) {
|
||||
try {
|
||||
const audioFilename = this.deps.generateAudioFilename();
|
||||
const audioBuffer = await this.deps.generateAudio(subtitleMiningContext ?? undefined);
|
||||
@@ -270,7 +271,7 @@ export class NoteUpdateWorkflow {
|
||||
}
|
||||
}
|
||||
|
||||
if (!mediaCacheQueued && config.media?.generateImage) {
|
||||
if (!mediaCacheQueued && generateImage) {
|
||||
try {
|
||||
const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo);
|
||||
const imageFilename = this.deps.generateImageFilename();
|
||||
|
||||
@@ -2,7 +2,10 @@ import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
|
||||
import type { AnkiConnectConfig } from '../types/anki';
|
||||
import { PendingYoutubeMediaQueue, type PendingYoutubeMediaQueueDeps } from './pending-youtube-media-queue';
|
||||
import {
|
||||
PendingYoutubeMediaQueue,
|
||||
type PendingYoutubeMediaQueueDeps,
|
||||
} from './pending-youtube-media-queue';
|
||||
|
||||
function createDeps(
|
||||
overrides: Partial<PendingYoutubeMediaQueueDeps> = {},
|
||||
@@ -70,3 +73,121 @@ test('PendingYoutubeMediaQueue treats cache lookup failures as an immediate gene
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
test('PendingYoutubeMediaQueue drains matching queued jobs when the cache download fails', async () => {
|
||||
const statusMessages: string[] = [];
|
||||
const notifications: Array<{ noteId: number; label: string | number; suffix?: string }> = [];
|
||||
const updatedNotes: number[] = [];
|
||||
const deps = createDeps({
|
||||
client: {
|
||||
notesInfo: async (noteIds) =>
|
||||
noteIds.map((noteId) => ({
|
||||
noteId,
|
||||
fields: {
|
||||
SentenceAudio: { value: '' },
|
||||
Picture: { value: '' },
|
||||
},
|
||||
})),
|
||||
updateNoteFields: async (noteId) => {
|
||||
updatedNotes.push(noteId);
|
||||
},
|
||||
storeMediaFile: async () => {},
|
||||
},
|
||||
showStatusNotification: (message) => {
|
||||
statusMessages.push(message);
|
||||
},
|
||||
showNotification: async (noteId, label, suffix) => {
|
||||
notifications.push({ noteId, label, suffix });
|
||||
},
|
||||
});
|
||||
const queue = new PendingYoutubeMediaQueue(deps);
|
||||
|
||||
queue.enqueue({
|
||||
sourceUrl: 'https://www.youtube.com/watch?v=abc123',
|
||||
noteId: 42,
|
||||
startTime: 1,
|
||||
endTime: 2,
|
||||
label: 'queued',
|
||||
generateAudio: true,
|
||||
generateImage: true,
|
||||
});
|
||||
|
||||
await queue.handleFailed('https://youtu.be/abc123');
|
||||
await queue.handleReady('https://youtu.be/abc123', '/tmp/media.mkv');
|
||||
|
||||
assert.deepEqual(updatedNotes, []);
|
||||
assert.deepEqual(notifications, [{ noteId: 42, label: 'queued', suffix: 'media cache failed' }]);
|
||||
assert.equal(
|
||||
statusMessages.includes('YouTube media cache failed. Media was not added to 1 queued card.'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('PendingYoutubeMediaQueue defaults missing media flags to enabled when queuing from notes', async () => {
|
||||
const updatedNotes: Array<{ noteId: number; fields: Record<string, string> }> = [];
|
||||
const storedMedia: string[] = [];
|
||||
const deps = createDeps({
|
||||
client: {
|
||||
notesInfo: async (noteIds) =>
|
||||
noteIds.map((noteId) => ({
|
||||
noteId,
|
||||
fields: {
|
||||
SentenceAudio: { value: '' },
|
||||
Picture: { value: '' },
|
||||
},
|
||||
})),
|
||||
updateNoteFields: async (noteId, fields) => {
|
||||
updatedNotes.push({ noteId, fields });
|
||||
},
|
||||
storeMediaFile: async (filename) => {
|
||||
storedMedia.push(filename);
|
||||
},
|
||||
},
|
||||
getConfig: () => ({ media: {}, fields: { image: 'Picture' } }) as AnkiConnectConfig,
|
||||
});
|
||||
const queue = new PendingYoutubeMediaQueue(deps);
|
||||
|
||||
const queued = await queue.queueFromNote({
|
||||
noteId: 42,
|
||||
noteInfo: { noteId: 42, fields: {} },
|
||||
label: 'demo',
|
||||
});
|
||||
await queue.handleReady('https://youtu.be/abc123', '/tmp/media.mkv');
|
||||
|
||||
assert.equal(queued, true);
|
||||
assert.equal(updatedNotes.length, 1);
|
||||
assert.equal(storedMedia.length, 2);
|
||||
assert.match(updatedNotes[0]?.fields.SentenceAudio ?? '', /^\[sound:audio\.mp3\]$/);
|
||||
assert.match(updatedNotes[0]?.fields.Picture ?? '', /^<img src="image\.webp">$/);
|
||||
});
|
||||
|
||||
test('PendingYoutubeMediaQueue only announces a download once per source while jobs collect', () => {
|
||||
const statusMessages: string[] = [];
|
||||
const deps = createDeps({
|
||||
showStatusNotification: (message) => {
|
||||
statusMessages.push(message);
|
||||
},
|
||||
});
|
||||
const queue = new PendingYoutubeMediaQueue(deps);
|
||||
|
||||
queue.enqueue({
|
||||
sourceUrl: 'https://youtu.be/abc123',
|
||||
noteId: 1,
|
||||
startTime: 1,
|
||||
endTime: 2,
|
||||
label: 'first',
|
||||
generateAudio: true,
|
||||
generateImage: false,
|
||||
});
|
||||
queue.enqueue({
|
||||
sourceUrl: 'https://www.youtube.com/watch?v=abc123',
|
||||
noteId: 2,
|
||||
startTime: 3,
|
||||
endTime: 4,
|
||||
label: 'second',
|
||||
generateAudio: false,
|
||||
generateImage: true,
|
||||
});
|
||||
|
||||
assert.equal(statusMessages.length, 1);
|
||||
});
|
||||
|
||||
@@ -12,6 +12,10 @@ export interface PendingYoutubeMediaQueueReadyOptions {
|
||||
notifyNoQueued?: boolean;
|
||||
}
|
||||
|
||||
export interface PendingYoutubeMediaQueueFailedOptions {
|
||||
notifyStatus?: boolean;
|
||||
}
|
||||
|
||||
export interface PendingYoutubeMediaNoteInfo {
|
||||
noteId: number;
|
||||
fields: Record<string, { value: string }>;
|
||||
@@ -65,6 +69,14 @@ function trimToNonEmptyString(value: unknown): string | null {
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
|
||||
function shouldGenerateAudio(config: AnkiConnectConfig): boolean {
|
||||
return config.media?.generateAudio !== false;
|
||||
}
|
||||
|
||||
function shouldGenerateImage(config: AnkiConnectConfig): boolean {
|
||||
return config.media?.generateImage !== false;
|
||||
}
|
||||
|
||||
export class PendingYoutubeMediaQueue {
|
||||
private updates: PendingYoutubeMediaUpdate[] = [];
|
||||
|
||||
@@ -74,11 +86,16 @@ export class PendingYoutubeMediaQueue {
|
||||
if (!job.generateAudio && !job.generateImage) {
|
||||
return;
|
||||
}
|
||||
const isFirstQueuedForSource = !this.updates.some((existing) =>
|
||||
youtubeMediaUrlsMatch(existing.sourceUrl, job.sourceUrl),
|
||||
);
|
||||
this.updates.push(job);
|
||||
this.deps.logInfo('Queued YouTube media update for note:', job.noteId);
|
||||
this.deps.showStatusNotification(
|
||||
'YouTube media cache is still downloading. Card media will be added when the cache is ready.',
|
||||
);
|
||||
if (isFirstQueuedForSource) {
|
||||
this.deps.showStatusNotification(
|
||||
'YouTube media cache is still downloading. Card media will be added when the cache is ready.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async queueFromNote(job: {
|
||||
@@ -124,8 +141,8 @@ export class PendingYoutubeMediaQueue {
|
||||
) ?? undefined,
|
||||
miscInfoFieldName:
|
||||
this.deps.resolveConfiguredFieldName(job.noteInfo, config.fields?.miscInfo) ?? undefined,
|
||||
generateAudio: config.media?.generateAudio === true,
|
||||
generateImage: config.media?.generateImage === true,
|
||||
generateAudio: shouldGenerateAudio(config),
|
||||
generateImage: shouldGenerateImage(config),
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -178,6 +195,39 @@ export class PendingYoutubeMediaQueue {
|
||||
}
|
||||
}
|
||||
|
||||
async handleFailed(
|
||||
sourceUrl: string,
|
||||
options: PendingYoutubeMediaQueueFailedOptions = {},
|
||||
): Promise<void> {
|
||||
const jobs = this.takeMatchingUpdates(sourceUrl);
|
||||
if (jobs.length === 0) {
|
||||
if (options.notifyStatus !== false) {
|
||||
this.deps.showStatusNotification('YouTube media cache failed.');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.notifyStatus !== false) {
|
||||
this.deps.showStatusNotification(
|
||||
`YouTube media cache failed. Media was not added to ${jobs.length} queued card${
|
||||
jobs.length === 1 ? '' : 's'
|
||||
}.`,
|
||||
);
|
||||
}
|
||||
this.deps.logWarn('Discarding queued YouTube media updates after cache failure:', jobs.length);
|
||||
|
||||
for (const job of jobs) {
|
||||
try {
|
||||
await this.deps.showNotification(job.noteId, job.label, 'media cache failed');
|
||||
} catch (error) {
|
||||
this.deps.logWarn(
|
||||
'Failed to show queued YouTube media failure notification:',
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private takeMatchingUpdates(sourceUrl: string): PendingYoutubeMediaUpdate[] {
|
||||
const matched: PendingYoutubeMediaUpdate[] = [];
|
||||
const remaining: PendingYoutubeMediaUpdate[] = [];
|
||||
|
||||
@@ -82,6 +82,10 @@ test('YouTube media cache exposes the downloaded file after the background job c
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
assert.equal(spawnCalls[0]?.command, 'yt-dlp');
|
||||
assert.ok(spawnCalls[0]?.args.includes('--no-playlist'));
|
||||
assert.ok(spawnCalls[0]?.args.includes('--force-ipv4'));
|
||||
assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--retries') + 1], '5');
|
||||
assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--fragment-retries') + 1], '5');
|
||||
assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--extractor-retries') + 1], '5');
|
||||
assert.ok(spawnCalls[0]?.args.includes('--merge-output-format'));
|
||||
assert.equal(
|
||||
spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-f') + 1],
|
||||
@@ -107,6 +111,68 @@ test('YouTube media cache exposes the downloaded file after the background job c
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache reports failed background downloads', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const failedEvents: Array<{ url: string }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
onFailed: (event) => {
|
||||
failedEvents.push(event);
|
||||
},
|
||||
spawn: () => {
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
spawnedProcesses[0]?.emit('close', 1);
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
assert.deepEqual(failedEvents, [{ url: 'https://youtu.be/demo' }]);
|
||||
assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), null);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache only reports a failed download once when error is followed by close', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
const failedEvents: Array<{ url: string }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
onFailed: (event) => {
|
||||
failedEvents.push(event);
|
||||
},
|
||||
spawn: () => {
|
||||
const proc = new FakeYtDlpProcess();
|
||||
spawnedProcesses.push(proc);
|
||||
return proc;
|
||||
},
|
||||
});
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
spawnedProcesses[0]?.emit('error', new Error('spawn failed'));
|
||||
spawnedProcesses[0]?.emit('close', 1);
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
assert.deepEqual(failedEvents, [{ url: 'https://youtu.be/demo' }]);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache can disable the download height cap', () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
@@ -159,6 +225,31 @@ test('YouTube media cache applies the configured download height cap', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache removes stale files from previous runs on startup', () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const staleDir = path.join(cacheRoot, 'stale-session');
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
|
||||
try {
|
||||
fs.mkdirSync(staleDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(staleDir, 'media.mkv'), 'stale cached media');
|
||||
|
||||
createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args) => {
|
||||
spawnCalls.push({ command, args });
|
||||
return new FakeYtDlpProcess();
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(fs.existsSync(staleDir), false);
|
||||
assert.deepEqual(spawnCalls, []);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache restarts when a ready cached file was deleted externally', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
@@ -198,6 +289,32 @@ test('YouTube media cache restarts when a ready cached file was deleted external
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache removes stale disk siblings before starting a new cache', () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const staleDir = path.join(cacheRoot, 'stale-sibling');
|
||||
const spawnCalls: Array<{ command: string; args: string[] }> = [];
|
||||
|
||||
try {
|
||||
const cache = createYoutubeMediaCacheService({
|
||||
cacheRoot,
|
||||
getYtDlpCommand: () => 'yt-dlp',
|
||||
spawn: (command, args) => {
|
||||
spawnCalls.push({ command, args });
|
||||
return new FakeYtDlpProcess();
|
||||
},
|
||||
});
|
||||
fs.mkdirSync(staleDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(staleDir, 'media.mkv'), 'stale cached media');
|
||||
|
||||
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||
|
||||
assert.equal(fs.existsSync(staleDir), false);
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('YouTube media cache drops old sessions when a new background cache starts', async () => {
|
||||
const cacheRoot = makeTempCacheRoot();
|
||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||
|
||||
@@ -40,6 +40,7 @@ export interface YoutubeMediaCacheServiceDeps {
|
||||
spawn?: SpawnProcess;
|
||||
onDownloadStarted?: (event: { url: string }) => void;
|
||||
onReady?: (event: { url: string; path: string }) => void;
|
||||
onFailed?: (event: { url: string }) => void;
|
||||
logInfo?: (message: string) => void;
|
||||
logWarn?: (message: string) => void;
|
||||
}
|
||||
@@ -88,6 +89,13 @@ function createYtDlpArgs(url: string, outputTemplate: string, maxHeight?: number
|
||||
return [
|
||||
'--no-playlist',
|
||||
'--no-warnings',
|
||||
'--force-ipv4',
|
||||
'--retries',
|
||||
'5',
|
||||
'--fragment-retries',
|
||||
'5',
|
||||
'--extractor-retries',
|
||||
'5',
|
||||
'-f',
|
||||
getFormatSelector(normalizeMaxHeight(maxHeight)),
|
||||
'--merge-output-format',
|
||||
@@ -109,6 +117,29 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
let activeKey: string | null = null;
|
||||
|
||||
const getSessionDir = (url: string): string => path.join(cacheRoot, cacheKeyForUrl(url));
|
||||
const removeCacheDir = (dir: string): void => {
|
||||
try {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Temp cache cleanup should not block shutdown or playback startup.
|
||||
}
|
||||
};
|
||||
const removeCacheRootEntriesExcept = (dirsToKeep: string[]): void => {
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = fs.readdirSync(cacheRoot, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const keepDirs = new Set(dirsToKeep.map((dir) => path.resolve(dir)));
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(cacheRoot, entry.name);
|
||||
if (keepDirs.has(path.resolve(entryPath))) {
|
||||
continue;
|
||||
}
|
||||
removeCacheDir(entryPath);
|
||||
}
|
||||
};
|
||||
const removeSession = (key: string): void => {
|
||||
const session = sessions.get(key);
|
||||
if (!session) {
|
||||
@@ -121,11 +152,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
if (activeKey === key) {
|
||||
activeKey = null;
|
||||
}
|
||||
try {
|
||||
fs.rmSync(session.dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Temp cache cleanup should not block shutdown or playback startup.
|
||||
}
|
||||
removeCacheDir(session.dir);
|
||||
};
|
||||
const removeInactiveSessions = (keyToKeep: string): void => {
|
||||
for (const key of [...sessions.keys()]) {
|
||||
@@ -134,6 +161,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
}
|
||||
}
|
||||
};
|
||||
removeCacheRootEntriesExcept([]);
|
||||
|
||||
const getCachedMediaPath = async (url: string): Promise<string | null> => {
|
||||
const key = cacheKeyForUrl(url);
|
||||
@@ -172,9 +200,11 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
|
||||
const key = cacheKeyForUrl(url);
|
||||
activeKey = key;
|
||||
const dir = getSessionDir(url);
|
||||
const existingSession = sessions.get(key);
|
||||
if (existingSession?.state === 'running') {
|
||||
removeInactiveSessions(key);
|
||||
removeCacheRootEntriesExcept([existingSession.dir]);
|
||||
return;
|
||||
}
|
||||
if (existingSession) {
|
||||
@@ -184,14 +214,15 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
findReadyMediaPath(existingSession.dir))
|
||||
) {
|
||||
removeInactiveSessions(key);
|
||||
removeCacheRootEntriesExcept([existingSession.dir]);
|
||||
return;
|
||||
}
|
||||
removeSession(key);
|
||||
activeKey = key;
|
||||
}
|
||||
removeInactiveSessions(key);
|
||||
removeCacheRootEntriesExcept([dir]);
|
||||
|
||||
const dir = getSessionDir(url);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const outputTemplate = path.join(dir, 'media.%(ext)s');
|
||||
const args = createYtDlpArgs(url, outputTemplate, options.maxHeight);
|
||||
@@ -219,6 +250,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
deps.onFailed?.({ url });
|
||||
});
|
||||
|
||||
child.once('close', (code) => {
|
||||
@@ -226,6 +258,9 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
if (currentSession !== session) {
|
||||
return;
|
||||
}
|
||||
if (session.state === 'failed') {
|
||||
return;
|
||||
}
|
||||
session.process = null;
|
||||
if (code === 0) {
|
||||
const readyPath = findReadyMediaPath(dir);
|
||||
@@ -239,6 +274,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
||||
}
|
||||
session.state = 'failed';
|
||||
deps.logWarn?.(`YouTube media cache download exited without a usable media file.`);
|
||||
deps.onFailed?.({ url });
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
+17
@@ -1200,6 +1200,23 @@ const youtubeMediaCache = createYoutubeMediaCacheService({
|
||||
);
|
||||
});
|
||||
},
|
||||
onFailed: (event) => {
|
||||
showConfiguredStatusNotification('YouTube media cache failed.', {
|
||||
id: 'youtube-media-cache-status',
|
||||
title: 'YouTube media cache',
|
||||
variant: 'error',
|
||||
persistent: false,
|
||||
});
|
||||
void appState.ankiIntegration
|
||||
?.handleYoutubeMediaCacheFailed(event.url, { notifyStatus: false })
|
||||
.catch((error) => {
|
||||
logger.warn(
|
||||
`Failed to drain queued YouTube media updates after cache failure: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
});
|
||||
},
|
||||
logInfo: (message) => logger.info(message),
|
||||
logWarn: (message) => logger.warn(message),
|
||||
});
|
||||
|
||||
@@ -212,6 +212,8 @@ test('generateAudio adds remote input options before the ffmpeg input', async ()
|
||||
assert.ok(args.indexOf('-reconnect') < inputIndex);
|
||||
assert.equal(args[args.indexOf('-reconnect') + 1], '1');
|
||||
assert.equal(args[args.indexOf('-reconnect_streamed') + 1], '1');
|
||||
assert.equal(args[args.indexOf('-reconnect_on_network_error') + 1], '1');
|
||||
assert.equal(args[args.indexOf('-reconnect_on_http_error') + 1], '403,5xx');
|
||||
assert.equal(args[args.indexOf('-reconnect_delay_max') + 1], '5');
|
||||
assert.equal(args[args.indexOf('-user_agent') + 1], 'Mozilla/5.0');
|
||||
assert.equal(
|
||||
|
||||
+12
-1
@@ -65,7 +65,18 @@ export function normalizeMediaInput(input: MediaInput): NormalizedMediaInput {
|
||||
|
||||
const inputArgs: string[] = [];
|
||||
if (input.inputOptions?.reconnect) {
|
||||
inputArgs.push('-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5');
|
||||
inputArgs.push(
|
||||
'-reconnect',
|
||||
'1',
|
||||
'-reconnect_streamed',
|
||||
'1',
|
||||
'-reconnect_on_network_error',
|
||||
'1',
|
||||
'-reconnect_on_http_error',
|
||||
'403,5xx',
|
||||
'-reconnect_delay_max',
|
||||
'5',
|
||||
);
|
||||
}
|
||||
|
||||
const userAgent = trimToNonEmptyString(input.inputOptions?.userAgent);
|
||||
|
||||
Reference in New Issue
Block a user