mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-29 01:01:34 -07:00
feat(youtube): harden media cache and drain queued cards on failure
- Add --force-ipv4 and retry flags to yt-dlp background downloads - Clean stale cache dirs on startup and before new downloads - Notify and drain pending card updates when cache download fails - Fix generateAudio/generateImage config checks (missing = enabled)
This commit is contained in:
@@ -3,3 +3,5 @@ area: youtube
|
|||||||
|
|
||||||
- Improved YouTube card media generation by sending safer ffmpeg request options for resolved streams and skipping stale stream maps, including cached YouTube files.
|
- Improved YouTube card media generation by sending safer ffmpeg request options for resolved streams and skipping stale stream maps, including cached YouTube files.
|
||||||
- Added `youtube.mediaCache.mode` with `direct` and `background` modes so YouTube card audio/image extraction can optionally use a background yt-dlp media cache when direct stream extraction is unreliable; background mode now announces cache download start/readiness through queued overlay/OSD notifications, creates text-only cards while the cache downloads, queues media updates for the mined note IDs, fills audio/image fields once the cached file is ready, and caps background downloads at `youtube.mediaCache.maxHeight` 720p by default.
|
- Added `youtube.mediaCache.mode` with `direct` and `background` modes so YouTube card audio/image extraction can optionally use a background yt-dlp media cache when direct stream extraction is unreliable; background mode now announces cache download start/readiness through queued overlay/OSD notifications, creates text-only cards while the cache downloads, queues media updates for the mined note IDs, fills audio/image fields once the cached file is ready, and caps background downloads at `youtube.mediaCache.maxHeight` 720p by default.
|
||||||
|
- Cleaned stale YouTube background media cache files on startup and before new background downloads so only the active cached media file remains.
|
||||||
|
- Hardened YouTube media cache downloads with IPv4/extractor retry flags and made failed background downloads notify users and clear queued media updates instead of leaving them pending silently.
|
||||||
|
|||||||
@@ -1535,7 +1535,9 @@ Set defaults used by managed subtitle auto-selection and the `subminer` launcher
|
|||||||
| `mediaCache.mode` | `direct` \| `background` | YouTube card audio/image extraction mode (default `direct`) |
|
| `mediaCache.mode` | `direct` \| `background` | YouTube card audio/image extraction mode (default `direct`) |
|
||||||
| `mediaCache.maxHeight` | number | Maximum background cache download height. Set `0` for unlimited (default `720`) |
|
| `mediaCache.maxHeight` | number | Maximum background cache download height. Set `0` for unlimited (default `720`) |
|
||||||
|
|
||||||
`mediaCache.mode: "direct"` extracts card media from the active YouTube stream URL. `mediaCache.mode: "background"` starts a separate yt-dlp media download after YouTube playback has loaded. Playback and subtitle loading do not wait for that download. Background cache downloads are capped by `mediaCache.maxHeight`, which defaults to 720p; set it to `0` to let yt-dlp choose the best available height. SubMiner announces when the background cache download starts and when the cache is ready, using the configured notification surface; overlay and OSD messages queue until the overlay or mpv is ready. If you mine cards before the cache is ready, SubMiner creates the text fields immediately, queues the audio/image work for those note IDs, shows a status notification, and fills the media fields once the cached file is ready.
|
`mediaCache.mode: "direct"` extracts card media from the active YouTube stream URL. `mediaCache.mode: "background"` starts a separate yt-dlp media download after YouTube playback has loaded. Playback and subtitle loading do not wait for that download. Use background mode if direct card media generation hits YouTube `403` errors from expiring stream URLs.
|
||||||
|
|
||||||
|
Background cache downloads are capped by `mediaCache.maxHeight`, which defaults to 720p; set it to `0` to let yt-dlp choose the best available height. Downloads use IPv4 and yt-dlp retry flags to reduce YouTube throttling failures. SubMiner announces when the background cache download starts and when the cache is ready, using the configured notification surface; overlay and OSD messages queue until the overlay or mpv is ready. If you mine cards before the cache is ready, SubMiner creates the text fields immediately, queues the audio/image work for those note IDs, shows a status notification, and fills the media fields once the cached file is ready. If the cache download fails, SubMiner shows a failure notification, shows queued-card failure notifications, and clears the pending updates.
|
||||||
|
|
||||||
Current launcher behavior:
|
Current launcher behavior:
|
||||||
|
|
||||||
|
|||||||
@@ -88,6 +88,12 @@ SubMiner handles several YouTube subtitle formats transparently:
|
|||||||
|
|
||||||
For auto-generated tracks, SubMiner prefers `srv3` > `srv2` > `srv1` > `vtt` (TimedText XML produces cleaner output). For manual tracks, `srt` > `vtt` is preferred.
|
For auto-generated tracks, SubMiner prefers `srv3` > `srv2` > `srv1` > `vtt` (TimedText XML produces cleaner output). For manual tracks, `srt` > `vtt` is preferred.
|
||||||
|
|
||||||
|
## Card Media Cache
|
||||||
|
|
||||||
|
By default, YouTube card audio and screenshots are extracted directly from mpv's active stream URLs. If generated card media fails with YouTube `403` errors, set `youtube.mediaCache.mode` to `"background"`. Background mode starts a separate `yt-dlp` media download after playback loads, creates text fields immediately, queues audio/image work for mined notes, and fills those fields once the local cache file is ready.
|
||||||
|
|
||||||
|
Background cache downloads use IPv4 and retry flags to reduce YouTube throttling failures. If the background download still fails, SubMiner shows a cache failure notification, shows queued-card failure notifications, and clears those pending updates so cards are not left waiting silently.
|
||||||
|
|
||||||
## Configuration Reference
|
## Configuration Reference
|
||||||
|
|
||||||
### Primary Subtitle Languages
|
### Primary Subtitle Languages
|
||||||
@@ -131,6 +137,7 @@ Precedence: CLI flag > environment variable > `config.jsonc` > built-in default.
|
|||||||
- **No subtitles found**: The video may not have Japanese subtitles. Open the picker with `Ctrl+Alt+C` to see all available tracks.
|
- **No subtitles found**: The video may not have Japanese subtitles. Open the picker with `Ctrl+Alt+C` to see all available tracks.
|
||||||
- **yt-dlp not found**: Install `yt-dlp` and ensure it is on `PATH`, or set `SUBMINER_YTDLP_BIN` to the binary path.
|
- **yt-dlp not found**: Install `yt-dlp` and ensure it is on `PATH`, or set `SUBMINER_YTDLP_BIN` to the binary path.
|
||||||
- **Probe timeout**: `yt-dlp` has a 15-second timeout per operation. Slow connections or rate-limited IPs may hit this. Retry or update `yt-dlp`.
|
- **Probe timeout**: `yt-dlp` has a 15-second timeout per operation. Slow connections or rate-limited IPs may hit this. Retry or update `yt-dlp`.
|
||||||
|
- **Card media `403` errors**: Switch `youtube.mediaCache.mode` from `"direct"` to `"background"` so card media is generated from a local `yt-dlp` cache instead of ffmpeg reading an expiring YouTube stream URL.
|
||||||
- **Auto-caption quality**: YouTube auto-generated captions vary in quality. Manual subtitles (when available) are always preferred.
|
- **Auto-caption quality**: YouTube auto-generated captions vary in quality. Manual subtitles (when available) are always preferred.
|
||||||
- **`ytsearch:` targets**: `subminer ytsearch:"keyword"` plays the first search result. Subtitle availability depends on the matched video.
|
- **`ytsearch:` targets**: `subminer ytsearch:"keyword"` plays the first search result. Subtitle availability depends on the matched video.
|
||||||
- **Secondary subtitle fails**: Secondary track failures never block playback. The primary subtitle loads independently.
|
- **Secondary subtitle fails**: Secondary track failures never block playback. The primary subtitle loads independently.
|
||||||
|
|||||||
+13
-3
@@ -69,7 +69,10 @@ import {
|
|||||||
} from './anki-integration/media-source';
|
} from './anki-integration/media-source';
|
||||||
import type { PendingYoutubeMediaUpdate } from './anki-integration/pending-youtube-media';
|
import type { PendingYoutubeMediaUpdate } from './anki-integration/pending-youtube-media';
|
||||||
import { PendingYoutubeMediaQueue } from './anki-integration/pending-youtube-media-queue';
|
import { PendingYoutubeMediaQueue } from './anki-integration/pending-youtube-media-queue';
|
||||||
import type { PendingYoutubeMediaQueueReadyOptions } from './anki-integration/pending-youtube-media-queue';
|
import type {
|
||||||
|
PendingYoutubeMediaQueueFailedOptions,
|
||||||
|
PendingYoutubeMediaQueueReadyOptions,
|
||||||
|
} from './anki-integration/pending-youtube-media-queue';
|
||||||
|
|
||||||
const log = createLogger('anki').child('integration');
|
const log = createLogger('anki').child('integration');
|
||||||
|
|
||||||
@@ -950,6 +953,13 @@ export class AnkiIntegration {
|
|||||||
await this.pendingYoutubeMediaQueue.handleReady(sourceUrl, cachedPath, options);
|
await this.pendingYoutubeMediaQueue.handleReady(sourceUrl, cachedPath, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async handleYoutubeMediaCacheFailed(
|
||||||
|
sourceUrl: string,
|
||||||
|
options?: PendingYoutubeMediaQueueFailedOptions,
|
||||||
|
): Promise<void> {
|
||||||
|
await this.pendingYoutubeMediaQueue.handleFailed(sourceUrl, options);
|
||||||
|
}
|
||||||
|
|
||||||
private async generateAudio(context?: SubtitleMiningContext): Promise<Buffer | null> {
|
private async generateAudio(context?: SubtitleMiningContext): Promise<Buffer | null> {
|
||||||
const mpvClient = this.mpvClient;
|
const mpvClient = this.mpvClient;
|
||||||
if (!mpvClient || !mpvClient.currentVideoPath) {
|
if (!mpvClient || !mpvClient.currentVideoPath) {
|
||||||
@@ -1595,7 +1605,7 @@ export class AnkiIntegration {
|
|||||||
miscInfoValue?: string;
|
miscInfoValue?: string;
|
||||||
} = {};
|
} = {};
|
||||||
|
|
||||||
if (this.config.media?.generateAudio && this.mpvClient?.currentVideoPath) {
|
if (this.config.media?.generateAudio !== false && this.mpvClient?.currentVideoPath) {
|
||||||
try {
|
try {
|
||||||
const audioFilename = this.generateAudioFilename();
|
const audioFilename = this.generateAudioFilename();
|
||||||
const audioBuffer = await this.generateAudio();
|
const audioBuffer = await this.generateAudio();
|
||||||
@@ -1615,7 +1625,7 @@ export class AnkiIntegration {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.config.media?.generateImage && this.mpvClient?.currentVideoPath) {
|
if (this.config.media?.generateImage !== false && this.mpvClient?.currentVideoPath) {
|
||||||
try {
|
try {
|
||||||
const animatedLeadInSeconds = noteInfo
|
const animatedLeadInSeconds = noteInfo
|
||||||
? await this.getAnimatedImageLeadInSeconds(noteInfo)
|
? await this.getAnimatedImageLeadInSeconds(noteInfo)
|
||||||
|
|||||||
@@ -20,6 +20,14 @@ import type { PendingYoutubeMediaUpdate } from './pending-youtube-media';
|
|||||||
|
|
||||||
const log = createLogger('anki').child('integration.card-creation');
|
const log = createLogger('anki').child('integration.card-creation');
|
||||||
|
|
||||||
|
function shouldGenerateAudio(config: AnkiConnectConfig): boolean {
|
||||||
|
return config.media?.generateAudio !== false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldGenerateImage(config: AnkiConnectConfig): boolean {
|
||||||
|
return config.media?.generateImage !== false;
|
||||||
|
}
|
||||||
|
|
||||||
export interface CardCreationNoteInfo {
|
export interface CardCreationNoteInfo {
|
||||||
noteId: number;
|
noteId: number;
|
||||||
fields: Record<string, { value: string }>;
|
fields: Record<string, { value: string }>;
|
||||||
@@ -268,15 +276,18 @@ export class CardCreationService {
|
|||||||
`Clipboard update: timing range ${rangeStart.toFixed(2)}s - ${rangeEnd.toFixed(2)}s`,
|
`Clipboard update: timing range ${rangeStart.toFixed(2)}s - ${rangeEnd.toFixed(2)}s`,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const config = this.deps.getConfig();
|
||||||
|
const generateAudio = shouldGenerateAudio(config);
|
||||||
|
const generateImage = shouldGenerateImage(config);
|
||||||
const mediaResolverOptions = this.getMediaResolverOptions();
|
const mediaResolverOptions = this.getMediaResolverOptions();
|
||||||
const audioSourcePath = this.deps.getConfig().media?.generateAudio
|
const audioSourcePath = generateAudio
|
||||||
? await resolveMediaGenerationInput(mpvClient, 'audio', mediaResolverOptions)
|
? await resolveMediaGenerationInput(mpvClient, 'audio', mediaResolverOptions)
|
||||||
: null;
|
: null;
|
||||||
const videoPath = this.deps.getConfig().media?.generateImage
|
const videoPath = generateImage
|
||||||
? await resolveMediaGenerationInput(mpvClient, 'video', mediaResolverOptions)
|
? await resolveMediaGenerationInput(mpvClient, 'video', mediaResolverOptions)
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
if (this.deps.getConfig().media?.generateAudio) {
|
if (generateAudio) {
|
||||||
try {
|
try {
|
||||||
const audioFilename = this.generateAudioFilename();
|
const audioFilename = this.generateAudioFilename();
|
||||||
const audioBuffer = audioSourcePath
|
const audioBuffer = audioSourcePath
|
||||||
@@ -304,7 +315,7 @@ export class CardCreationService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.deps.getConfig().media?.generateImage) {
|
if (generateImage) {
|
||||||
try {
|
try {
|
||||||
const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo);
|
const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo);
|
||||||
const imageFilename = this.generateImageFilename();
|
const imageFilename = this.generateImageFilename();
|
||||||
@@ -464,7 +475,7 @@ export class CardCreationService {
|
|||||||
errors.push('audio');
|
errors.push('audio');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.deps.getConfig().media?.generateImage) {
|
if (shouldGenerateImage(this.deps.getConfig())) {
|
||||||
try {
|
try {
|
||||||
const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo);
|
const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo);
|
||||||
const imageFilename = this.generateImageFilename();
|
const imageFilename = this.generateImageFilename();
|
||||||
@@ -545,22 +556,23 @@ export class CardCreationService {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
return await this.deps.withUpdateProgress('Creating sentence card', async () => {
|
return await this.deps.withUpdateProgress('Creating sentence card', async () => {
|
||||||
|
const config = this.deps.getConfig();
|
||||||
|
const generateAudio = shouldGenerateAudio(config);
|
||||||
|
const generateImage = shouldGenerateImage(config);
|
||||||
const mediaResolverOptions = this.getMediaResolverOptions();
|
const mediaResolverOptions = this.getMediaResolverOptions();
|
||||||
const videoPath = await resolveMediaGenerationInput(
|
const videoPath = generateImage
|
||||||
mpvClient,
|
? await resolveMediaGenerationInput(mpvClient, 'video', mediaResolverOptions)
|
||||||
'video',
|
: null;
|
||||||
mediaResolverOptions,
|
const audioSourcePath = generateAudio
|
||||||
);
|
? await resolveMediaGenerationInput(mpvClient, 'audio', mediaResolverOptions)
|
||||||
const audioSourcePath = await resolveMediaGenerationInput(
|
: null;
|
||||||
mpvClient,
|
const missingRequestedMediaInput =
|
||||||
'audio',
|
(generateImage && !videoPath) || (generateAudio && !audioSourcePath);
|
||||||
mediaResolverOptions,
|
|
||||||
);
|
|
||||||
const shouldQueuePendingYoutubeMedia =
|
const shouldQueuePendingYoutubeMedia =
|
||||||
!videoPath &&
|
missingRequestedMediaInput &&
|
||||||
this.deps.shouldRequireRemoteMediaCache?.() === true &&
|
this.deps.shouldRequireRemoteMediaCache?.() === true &&
|
||||||
typeof this.deps.queuePendingYoutubeMediaUpdate === 'function';
|
typeof this.deps.queuePendingYoutubeMediaUpdate === 'function';
|
||||||
if (!videoPath && !shouldQueuePendingYoutubeMedia) {
|
if (missingRequestedMediaInput && !shouldQueuePendingYoutubeMedia) {
|
||||||
this.deps.showOsdNotification('No video loaded');
|
this.deps.showOsdNotification('No video loaded');
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -570,13 +582,13 @@ export class CardCreationService {
|
|||||||
|
|
||||||
const sentenceField = sentenceCardConfig.sentenceField;
|
const sentenceField = sentenceCardConfig.sentenceField;
|
||||||
const audioFieldName = sentenceCardConfig.audioField || 'SentenceAudio';
|
const audioFieldName = sentenceCardConfig.audioField || 'SentenceAudio';
|
||||||
const translationField = this.deps.getConfig().fields?.translation || 'SelectionText';
|
const translationField = config.fields?.translation || 'SelectionText';
|
||||||
let resolvedMiscInfoField: string | null = null;
|
let resolvedMiscInfoField: string | null = null;
|
||||||
let resolvedSentenceAudioField: string = audioFieldName;
|
let resolvedSentenceAudioField: string = audioFieldName;
|
||||||
|
|
||||||
fields[sentenceField] = sentence;
|
fields[sentenceField] = sentence;
|
||||||
|
|
||||||
const ankiAiConfig = this.deps.getConfig().ai;
|
const ankiAiConfig = config.ai;
|
||||||
const ankiAiEnabled =
|
const ankiAiEnabled =
|
||||||
typeof ankiAiConfig === 'object' && ankiAiConfig !== null
|
typeof ankiAiConfig === 'object' && ankiAiConfig !== null
|
||||||
? ankiAiConfig.enabled === true
|
? ankiAiConfig.enabled === true
|
||||||
@@ -599,7 +611,7 @@ export class CardCreationService {
|
|||||||
|
|
||||||
if (sentenceCardConfig.lapisEnabled || sentenceCardConfig.kikuEnabled) {
|
if (sentenceCardConfig.lapisEnabled || sentenceCardConfig.kikuEnabled) {
|
||||||
fields.IsSentenceCard = 'x';
|
fields.IsSentenceCard = 'x';
|
||||||
fields[getConfiguredWordFieldName(this.deps.getConfig())] = sentence;
|
fields[getConfiguredWordFieldName(config)] = sentence;
|
||||||
}
|
}
|
||||||
|
|
||||||
const pendingNoteInfo = this.createPendingNoteInfo(fields);
|
const pendingNoteInfo = this.createPendingNoteInfo(fields);
|
||||||
@@ -608,7 +620,7 @@ export class CardCreationService {
|
|||||||
);
|
);
|
||||||
const pendingExpressionText = getPreferredWordValueFromExtractedFields(
|
const pendingExpressionText = getPreferredWordValueFromExtractedFields(
|
||||||
pendingNoteFields,
|
pendingNoteFields,
|
||||||
this.deps.getConfig(),
|
config,
|
||||||
).trim();
|
).trim();
|
||||||
let duplicateNoteIds: number[] = [];
|
let duplicateNoteIds: number[] = [];
|
||||||
if (
|
if (
|
||||||
@@ -626,7 +638,7 @@ export class CardCreationService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const deck = this.deps.getConfig().deck || 'Default';
|
const deck = config.deck || 'Default';
|
||||||
let noteId: number;
|
let noteId: number;
|
||||||
try {
|
try {
|
||||||
noteId = await this.deps.client.addNote(
|
noteId = await this.deps.client.addNote(
|
||||||
@@ -672,7 +684,7 @@ export class CardCreationService {
|
|||||||
this.deps.resolveNoteFieldName(createdNoteInfo, audioFieldName) || audioFieldName;
|
this.deps.resolveNoteFieldName(createdNoteInfo, audioFieldName) || audioFieldName;
|
||||||
resolvedMiscInfoField = this.deps.resolveConfiguredFieldName(
|
resolvedMiscInfoField = this.deps.resolveConfiguredFieldName(
|
||||||
createdNoteInfo,
|
createdNoteInfo,
|
||||||
this.deps.getConfig().fields?.miscInfo,
|
config.fields?.miscInfo,
|
||||||
);
|
);
|
||||||
|
|
||||||
const cardTypeFields: Record<string, string> = {};
|
const cardTypeFields: Record<string, string> = {};
|
||||||
@@ -699,54 +711,58 @@ export class CardCreationService {
|
|||||||
endTime,
|
endTime,
|
||||||
label,
|
label,
|
||||||
audioFieldName: resolvedSentenceAudioField,
|
audioFieldName: resolvedSentenceAudioField,
|
||||||
imageFieldName: this.deps.getConfig().fields?.image,
|
imageFieldName: config.fields?.image,
|
||||||
miscInfoFieldName: resolvedMiscInfoField ?? undefined,
|
miscInfoFieldName: resolvedMiscInfoField ?? undefined,
|
||||||
generateAudio: this.deps.getConfig().media?.generateAudio !== false,
|
generateAudio,
|
||||||
generateImage: this.deps.getConfig().media?.generateImage !== false,
|
generateImage,
|
||||||
});
|
});
|
||||||
await this.deps.showNotification(noteId, label, 'media queued');
|
await this.deps.showNotification(noteId, label, 'media queued');
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!videoPath) {
|
if (missingRequestedMediaInput) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
const mediaFields: Record<string, string> = {};
|
const mediaFields: Record<string, string> = {};
|
||||||
|
|
||||||
try {
|
if (generateAudio) {
|
||||||
const audioFilename = this.generateAudioFilename();
|
try {
|
||||||
const audioBuffer = audioSourcePath
|
const audioFilename = this.generateAudioFilename();
|
||||||
? await this.mediaGenerateAudio(audioSourcePath, startTime, endTime)
|
const audioBuffer = audioSourcePath
|
||||||
: null;
|
? await this.mediaGenerateAudio(audioSourcePath, startTime, endTime)
|
||||||
|
: null;
|
||||||
|
|
||||||
if (audioBuffer) {
|
if (audioBuffer) {
|
||||||
await this.deps.client.storeMediaFile(audioFilename, audioBuffer);
|
await this.deps.client.storeMediaFile(audioFilename, audioBuffer);
|
||||||
const audioValue = `[sound:${audioFilename}]`;
|
const audioValue = `[sound:${audioFilename}]`;
|
||||||
mediaFields[resolvedSentenceAudioField] = audioValue;
|
mediaFields[resolvedSentenceAudioField] = audioValue;
|
||||||
miscInfoFilename = audioFilename;
|
miscInfoFilename = audioFilename;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
log.error('Failed to generate sentence audio:', (error as Error).message);
|
||||||
|
errors.push('audio');
|
||||||
}
|
}
|
||||||
} catch (error) {
|
|
||||||
log.error('Failed to generate sentence audio:', (error as Error).message);
|
|
||||||
errors.push('audio');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
if (generateImage) {
|
||||||
const imageFilename = this.generateImageFilename();
|
try {
|
||||||
const imageBuffer = await this.generateImageBuffer(videoPath, startTime, endTime);
|
const imageFilename = this.generateImageFilename();
|
||||||
|
const imageBuffer = await this.generateImageBuffer(videoPath!, startTime, endTime);
|
||||||
|
|
||||||
const imageField = this.deps.getConfig().fields?.image;
|
const imageField = config.fields?.image;
|
||||||
if (imageBuffer && imageField) {
|
if (imageBuffer && imageField) {
|
||||||
await this.deps.client.storeMediaFile(imageFilename, imageBuffer);
|
await this.deps.client.storeMediaFile(imageFilename, imageBuffer);
|
||||||
mediaFields[imageField] = `<img src="${imageFilename}">`;
|
mediaFields[imageField] = `<img src="${imageFilename}">`;
|
||||||
miscInfoFilename = imageFilename;
|
miscInfoFilename = imageFilename;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
log.error('Failed to generate sentence image:', (error as Error).message);
|
||||||
|
errors.push('image');
|
||||||
}
|
}
|
||||||
} catch (error) {
|
|
||||||
log.error('Failed to generate sentence image:', (error as Error).message);
|
|
||||||
errors.push('image');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.deps.getConfig().fields?.miscInfo) {
|
if (config.fields?.miscInfo) {
|
||||||
const miscInfo = this.deps.formatMiscInfoPattern(miscInfoFilename || '', startTime);
|
const miscInfo = this.deps.formatMiscInfoPattern(miscInfoFilename || '', startTime);
|
||||||
if (miscInfo && resolvedMiscInfoField) {
|
if (miscInfo && resolvedMiscInfoField) {
|
||||||
mediaFields[resolvedMiscInfoField] = miscInfo;
|
mediaFields[resolvedMiscInfoField] = miscInfo;
|
||||||
|
|||||||
@@ -234,9 +234,10 @@ export class NoteUpdateWorkflow {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const generateAudio = config.media?.generateAudio !== false;
|
||||||
|
const generateImage = config.media?.generateImage !== false;
|
||||||
const mediaCacheQueued =
|
const mediaCacheQueued =
|
||||||
(config.media?.generateAudio || config.media?.generateImage) &&
|
(generateAudio || generateImage) && this.deps.queuePendingYoutubeMediaUpdate
|
||||||
this.deps.queuePendingYoutubeMediaUpdate
|
|
||||||
? await this.deps.queuePendingYoutubeMediaUpdate({
|
? await this.deps.queuePendingYoutubeMediaUpdate({
|
||||||
noteId,
|
noteId,
|
||||||
noteInfo,
|
noteInfo,
|
||||||
@@ -245,7 +246,7 @@ export class NoteUpdateWorkflow {
|
|||||||
})
|
})
|
||||||
: false;
|
: false;
|
||||||
|
|
||||||
if (!mediaCacheQueued && config.media?.generateAudio) {
|
if (!mediaCacheQueued && generateAudio) {
|
||||||
try {
|
try {
|
||||||
const audioFilename = this.deps.generateAudioFilename();
|
const audioFilename = this.deps.generateAudioFilename();
|
||||||
const audioBuffer = await this.deps.generateAudio(subtitleMiningContext ?? undefined);
|
const audioBuffer = await this.deps.generateAudio(subtitleMiningContext ?? undefined);
|
||||||
@@ -270,7 +271,7 @@ export class NoteUpdateWorkflow {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!mediaCacheQueued && config.media?.generateImage) {
|
if (!mediaCacheQueued && generateImage) {
|
||||||
try {
|
try {
|
||||||
const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo);
|
const animatedLeadInSeconds = await this.deps.getAnimatedImageLeadInSeconds(noteInfo);
|
||||||
const imageFilename = this.deps.generateImageFilename();
|
const imageFilename = this.deps.generateImageFilename();
|
||||||
|
|||||||
@@ -2,7 +2,10 @@ import assert from 'node:assert/strict';
|
|||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
|
|
||||||
import type { AnkiConnectConfig } from '../types/anki';
|
import type { AnkiConnectConfig } from '../types/anki';
|
||||||
import { PendingYoutubeMediaQueue, type PendingYoutubeMediaQueueDeps } from './pending-youtube-media-queue';
|
import {
|
||||||
|
PendingYoutubeMediaQueue,
|
||||||
|
type PendingYoutubeMediaQueueDeps,
|
||||||
|
} from './pending-youtube-media-queue';
|
||||||
|
|
||||||
function createDeps(
|
function createDeps(
|
||||||
overrides: Partial<PendingYoutubeMediaQueueDeps> = {},
|
overrides: Partial<PendingYoutubeMediaQueueDeps> = {},
|
||||||
@@ -70,3 +73,121 @@ test('PendingYoutubeMediaQueue treats cache lookup failures as an immediate gene
|
|||||||
],
|
],
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('PendingYoutubeMediaQueue drains matching queued jobs when the cache download fails', async () => {
|
||||||
|
const statusMessages: string[] = [];
|
||||||
|
const notifications: Array<{ noteId: number; label: string | number; suffix?: string }> = [];
|
||||||
|
const updatedNotes: number[] = [];
|
||||||
|
const deps = createDeps({
|
||||||
|
client: {
|
||||||
|
notesInfo: async (noteIds) =>
|
||||||
|
noteIds.map((noteId) => ({
|
||||||
|
noteId,
|
||||||
|
fields: {
|
||||||
|
SentenceAudio: { value: '' },
|
||||||
|
Picture: { value: '' },
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
updateNoteFields: async (noteId) => {
|
||||||
|
updatedNotes.push(noteId);
|
||||||
|
},
|
||||||
|
storeMediaFile: async () => {},
|
||||||
|
},
|
||||||
|
showStatusNotification: (message) => {
|
||||||
|
statusMessages.push(message);
|
||||||
|
},
|
||||||
|
showNotification: async (noteId, label, suffix) => {
|
||||||
|
notifications.push({ noteId, label, suffix });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const queue = new PendingYoutubeMediaQueue(deps);
|
||||||
|
|
||||||
|
queue.enqueue({
|
||||||
|
sourceUrl: 'https://www.youtube.com/watch?v=abc123',
|
||||||
|
noteId: 42,
|
||||||
|
startTime: 1,
|
||||||
|
endTime: 2,
|
||||||
|
label: 'queued',
|
||||||
|
generateAudio: true,
|
||||||
|
generateImage: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
await queue.handleFailed('https://youtu.be/abc123');
|
||||||
|
await queue.handleReady('https://youtu.be/abc123', '/tmp/media.mkv');
|
||||||
|
|
||||||
|
assert.deepEqual(updatedNotes, []);
|
||||||
|
assert.deepEqual(notifications, [{ noteId: 42, label: 'queued', suffix: 'media cache failed' }]);
|
||||||
|
assert.equal(
|
||||||
|
statusMessages.includes('YouTube media cache failed. Media was not added to 1 queued card.'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('PendingYoutubeMediaQueue defaults missing media flags to enabled when queuing from notes', async () => {
|
||||||
|
const updatedNotes: Array<{ noteId: number; fields: Record<string, string> }> = [];
|
||||||
|
const storedMedia: string[] = [];
|
||||||
|
const deps = createDeps({
|
||||||
|
client: {
|
||||||
|
notesInfo: async (noteIds) =>
|
||||||
|
noteIds.map((noteId) => ({
|
||||||
|
noteId,
|
||||||
|
fields: {
|
||||||
|
SentenceAudio: { value: '' },
|
||||||
|
Picture: { value: '' },
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
updateNoteFields: async (noteId, fields) => {
|
||||||
|
updatedNotes.push({ noteId, fields });
|
||||||
|
},
|
||||||
|
storeMediaFile: async (filename) => {
|
||||||
|
storedMedia.push(filename);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
getConfig: () => ({ media: {}, fields: { image: 'Picture' } }) as AnkiConnectConfig,
|
||||||
|
});
|
||||||
|
const queue = new PendingYoutubeMediaQueue(deps);
|
||||||
|
|
||||||
|
const queued = await queue.queueFromNote({
|
||||||
|
noteId: 42,
|
||||||
|
noteInfo: { noteId: 42, fields: {} },
|
||||||
|
label: 'demo',
|
||||||
|
});
|
||||||
|
await queue.handleReady('https://youtu.be/abc123', '/tmp/media.mkv');
|
||||||
|
|
||||||
|
assert.equal(queued, true);
|
||||||
|
assert.equal(updatedNotes.length, 1);
|
||||||
|
assert.equal(storedMedia.length, 2);
|
||||||
|
assert.match(updatedNotes[0]?.fields.SentenceAudio ?? '', /^\[sound:audio\.mp3\]$/);
|
||||||
|
assert.match(updatedNotes[0]?.fields.Picture ?? '', /^<img src="image\.webp">$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('PendingYoutubeMediaQueue only announces a download once per source while jobs collect', () => {
|
||||||
|
const statusMessages: string[] = [];
|
||||||
|
const deps = createDeps({
|
||||||
|
showStatusNotification: (message) => {
|
||||||
|
statusMessages.push(message);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const queue = new PendingYoutubeMediaQueue(deps);
|
||||||
|
|
||||||
|
queue.enqueue({
|
||||||
|
sourceUrl: 'https://youtu.be/abc123',
|
||||||
|
noteId: 1,
|
||||||
|
startTime: 1,
|
||||||
|
endTime: 2,
|
||||||
|
label: 'first',
|
||||||
|
generateAudio: true,
|
||||||
|
generateImage: false,
|
||||||
|
});
|
||||||
|
queue.enqueue({
|
||||||
|
sourceUrl: 'https://www.youtube.com/watch?v=abc123',
|
||||||
|
noteId: 2,
|
||||||
|
startTime: 3,
|
||||||
|
endTime: 4,
|
||||||
|
label: 'second',
|
||||||
|
generateAudio: false,
|
||||||
|
generateImage: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(statusMessages.length, 1);
|
||||||
|
});
|
||||||
|
|||||||
@@ -12,6 +12,10 @@ export interface PendingYoutubeMediaQueueReadyOptions {
|
|||||||
notifyNoQueued?: boolean;
|
notifyNoQueued?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface PendingYoutubeMediaQueueFailedOptions {
|
||||||
|
notifyStatus?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
export interface PendingYoutubeMediaNoteInfo {
|
export interface PendingYoutubeMediaNoteInfo {
|
||||||
noteId: number;
|
noteId: number;
|
||||||
fields: Record<string, { value: string }>;
|
fields: Record<string, { value: string }>;
|
||||||
@@ -65,6 +69,14 @@ function trimToNonEmptyString(value: unknown): string | null {
|
|||||||
return trimmed.length > 0 ? trimmed : null;
|
return trimmed.length > 0 ? trimmed : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function shouldGenerateAudio(config: AnkiConnectConfig): boolean {
|
||||||
|
return config.media?.generateAudio !== false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldGenerateImage(config: AnkiConnectConfig): boolean {
|
||||||
|
return config.media?.generateImage !== false;
|
||||||
|
}
|
||||||
|
|
||||||
export class PendingYoutubeMediaQueue {
|
export class PendingYoutubeMediaQueue {
|
||||||
private updates: PendingYoutubeMediaUpdate[] = [];
|
private updates: PendingYoutubeMediaUpdate[] = [];
|
||||||
|
|
||||||
@@ -74,11 +86,16 @@ export class PendingYoutubeMediaQueue {
|
|||||||
if (!job.generateAudio && !job.generateImage) {
|
if (!job.generateAudio && !job.generateImage) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const isFirstQueuedForSource = !this.updates.some((existing) =>
|
||||||
|
youtubeMediaUrlsMatch(existing.sourceUrl, job.sourceUrl),
|
||||||
|
);
|
||||||
this.updates.push(job);
|
this.updates.push(job);
|
||||||
this.deps.logInfo('Queued YouTube media update for note:', job.noteId);
|
this.deps.logInfo('Queued YouTube media update for note:', job.noteId);
|
||||||
this.deps.showStatusNotification(
|
if (isFirstQueuedForSource) {
|
||||||
'YouTube media cache is still downloading. Card media will be added when the cache is ready.',
|
this.deps.showStatusNotification(
|
||||||
);
|
'YouTube media cache is still downloading. Card media will be added when the cache is ready.',
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async queueFromNote(job: {
|
async queueFromNote(job: {
|
||||||
@@ -124,8 +141,8 @@ export class PendingYoutubeMediaQueue {
|
|||||||
) ?? undefined,
|
) ?? undefined,
|
||||||
miscInfoFieldName:
|
miscInfoFieldName:
|
||||||
this.deps.resolveConfiguredFieldName(job.noteInfo, config.fields?.miscInfo) ?? undefined,
|
this.deps.resolveConfiguredFieldName(job.noteInfo, config.fields?.miscInfo) ?? undefined,
|
||||||
generateAudio: config.media?.generateAudio === true,
|
generateAudio: shouldGenerateAudio(config),
|
||||||
generateImage: config.media?.generateImage === true,
|
generateImage: shouldGenerateImage(config),
|
||||||
});
|
});
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -178,6 +195,39 @@ export class PendingYoutubeMediaQueue {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async handleFailed(
|
||||||
|
sourceUrl: string,
|
||||||
|
options: PendingYoutubeMediaQueueFailedOptions = {},
|
||||||
|
): Promise<void> {
|
||||||
|
const jobs = this.takeMatchingUpdates(sourceUrl);
|
||||||
|
if (jobs.length === 0) {
|
||||||
|
if (options.notifyStatus !== false) {
|
||||||
|
this.deps.showStatusNotification('YouTube media cache failed.');
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.notifyStatus !== false) {
|
||||||
|
this.deps.showStatusNotification(
|
||||||
|
`YouTube media cache failed. Media was not added to ${jobs.length} queued card${
|
||||||
|
jobs.length === 1 ? '' : 's'
|
||||||
|
}.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
this.deps.logWarn('Discarding queued YouTube media updates after cache failure:', jobs.length);
|
||||||
|
|
||||||
|
for (const job of jobs) {
|
||||||
|
try {
|
||||||
|
await this.deps.showNotification(job.noteId, job.label, 'media cache failed');
|
||||||
|
} catch (error) {
|
||||||
|
this.deps.logWarn(
|
||||||
|
'Failed to show queued YouTube media failure notification:',
|
||||||
|
error instanceof Error ? error.message : String(error),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private takeMatchingUpdates(sourceUrl: string): PendingYoutubeMediaUpdate[] {
|
private takeMatchingUpdates(sourceUrl: string): PendingYoutubeMediaUpdate[] {
|
||||||
const matched: PendingYoutubeMediaUpdate[] = [];
|
const matched: PendingYoutubeMediaUpdate[] = [];
|
||||||
const remaining: PendingYoutubeMediaUpdate[] = [];
|
const remaining: PendingYoutubeMediaUpdate[] = [];
|
||||||
|
|||||||
@@ -82,6 +82,10 @@ test('YouTube media cache exposes the downloaded file after the background job c
|
|||||||
assert.equal(spawnCalls.length, 1);
|
assert.equal(spawnCalls.length, 1);
|
||||||
assert.equal(spawnCalls[0]?.command, 'yt-dlp');
|
assert.equal(spawnCalls[0]?.command, 'yt-dlp');
|
||||||
assert.ok(spawnCalls[0]?.args.includes('--no-playlist'));
|
assert.ok(spawnCalls[0]?.args.includes('--no-playlist'));
|
||||||
|
assert.ok(spawnCalls[0]?.args.includes('--force-ipv4'));
|
||||||
|
assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--retries') + 1], '5');
|
||||||
|
assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--fragment-retries') + 1], '5');
|
||||||
|
assert.equal(spawnCalls[0]?.args[spawnCalls[0].args.indexOf('--extractor-retries') + 1], '5');
|
||||||
assert.ok(spawnCalls[0]?.args.includes('--merge-output-format'));
|
assert.ok(spawnCalls[0]?.args.includes('--merge-output-format'));
|
||||||
assert.equal(
|
assert.equal(
|
||||||
spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-f') + 1],
|
spawnCalls[0]?.args[spawnCalls[0].args.indexOf('-f') + 1],
|
||||||
@@ -107,6 +111,68 @@ test('YouTube media cache exposes the downloaded file after the background job c
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('YouTube media cache reports failed background downloads', async () => {
|
||||||
|
const cacheRoot = makeTempCacheRoot();
|
||||||
|
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||||
|
const failedEvents: Array<{ url: string }> = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
const cache = createYoutubeMediaCacheService({
|
||||||
|
cacheRoot,
|
||||||
|
getYtDlpCommand: () => 'yt-dlp',
|
||||||
|
onFailed: (event) => {
|
||||||
|
failedEvents.push(event);
|
||||||
|
},
|
||||||
|
spawn: () => {
|
||||||
|
const proc = new FakeYtDlpProcess();
|
||||||
|
spawnedProcesses.push(proc);
|
||||||
|
return proc;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||||
|
spawnedProcesses[0]?.emit('close', 1);
|
||||||
|
|
||||||
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
|
|
||||||
|
assert.deepEqual(failedEvents, [{ url: 'https://youtu.be/demo' }]);
|
||||||
|
assert.equal(await cache.getCachedMediaPath('https://youtu.be/demo'), null);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('YouTube media cache only reports a failed download once when error is followed by close', async () => {
|
||||||
|
const cacheRoot = makeTempCacheRoot();
|
||||||
|
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||||
|
const failedEvents: Array<{ url: string }> = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
const cache = createYoutubeMediaCacheService({
|
||||||
|
cacheRoot,
|
||||||
|
getYtDlpCommand: () => 'yt-dlp',
|
||||||
|
onFailed: (event) => {
|
||||||
|
failedEvents.push(event);
|
||||||
|
},
|
||||||
|
spawn: () => {
|
||||||
|
const proc = new FakeYtDlpProcess();
|
||||||
|
spawnedProcesses.push(proc);
|
||||||
|
return proc;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||||
|
spawnedProcesses[0]?.emit('error', new Error('spawn failed'));
|
||||||
|
spawnedProcesses[0]?.emit('close', 1);
|
||||||
|
|
||||||
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
|
|
||||||
|
assert.deepEqual(failedEvents, [{ url: 'https://youtu.be/demo' }]);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('YouTube media cache can disable the download height cap', () => {
|
test('YouTube media cache can disable the download height cap', () => {
|
||||||
const cacheRoot = makeTempCacheRoot();
|
const cacheRoot = makeTempCacheRoot();
|
||||||
const spawnCalls: SpawnCall[] = [];
|
const spawnCalls: SpawnCall[] = [];
|
||||||
@@ -159,6 +225,31 @@ test('YouTube media cache applies the configured download height cap', () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('YouTube media cache removes stale files from previous runs on startup', () => {
|
||||||
|
const cacheRoot = makeTempCacheRoot();
|
||||||
|
const staleDir = path.join(cacheRoot, 'stale-session');
|
||||||
|
const spawnCalls: SpawnCall[] = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
fs.mkdirSync(staleDir, { recursive: true });
|
||||||
|
fs.writeFileSync(path.join(staleDir, 'media.mkv'), 'stale cached media');
|
||||||
|
|
||||||
|
createYoutubeMediaCacheService({
|
||||||
|
cacheRoot,
|
||||||
|
getYtDlpCommand: () => 'yt-dlp',
|
||||||
|
spawn: (command, args) => {
|
||||||
|
spawnCalls.push({ command, args });
|
||||||
|
return new FakeYtDlpProcess();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(fs.existsSync(staleDir), false);
|
||||||
|
assert.deepEqual(spawnCalls, []);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('YouTube media cache restarts when a ready cached file was deleted externally', async () => {
|
test('YouTube media cache restarts when a ready cached file was deleted externally', async () => {
|
||||||
const cacheRoot = makeTempCacheRoot();
|
const cacheRoot = makeTempCacheRoot();
|
||||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||||
@@ -198,6 +289,32 @@ test('YouTube media cache restarts when a ready cached file was deleted external
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('YouTube media cache removes stale disk siblings before starting a new cache', () => {
|
||||||
|
const cacheRoot = makeTempCacheRoot();
|
||||||
|
const staleDir = path.join(cacheRoot, 'stale-sibling');
|
||||||
|
const spawnCalls: Array<{ command: string; args: string[] }> = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
const cache = createYoutubeMediaCacheService({
|
||||||
|
cacheRoot,
|
||||||
|
getYtDlpCommand: () => 'yt-dlp',
|
||||||
|
spawn: (command, args) => {
|
||||||
|
spawnCalls.push({ command, args });
|
||||||
|
return new FakeYtDlpProcess();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
fs.mkdirSync(staleDir, { recursive: true });
|
||||||
|
fs.writeFileSync(path.join(staleDir, 'media.mkv'), 'stale cached media');
|
||||||
|
|
||||||
|
cache.start('https://youtu.be/demo', { mode: 'background' });
|
||||||
|
|
||||||
|
assert.equal(fs.existsSync(staleDir), false);
|
||||||
|
assert.equal(spawnCalls.length, 1);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(cacheRoot, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('YouTube media cache drops old sessions when a new background cache starts', async () => {
|
test('YouTube media cache drops old sessions when a new background cache starts', async () => {
|
||||||
const cacheRoot = makeTempCacheRoot();
|
const cacheRoot = makeTempCacheRoot();
|
||||||
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
const spawnedProcesses: FakeYtDlpProcess[] = [];
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ export interface YoutubeMediaCacheServiceDeps {
|
|||||||
spawn?: SpawnProcess;
|
spawn?: SpawnProcess;
|
||||||
onDownloadStarted?: (event: { url: string }) => void;
|
onDownloadStarted?: (event: { url: string }) => void;
|
||||||
onReady?: (event: { url: string; path: string }) => void;
|
onReady?: (event: { url: string; path: string }) => void;
|
||||||
|
onFailed?: (event: { url: string }) => void;
|
||||||
logInfo?: (message: string) => void;
|
logInfo?: (message: string) => void;
|
||||||
logWarn?: (message: string) => void;
|
logWarn?: (message: string) => void;
|
||||||
}
|
}
|
||||||
@@ -88,6 +89,13 @@ function createYtDlpArgs(url: string, outputTemplate: string, maxHeight?: number
|
|||||||
return [
|
return [
|
||||||
'--no-playlist',
|
'--no-playlist',
|
||||||
'--no-warnings',
|
'--no-warnings',
|
||||||
|
'--force-ipv4',
|
||||||
|
'--retries',
|
||||||
|
'5',
|
||||||
|
'--fragment-retries',
|
||||||
|
'5',
|
||||||
|
'--extractor-retries',
|
||||||
|
'5',
|
||||||
'-f',
|
'-f',
|
||||||
getFormatSelector(normalizeMaxHeight(maxHeight)),
|
getFormatSelector(normalizeMaxHeight(maxHeight)),
|
||||||
'--merge-output-format',
|
'--merge-output-format',
|
||||||
@@ -109,6 +117,29 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
|||||||
let activeKey: string | null = null;
|
let activeKey: string | null = null;
|
||||||
|
|
||||||
const getSessionDir = (url: string): string => path.join(cacheRoot, cacheKeyForUrl(url));
|
const getSessionDir = (url: string): string => path.join(cacheRoot, cacheKeyForUrl(url));
|
||||||
|
const removeCacheDir = (dir: string): void => {
|
||||||
|
try {
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
|
} catch {
|
||||||
|
// Temp cache cleanup should not block shutdown or playback startup.
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const removeCacheRootEntriesExcept = (dirsToKeep: string[]): void => {
|
||||||
|
let entries: fs.Dirent[];
|
||||||
|
try {
|
||||||
|
entries = fs.readdirSync(cacheRoot, { withFileTypes: true });
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const keepDirs = new Set(dirsToKeep.map((dir) => path.resolve(dir)));
|
||||||
|
for (const entry of entries) {
|
||||||
|
const entryPath = path.join(cacheRoot, entry.name);
|
||||||
|
if (keepDirs.has(path.resolve(entryPath))) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
removeCacheDir(entryPath);
|
||||||
|
}
|
||||||
|
};
|
||||||
const removeSession = (key: string): void => {
|
const removeSession = (key: string): void => {
|
||||||
const session = sessions.get(key);
|
const session = sessions.get(key);
|
||||||
if (!session) {
|
if (!session) {
|
||||||
@@ -121,11 +152,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
|||||||
if (activeKey === key) {
|
if (activeKey === key) {
|
||||||
activeKey = null;
|
activeKey = null;
|
||||||
}
|
}
|
||||||
try {
|
removeCacheDir(session.dir);
|
||||||
fs.rmSync(session.dir, { recursive: true, force: true });
|
|
||||||
} catch {
|
|
||||||
// Temp cache cleanup should not block shutdown or playback startup.
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
const removeInactiveSessions = (keyToKeep: string): void => {
|
const removeInactiveSessions = (keyToKeep: string): void => {
|
||||||
for (const key of [...sessions.keys()]) {
|
for (const key of [...sessions.keys()]) {
|
||||||
@@ -134,6 +161,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
removeCacheRootEntriesExcept([]);
|
||||||
|
|
||||||
const getCachedMediaPath = async (url: string): Promise<string | null> => {
|
const getCachedMediaPath = async (url: string): Promise<string | null> => {
|
||||||
const key = cacheKeyForUrl(url);
|
const key = cacheKeyForUrl(url);
|
||||||
@@ -172,9 +200,11 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
|||||||
|
|
||||||
const key = cacheKeyForUrl(url);
|
const key = cacheKeyForUrl(url);
|
||||||
activeKey = key;
|
activeKey = key;
|
||||||
|
const dir = getSessionDir(url);
|
||||||
const existingSession = sessions.get(key);
|
const existingSession = sessions.get(key);
|
||||||
if (existingSession?.state === 'running') {
|
if (existingSession?.state === 'running') {
|
||||||
removeInactiveSessions(key);
|
removeInactiveSessions(key);
|
||||||
|
removeCacheRootEntriesExcept([existingSession.dir]);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (existingSession) {
|
if (existingSession) {
|
||||||
@@ -184,14 +214,15 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
|||||||
findReadyMediaPath(existingSession.dir))
|
findReadyMediaPath(existingSession.dir))
|
||||||
) {
|
) {
|
||||||
removeInactiveSessions(key);
|
removeInactiveSessions(key);
|
||||||
|
removeCacheRootEntriesExcept([existingSession.dir]);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
removeSession(key);
|
removeSession(key);
|
||||||
activeKey = key;
|
activeKey = key;
|
||||||
}
|
}
|
||||||
removeInactiveSessions(key);
|
removeInactiveSessions(key);
|
||||||
|
removeCacheRootEntriesExcept([dir]);
|
||||||
|
|
||||||
const dir = getSessionDir(url);
|
|
||||||
fs.mkdirSync(dir, { recursive: true });
|
fs.mkdirSync(dir, { recursive: true });
|
||||||
const outputTemplate = path.join(dir, 'media.%(ext)s');
|
const outputTemplate = path.join(dir, 'media.%(ext)s');
|
||||||
const args = createYtDlpArgs(url, outputTemplate, options.maxHeight);
|
const args = createYtDlpArgs(url, outputTemplate, options.maxHeight);
|
||||||
@@ -219,6 +250,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
|||||||
error instanceof Error ? error.message : String(error)
|
error instanceof Error ? error.message : String(error)
|
||||||
}`,
|
}`,
|
||||||
);
|
);
|
||||||
|
deps.onFailed?.({ url });
|
||||||
});
|
});
|
||||||
|
|
||||||
child.once('close', (code) => {
|
child.once('close', (code) => {
|
||||||
@@ -226,6 +258,9 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
|||||||
if (currentSession !== session) {
|
if (currentSession !== session) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (session.state === 'failed') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
session.process = null;
|
session.process = null;
|
||||||
if (code === 0) {
|
if (code === 0) {
|
||||||
const readyPath = findReadyMediaPath(dir);
|
const readyPath = findReadyMediaPath(dir);
|
||||||
@@ -239,6 +274,7 @@ export function createYoutubeMediaCacheService(deps: YoutubeMediaCacheServiceDep
|
|||||||
}
|
}
|
||||||
session.state = 'failed';
|
session.state = 'failed';
|
||||||
deps.logWarn?.(`YouTube media cache download exited without a usable media file.`);
|
deps.logWarn?.(`YouTube media cache download exited without a usable media file.`);
|
||||||
|
deps.onFailed?.({ url });
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
+17
@@ -1200,6 +1200,23 @@ const youtubeMediaCache = createYoutubeMediaCacheService({
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
onFailed: (event) => {
|
||||||
|
showConfiguredStatusNotification('YouTube media cache failed.', {
|
||||||
|
id: 'youtube-media-cache-status',
|
||||||
|
title: 'YouTube media cache',
|
||||||
|
variant: 'error',
|
||||||
|
persistent: false,
|
||||||
|
});
|
||||||
|
void appState.ankiIntegration
|
||||||
|
?.handleYoutubeMediaCacheFailed(event.url, { notifyStatus: false })
|
||||||
|
.catch((error) => {
|
||||||
|
logger.warn(
|
||||||
|
`Failed to drain queued YouTube media updates after cache failure: ${
|
||||||
|
error instanceof Error ? error.message : String(error)
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
},
|
||||||
logInfo: (message) => logger.info(message),
|
logInfo: (message) => logger.info(message),
|
||||||
logWarn: (message) => logger.warn(message),
|
logWarn: (message) => logger.warn(message),
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -212,6 +212,8 @@ test('generateAudio adds remote input options before the ffmpeg input', async ()
|
|||||||
assert.ok(args.indexOf('-reconnect') < inputIndex);
|
assert.ok(args.indexOf('-reconnect') < inputIndex);
|
||||||
assert.equal(args[args.indexOf('-reconnect') + 1], '1');
|
assert.equal(args[args.indexOf('-reconnect') + 1], '1');
|
||||||
assert.equal(args[args.indexOf('-reconnect_streamed') + 1], '1');
|
assert.equal(args[args.indexOf('-reconnect_streamed') + 1], '1');
|
||||||
|
assert.equal(args[args.indexOf('-reconnect_on_network_error') + 1], '1');
|
||||||
|
assert.equal(args[args.indexOf('-reconnect_on_http_error') + 1], '403,5xx');
|
||||||
assert.equal(args[args.indexOf('-reconnect_delay_max') + 1], '5');
|
assert.equal(args[args.indexOf('-reconnect_delay_max') + 1], '5');
|
||||||
assert.equal(args[args.indexOf('-user_agent') + 1], 'Mozilla/5.0');
|
assert.equal(args[args.indexOf('-user_agent') + 1], 'Mozilla/5.0');
|
||||||
assert.equal(
|
assert.equal(
|
||||||
|
|||||||
+12
-1
@@ -65,7 +65,18 @@ export function normalizeMediaInput(input: MediaInput): NormalizedMediaInput {
|
|||||||
|
|
||||||
const inputArgs: string[] = [];
|
const inputArgs: string[] = [];
|
||||||
if (input.inputOptions?.reconnect) {
|
if (input.inputOptions?.reconnect) {
|
||||||
inputArgs.push('-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5');
|
inputArgs.push(
|
||||||
|
'-reconnect',
|
||||||
|
'1',
|
||||||
|
'-reconnect_streamed',
|
||||||
|
'1',
|
||||||
|
'-reconnect_on_network_error',
|
||||||
|
'1',
|
||||||
|
'-reconnect_on_http_error',
|
||||||
|
'403,5xx',
|
||||||
|
'-reconnect_delay_max',
|
||||||
|
'5',
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const userAgent = trimToNonEmptyString(input.inputOptions?.userAgent);
|
const userAgent = trimToNonEmptyString(input.inputOptions?.userAgent);
|
||||||
|
|||||||
Reference in New Issue
Block a user