feat(youtube): add YouTube browser window and Whisper subtitle source (#273)

src/core/services/youtube/track-download.test.ts

@@ -478,6 +478,35 @@ test('downloadYoutubeSubtitleTrack prefers direct download URL when available',
   });
 });
 
+test('downloadYoutubeSubtitleTrack falls back to yt-dlp when the direct URL is rejected', async () => {
+  if (process.platform === 'win32') {
+    return;
+  }
+
+  await withFakeYtDlp('both', async (root) => {
+    await withStubFetch(
+      async () => new Response('Too Many Requests', { status: 429 }),
+      async () => {
+        const result = await downloadYoutubeSubtitleTrack({
+          targetUrl: 'https://www.youtube.com/watch?v=abc123',
+          outputDir: path.join(root, 'out'),
+          track: {
+            id: 'auto:ja',
+            language: 'ja',
+            sourceLanguage: 'ja',
+            kind: 'auto',
+            label: 'Japanese (auto)',
+            downloadUrl: 'https://www.youtube.com/api/timedtext?v=abc123&lang=ja&fmt=vtt',
+            fileExtension: 'vtt',
+          },
+        });
+
+        assert.equal(path.extname(result.path), '.vtt');
+      },
+    );
+  });
+});
+
 test('downloadYoutubeSubtitleTrack sanitizes metadata source language in filenames', async () => {
   await withTempDir(async (root) => {
     await withStubFetch(

src/core/services/youtube/track-download.ts

@@ -221,11 +221,16 @@ export async function downloadYoutubeSubtitleTrack(input: {
     }
   }
   if (canDownloadSubtitleFromUrl(input.track)) {
-    return await downloadSubtitleFromUrl({
-      outputDir: input.outputDir,
-      prefix,
-      track: input.track,
-    });
+    try {
+      return await downloadSubtitleFromUrl({
+        outputDir: input.outputDir,
+        prefix,
+        track: input.track,
+      });
+    } catch {
+      // YouTube rejects some direct timedtext requests (HTTP 429) that yt-dlp still gets
+      // through with its own client handling, so retry the same track via yt-dlp.
+    }
   }
   const outputTemplate = path.join(input.outputDir, `${prefix}.%(ext)s`);
   const args = [

src/core/services/youtube/track-probe.test.ts

@@ -116,6 +116,31 @@ test('probeYoutubeTracks prefers srv3 over vtt for automatic captions', async ()
   );
 });
 
+test('probeYoutubeTracks skips machine-translated caption URLs when the real track exists', async () => {
+  const translated = 'https://www.youtube.com/api/timedtext?v=abc123&lang=en-US&tlang=ja&fmt=srv3';
+  const original = 'https://www.youtube.com/api/timedtext?v=abc123&lang=ja&fmt=srv3';
+  await withFakeYtDlp(
+    {
+      id: 'abc123',
+      title: 'Example',
+      automatic_captions: {
+        ja: [
+          { ext: 'srv3', url: translated, name: 'Japanese' },
+          { ext: 'srv3', url: original, name: 'Japanese' },
+        ],
+        fr: [{ ext: 'srv3', url: `${translated}&x=fr`, name: 'French' }],
+      },
+    },
+    async () => {
+      const result = await probeYoutubeTracks('https://www.youtube.com/watch?v=abc123');
+      const byLanguage = new Map(result.tracks.map((track) => [track.sourceLanguage, track]));
+      assert.equal(byLanguage.get('ja')?.downloadUrl, original);
+      // A translation-only language still gets a track rather than disappearing.
+      assert.equal(byLanguage.get('fr')?.downloadUrl, `${translated}&x=fr`);
+    },
+  );
+});
+
 test('probeYoutubeTracks honors SUBMINER_YTDLP_BIN when yt-dlp is not on PATH', async () => {
   if (process.platform === 'win32') {
     return;

src/core/services/youtube/track-probe.ts

@@ -56,10 +56,25 @@ function runCapture(
   });
 }
 
+// YouTube machine-translates captions on request (`tlang`) and lists them under the target
+// language, next to the real track. Those URLs reliably fail with HTTP 429, so they are only
+// used when a language has nothing else.
+function isTranslatedCaptionUrl(url: string): boolean {
+  try {
+    return new URL(url).searchParams.has('tlang');
+  } catch {
+    return false;
+  }
+}
+
 function choosePreferredFormat(
-  formats: YtDlpSubtitleEntry,
+  allFormats: YtDlpSubtitleEntry,
   kind: YoutubeTrackKind,
 ): { ext: string; url: string; title?: string } | null {
+  const untranslated = allFormats.filter(
+    (format) => typeof format.url === 'string' && !isTranslatedCaptionUrl(format.url),
+  );
+  const formats = untranslated.length > 0 ? untranslated : allFormats;
   const preferredOrder =
     kind === 'auto'
       ? ['srv3', 'srv2', 'srv1', 'vtt', 'srt', 'ttml', 'json3']

src/core/services/youtube/whisper-subtitles.test.ts

@@ -0,0 +1,168 @@
+import assert from 'node:assert/strict';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import test from 'node:test';
+import { DEFAULT_SUBTITLE_GENERATION_CONFIG } from '../../../shared/subtitle-generation';
+import {
+  buildYoutubeWhisperAudioArgs,
+  createYoutubeWhisperSubtitleService,
+  parseYtDlpDownloadPercent,
+  YOUTUBE_WHISPER_AUDIO_DIR_PREFIX,
+} from './whisper-subtitles';
+
+const URL = 'https://www.youtube.com/watch?v=abcdefghijk';
+
+function makeTempRoot(): string {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-youtube-whisper-test-'));
+}
+
+function audioDirs(root: string): string[] {
+  return fs.readdirSync(root).filter((name) => name.startsWith(YOUTUBE_WHISPER_AUDIO_DIR_PREFIX));
+}
+
+/** Stands in for yt-dlp: writes the file its `-o` template names and reports progress. */
+function fakeYtDlp(calls: string[][]) {
+  return async (input: { args: string[]; onLine?: (line: string) => void }) => {
+    calls.push(input.args);
+    const template = input.args[input.args.indexOf('-o') + 1]!;
+    fs.writeFileSync(template.replace('%(ext)s', 'm4a'), 'audio');
+    input.onLine?.('[download]  50.0% of 3.89MiB at 1.00MiB/s ETA 00:02');
+    return '';
+  };
+}
+
+function createService(root: string, overrides: Record<string, unknown> = {}) {
+  return createYoutubeWhisperSubtitleService({
+    getConfig: () => DEFAULT_SUBTITLE_GENERATION_CONFIG,
+    getModelDirectory: () => '/models',
+    tempRoot: root,
+    getYtDlpCommand: () => 'yt-dlp',
+    requireSetup: async () => ({
+      modelPath: '/models/ggml-small.bin',
+      tools: { ffmpeg: 'ffmpeg', ffprobe: 'ffprobe', whisper: 'whisper-cli', vad: null },
+    }),
+    ...overrides,
+  });
+}
+
+test('audio download picks a small original-language stream and ignores user embeds', () => {
+  const args = buildYoutubeWhisperAudioArgs(URL, '/tmp/audio.%(ext)s');
+  assert.deepEqual(args.slice(args.indexOf('-f'), args.indexOf('-f') + 4), [
+    '-f',
+    'ba[abr>=48]/ba',
+    '-S',
+    'lang,+abr',
+  ]);
+  for (const flag of ['--no-playlist', '--no-sponsorblock', '--no-embed-subs']) {
+    assert.ok(args.includes(flag), flag);
+  }
+  assert.equal(args.at(-1), URL);
+});
+
+test('parseYtDlpDownloadPercent reads yt-dlp progress lines only', () => {
+  assert.equal(parseYtDlpDownloadPercent('[download]  42.5% of 3.89MiB'), 42.5);
+  assert.equal(parseYtDlpDownloadPercent('[download] Destination: audio.m4a'), null);
+});
+
+test('generate transcribes the downloaded audio and deletes it afterwards', async () => {
+  const root = makeTempRoot();
+  const calls: string[][] = [];
+  const progress: string[] = [];
+  let transcribedPath = '';
+  const service = createService(root, {
+    runProcess: fakeYtDlp(calls),
+    generate: async (input: { mediaPath: string; outputPath: string; workDirectory?: string }) => {
+      transcribedPath = input.mediaPath;
+      assert.equal(fs.readFileSync(input.mediaPath, 'utf8'), 'audio');
+      // The extracted WAV must land where cleanup already looks.
+      assert.equal(input.workDirectory, path.dirname(input.mediaPath));
+      return input.outputPath;
+    },
+  });
+
+  const result = await service.generate({
+    url: URL,
+    outputPath: '/subs/youtube-whisper.ja.srt',
+    onProgress: (update) => progress.push(`${update.stage}:${update.percent}`),
+  });
+
+  assert.equal(result, '/subs/youtube-whisper.ja.srt');
+  assert.equal(calls.length, 1);
+  assert.equal(path.basename(transcribedPath), 'audio.m4a');
+  assert.deepEqual(progress, ['download:0', 'download:50']);
+  assert.deepEqual(audioDirs(root), []);
+});
+
+test('generate deletes the audio when transcription fails', async () => {
+  const root = makeTempRoot();
+  const service = createService(root, {
+    runProcess: fakeYtDlp([]),
+    generate: async () => {
+      throw new Error('whisper crashed');
+    },
+  });
+
+  await assert.rejects(
+    service.generate({ url: URL, outputPath: '/subs/x.srt' }),
+    /whisper crashed/,
+  );
+  assert.deepEqual(audioDirs(root), []);
+});
+
+test('generate skips the download when Whisper is not set up', async () => {
+  const root = makeTempRoot();
+  const calls: string[][] = [];
+  const service = createService(root, {
+    runProcess: fakeYtDlp(calls),
+    requireSetup: async () => {
+      throw new Error('No Whisper model found.');
+    },
+  });
+
+  await assert.rejects(
+    service.generate({ url: URL, outputPath: '/subs/x.srt' }),
+    /No Whisper model/,
+  );
+  assert.equal(calls.length, 0);
+  assert.deepEqual(audioDirs(root), []);
+});
+
+test('leftover audio from a crashed run is removed before the next download', async () => {
+  const root = makeTempRoot();
+  fs.mkdirSync(path.join(root, `${YOUTUBE_WHISPER_AUDIO_DIR_PREFIX}stale`));
+  let dirsDuringDownload: string[] = [];
+  const service = createService(root, {
+    runProcess: async (input: { args: string[] }) => {
+      dirsDuringDownload = audioDirs(root);
+      return fakeYtDlp([])(input);
+    },
+    generate: async (input: { outputPath: string }) => input.outputPath,
+  });
+
+  await service.generate({ url: URL, outputPath: '/subs/x.srt' });
+  assert.equal(dirsDuringDownload.length, 1);
+  assert.notEqual(dirsDuringDownload[0], `${YOUTUBE_WHISPER_AUDIO_DIR_PREFIX}stale`);
+});
+
+test('removeActiveAudio deletes audio for a job that is still running', async () => {
+  const root = makeTempRoot();
+  let release: (() => void) | null = null;
+  const service = createService(root, {
+    runProcess: fakeYtDlp([]),
+    generate: async (input: { outputPath: string }) => {
+      await new Promise<void>((resolve) => {
+        release = resolve;
+      });
+      return input.outputPath;
+    },
+  });
+
+  const job = service.generate({ url: URL, outputPath: '/subs/x.srt' });
+  while (!release) await new Promise((resolve) => setImmediate(resolve));
+  assert.equal(audioDirs(root).length, 1);
+  service.removeActiveAudio();
+  assert.deepEqual(audioDirs(root), []);
+  (release as () => void)();
+  await job;
+});

src/core/services/youtube/whisper-subtitles.ts

@@ -0,0 +1,148 @@
+import { mkdtemp, readdir, rm } from 'node:fs/promises';
+import { rmSync } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import type {
+  SubtitleGenerationConfig,
+  SubtitleGenerationProgress,
+} from '../../../shared/subtitle-generation';
+import { generateJapaneseSubtitles, requireSubtitleGenerationSetup } from '../subtitle-generation';
+import { runSubtitleGenerationProcess } from '../subtitle-generation-process';
+import { getYoutubeYtDlpCommand, YTDLP_SINGLE_VIDEO_ARG } from './ytdlp-command';
+
+export const YOUTUBE_WHISPER_AUDIO_DIR_PREFIX = 'subminer-youtube-audio-';
+
+/**
+ * Whisper resamples everything to 16 kHz mono, so bitrate beyond plain speech clarity only costs
+ * download time. Take the smallest audio-only stream of at least 48 kbps, falling back to the
+ * smallest one. Sorting on `lang` first keeps the original track ahead of auto-dubbed ones.
+ */
+const AUDIO_FORMAT = 'ba[abr>=48]/ba';
+const AUDIO_SORT = 'lang,+abr';
+
+export function buildYoutubeWhisperAudioArgs(url: string, outputTemplate: string): string[] {
+  return [
+    YTDLP_SINGLE_VIDEO_ARG,
+    '--no-warnings',
+    '--newline',
+    // Override user yt-dlp config that would change the audio or add requests: SponsorBlock
+    // removal shifts every subtitle after a cut, and subtitle embedding hits the timedtext API.
+    '--no-sponsorblock',
+    '--no-write-subs',
+    '--no-write-auto-subs',
+    '--no-embed-subs',
+    '--no-write-thumbnail',
+    '--no-embed-thumbnail',
+    '--no-embed-metadata',
+    '--no-embed-chapters',
+    '-f',
+    AUDIO_FORMAT,
+    '-S',
+    AUDIO_SORT,
+    '-o',
+    outputTemplate,
+    url,
+  ];
+}
+
+export function parseYtDlpDownloadPercent(line: string): number | null {
+  const match = /^\[download\]\s+(\d+(?:\.\d+)?)%/.exec(line.trim());
+  return match ? Math.min(100, Number(match[1])) : null;
+}
+
+async function findDownloadedAudio(directory: string): Promise<string> {
+  const names = await readdir(directory);
+  const audio = names.find((name) => name.startsWith('audio.') && !/\.(part|ytdl)$/.test(name));
+  if (!audio) throw new Error('yt-dlp finished without saving the audio.');
+  return path.join(directory, audio);
+}
+
+export type YoutubeWhisperSubtitleServiceDeps = {
+  getConfig: () => SubtitleGenerationConfig;
+  getModelDirectory: () => string;
+  tempRoot?: string;
+  getYtDlpCommand?: () => string;
+  requireSetup?: typeof requireSubtitleGenerationSetup;
+  runProcess?: typeof runSubtitleGenerationProcess;
+  generate?: typeof generateJapaneseSubtitles;
+};
+
+/**
+ * Generates Japanese subtitles for a YouTube video by transcribing its audio with Whisper.
+ * The audio lives in a private temp directory that is removed as soon as generation ends,
+ * whether it succeeds, fails, or is aborted.
+ */
+export function createYoutubeWhisperSubtitleService(deps: YoutubeWhisperSubtitleServiceDeps) {
+  const tempRoot = deps.tempRoot ?? os.tmpdir();
+  const activeDirs = new Set<string>();
+
+  // Audio directories only outlive a job when the app crashes mid-generation.
+  const removeStaleAudioDirs = async (): Promise<void> => {
+    const names = await readdir(tempRoot).catch(() => []);
+    await Promise.all(
+      names
+        .filter((name) => name.startsWith(YOUTUBE_WHISPER_AUDIO_DIR_PREFIX))
+        .map((name) => path.join(tempRoot, name))
+        .filter((dir) => !activeDirs.has(dir))
+        .map((dir) => rm(dir, { recursive: true, force: true }).catch(() => undefined)),
+    );
+  };
+
+  async function generate(input: {
+    url: string;
+    outputPath: string;
+    /** Overrides the configured settings, e.g. the modal's per-session model choice. */
+    config?: SubtitleGenerationConfig;
+    signal?: AbortSignal;
+    onProgress?: (progress: SubtitleGenerationProgress) => void;
+  }): Promise<string> {
+    input.signal?.throwIfAborted();
+    const config = input.config ?? deps.getConfig();
+    // Fail before spending time on the download when Whisper cannot run.
+    await (deps.requireSetup ?? requireSubtitleGenerationSetup)(config, deps.getModelDirectory());
+    await removeStaleAudioDirs();
+    const directory = await mkdtemp(path.join(tempRoot, YOUTUBE_WHISPER_AUDIO_DIR_PREFIX));
+    activeDirs.add(directory);
+    try {
+      input.onProgress?.({ stage: 'download', percent: 0, message: 'Downloading audio...' });
+      await (deps.runProcess ?? runSubtitleGenerationProcess)({
+        command: (deps.getYtDlpCommand ?? getYoutubeYtDlpCommand)(),
+        args: buildYoutubeWhisperAudioArgs(input.url, path.join(directory, 'audio.%(ext)s')),
+        signal: input.signal,
+        missingMessage: 'yt-dlp was not found. Install it or set SUBMINER_YTDLP_BIN.',
+        onLine: (line) => {
+          const percent = parseYtDlpDownloadPercent(line);
+          if (percent !== null)
+            input.onProgress?.({ stage: 'download', percent, message: 'Downloading audio...' });
+        },
+      });
+      return await (deps.generate ?? generateJapaneseSubtitles)({
+        config,
+        modelDirectory: deps.getModelDirectory(),
+        mediaPath: await findDownloadedAudio(directory),
+        outputPath: input.outputPath,
+        // Keep the extracted WAV inside the managed directory so every cleanup path removes it.
+        workDirectory: directory,
+        onProgress: input.onProgress,
+        signal: input.signal,
+      });
+    } finally {
+      await rm(directory, { recursive: true, force: true }).catch(() => undefined);
+      activeDirs.delete(directory);
+    }
+  }
+
+  /** Synchronous so it still runs while the app is quitting. */
+  function removeActiveAudio(): void {
+    for (const directory of activeDirs) {
+      try {
+        rmSync(directory, { recursive: true, force: true });
+      } catch {
+        // Leftovers are swept before the next download.
+      }
+    }
+    activeDirs.clear();
+  }
+
+  return { generate, removeActiveAudio };
+}