fix(subtitles): allow Whisper generation during Windows YouTube stream playback (#282)

changes/whisper-windows-youtube-stream.md

@@ -0,0 +1,4 @@
+type: fixed
+area: youtube
+
+- The subtitle generation modal now works during Windows YouTube playback, where mpv plays a resolved stream URL instead of the YouTube page; it no longer asks you to open a local file or YouTube video, and generated subtitles load into the playing video.

src/main.ts

@@ -6818,6 +6818,8 @@ const subtitleGenerationRuntime = createSubtitleGenerationRuntime({
         'youtube-whisper.ja.srt',
       ),
     }),
+  resolveYoutubeSourceUrl: (mediaPath) =>
+    youtubePlaybackRuntime.getYoutubeSourceUrlForStream(mediaPath),
   getMpvClient: () => appState.mpvClient,
   onProgress: (progress) => {
     for (const window of [overlayManager.getMainWindow(), overlayManager.getModalWindow()]) {

src/main/runtime/subtitle-generation-runtime.test.ts

@@ -368,7 +368,10 @@ test('cancelling a YouTube Whisper job resolves to null, and failures throw', as
   );
 });
 
-function youtubeFixture(generateYoutube: SubtitleGenerationRuntimeDeps['generateYoutube']) {
+function youtubeFixture(
+  generateYoutube: SubtitleGenerationRuntimeDeps['generateYoutube'],
+  overrides: Partial<SubtitleGenerationRuntimeDeps> = {},
+) {
   const player = {
     path: 'https://www.youtube.com/watch?v=abcdefghijk&pp=search',
     paused: false,
@@ -385,7 +388,7 @@ function youtubeFixture(generateYoutube: SubtitleGenerationRuntimeDeps['generate
       return { error: 'success' };
     },
   };
-  const { runtime } = fixture({ getMpvClient: () => client, generateYoutube });
+  const { runtime } = fixture({ getMpvClient: () => client, generateYoutube, ...overrides });
   return { runtime, player, commands };
 }
 
@@ -443,3 +446,63 @@ test('switching videos cancels a YouTube job from the modal without loading its
     false,
   );
 });
+
+// Windows plays YouTube through a resolved googlevideo stream, so mpv's path is not a YouTube URL.
+const STREAM_URL = 'https://rr1---sn.example.googlevideo.com/videoplayback?expire=1777777777';
+const OTHER_STREAM_URL = 'https://rr2---sn.example.googlevideo.com/videoplayback?expire=1888888888';
+const resolveStreamSource = (mediaPath: string): string | null =>
+  mediaPath === STREAM_URL
+    ? 'https://www.youtube.com/watch?v=abcdefghijk'
+    : mediaPath === OTHER_STREAM_URL
+      ? 'https://www.youtube.com/watch?v=zyxwvutsrqp'
+      : null;
+
+test('the modal generates for a YouTube video playing through a resolved stream URL', async () => {
+  let jobSignal: AbortSignal | null = null;
+  const { runtime, player, commands } = youtubeFixture(
+    async (input) => {
+      jobSignal = input.signal;
+      assert.equal(input.url, 'https://www.youtube.com/watch?v=abcdefghijk');
+      // Reloading the same video's stream must not cancel the job.
+      runtime.handleMediaPathChange(STREAM_URL);
+      return '/tmp/subs/youtube-whisper.ja.srt';
+    },
+    { resolveYoutubeSourceUrl: resolveStreamSource },
+  );
+  player.path = STREAM_URL;
+
+  assert.equal(
+    (await runtime.getStatus()).mediaPath,
+    'https://www.youtube.com/watch?v=abcdefghijk',
+  );
+  assert.deepEqual(await runtime.start(), {
+    ok: true,
+    outputPath: '/tmp/subs/youtube-whisper.ja.srt',
+    message: 'Japanese subtitles generated and loaded.',
+  });
+  assert.equal((jobSignal as AbortSignal | null)?.aborted, false);
+  assert.deepEqual(
+    commands.find((command) => command[0] === 'sub-add'),
+    ['sub-add', '/tmp/subs/youtube-whisper.ja.srt', 'select', 'Generated Japanese', 'ja'],
+  );
+});
+
+test('switching to another resolved YouTube stream cancels the modal job', async () => {
+  let jobSignal: AbortSignal | null = null;
+  const { runtime, player } = youtubeFixture(
+    (input) =>
+      new Promise((_, reject) => {
+        jobSignal = input.signal;
+        input.signal.addEventListener('abort', () => reject(new Error('Aborted')), { once: true });
+      }),
+    { resolveYoutubeSourceUrl: resolveStreamSource },
+  );
+  player.path = STREAM_URL;
+
+  const job = runtime.start();
+  while (!jobSignal) await new Promise((resolve) => setImmediate(resolve));
+  player.path = OTHER_STREAM_URL;
+  runtime.handleMediaPathChange(player.path);
+
+  assert.deepEqual(await job, { ok: false, message: 'Cancelled.' });
+});

src/main/runtime/subtitle-generation-runtime.ts

@@ -51,6 +51,11 @@ export interface SubtitleGenerationRuntimeDeps {
     signal: AbortSignal;
     onProgress: (progress: SubtitleGenerationProgress) => void;
   }) => Promise<string>;
+  /**
+   * Maps a playing stream URL back to its YouTube page URL. Windows YouTube playback opens a
+   * resolved direct stream in mpv, so mpv's path alone does not identify the video.
+   */
+  resolveYoutubeSourceUrl?: (mediaPath: string) => string | null;
 }
 
 async function currentLocalMedia(client: GenerationMpvClient | null): Promise<string | null> {
@@ -62,12 +67,6 @@ async function currentLocalMedia(client: GenerationMpvClient | null): Promise<st
   return typeof directory === 'string' ? path.resolve(directory, media) : null;
 }
 
-async function currentYoutubeMedia(client: GenerationMpvClient | null): Promise<string | null> {
-  if (!client?.connected) return null;
-  const media = await client.requestProperty('path');
-  return typeof media === 'string' ? toYoutubeWatchUrl(media) : null;
-}
-
 async function loadGeneratedSubtitles(client: GenerationMpvClient, outputPath: string) {
   const loaded = await client.request([
     'sub-add',
@@ -136,6 +135,15 @@ export function createSubtitleGenerationRuntime(deps: SubtitleGenerationRuntimeD
             : '',
     };
   }
+  // The YouTube page behind a resolved stream URL, or the media path itself.
+  function youtubeSource(mediaPath: string | null | undefined): string | null | undefined {
+    return (mediaPath && deps.resolveYoutubeSourceUrl?.(mediaPath)) || mediaPath;
+  }
+  async function currentYoutubeMedia(client: GenerationMpvClient | null): Promise<string | null> {
+    if (!client?.connected) return null;
+    const media = await client.requestProperty('path');
+    return typeof media === 'string' ? toYoutubeWatchUrl(youtubeSource(media)) : null;
+  }
   const report = (update: SubtitleGenerationProgress) => {
     progress = update;
     deps.onProgress(update);
@@ -246,7 +254,7 @@ export function createSubtitleGenerationRuntime(deps: SubtitleGenerationRuntimeD
     /** Stops a YouTube job once its video is no longer playing, which also deletes its audio. */
     handleMediaPathChange(mediaPath: string | null | undefined): void {
       if (!youtubeJobUrl) return;
-      if (extractYoutubeVideoId(mediaPath) !== extractYoutubeVideoId(youtubeJobUrl))
+      if (extractYoutubeVideoId(youtubeSource(mediaPath)) !== extractYoutubeVideoId(youtubeJobUrl))
         controller?.abort();
     },
     async setVadEnabled(enabled: boolean): Promise<SubtitleGenerationStatus> {

src/main/runtime/youtube-playback-runtime.test.ts

@@ -145,6 +145,49 @@ test('youtube playback runtime resolves the socket path lazily for windows start
   assert.ok(calls.some((entry) => entry.includes('--input-ipc-server=/tmp/updated.sock')));
 });
 
+test('youtube playback runtime maps resolved windows streams back to their page urls', async () => {
+  const streamFor = (url: string) =>
+    `https://rr1---sn.example.googlevideo.com/videoplayback?src=${encodeURIComponent(url)}`;
+  const firstUrl = 'https://www.youtube.com/watch?v=abcdefghijk';
+  const secondUrl = 'https://www.youtube.com/watch?v=bcdefghijkl';
+  const runtime = createYoutubePlaybackRuntime({
+    platform: 'win32',
+    directPlaybackFormat: 'b',
+    mpvYtdlFormat: 'bestvideo+bestaudio',
+    autoLaunchTimeoutMs: 2_000,
+    connectTimeoutMs: 1_000,
+    getSocketPath: () => '/tmp/mpv.sock',
+    getMpvConnected: () => true,
+    invalidatePendingAutoplayReadyFallbacks: () => {},
+    setAppOwnedFlowInFlight: () => {},
+    ensureYoutubePlaybackRuntimeReady: async () => {},
+    resolveYoutubePlaybackUrl: async (url) => streamFor(url),
+    launchWindowsMpv: async () => ({ ok: false }),
+    waitForYoutubeMpvConnected: async () => true,
+    // The second video never loads, so the first stream keeps playing.
+    prepareYoutubePlaybackInMpv: async ({ url }) => url === streamFor(firstUrl),
+    runYoutubePlaybackFlow: async () => {},
+    logInfo: () => {},
+    logWarn: () => {},
+    schedule: () => 1 as never,
+    clearScheduled: () => {},
+  });
+
+  assert.equal(runtime.getYoutubeSourceUrlForStream(streamFor(firstUrl)), null);
+  await runtime.runYoutubePlaybackFlow({ url: firstUrl, source: 'second-instance' });
+  await runtime
+    .runYoutubePlaybackFlow({ url: secondUrl, source: 'second-instance' })
+    .catch(() => {});
+
+  assert.equal(runtime.getYoutubeSourceUrlForStream(streamFor(firstUrl)), firstUrl);
+  assert.equal(runtime.getYoutubeSourceUrlForStream(streamFor(secondUrl)), secondUrl);
+  assert.equal(
+    runtime.getYoutubeSourceUrlForStream('https://rr2---sn.example.googlevideo.com/other'),
+    null,
+  );
+  assert.equal(runtime.getYoutubeSourceUrlForStream('/video/episode.mkv'), null);
+});
+
 test('youtube playback runtime starts media cache without blocking the subtitle flow', async () => {
   const calls: string[] = [];
   let resolveCache: (() => void) | undefined;

src/main/runtime/youtube-playback-runtime.ts

@@ -35,6 +35,10 @@ export function createYoutubePlaybackRuntime(deps: YoutubePlaybackRuntimeDeps) {
   let quitOnDisconnectArmed = false;
   let quitOnDisconnectArmTimer: ReturnType<typeof setTimeout> | null = null;
   let playbackFlowGeneration = 0;
+  // Windows mpv plays resolved stream URLs, so remember which YouTube page each came from. Kept for
+  // every resolved stream: a later failed load must not orphan the stream still playing, and mpv
+  // can return to an earlier swapped-in playlist entry.
+  const directPlaybackSources = new Map<string, string>();
 
   const clearYoutubePlayQuitOnDisconnectArmTimer = (): void => {
     if (quitOnDisconnectArmTimer) {
@@ -65,6 +69,7 @@ export function createYoutubePlaybackRuntime(deps: YoutubePlaybackRuntimeDeps) {
             request.url,
             deps.directPlaybackFormat,
           );
+          directPlaybackSources.set(playbackUrl, request.url);
           deps.logInfo('Resolved direct YouTube playback URL for Windows MPV startup.');
         } catch (error) {
           deps.logWarn(
@@ -156,5 +161,8 @@ export function createYoutubePlaybackRuntime(deps: YoutubePlaybackRuntimeDeps) {
     clearYoutubePlayQuitOnDisconnectArmTimer,
     getQuitOnDisconnectArmed: (): boolean => quitOnDisconnectArmed,
     runYoutubePlaybackFlow,
+    /** YouTube page URL for a stream URL this runtime resolved for Windows playback, else null. */
+    getYoutubeSourceUrlForStream: (mediaPath: string | null | undefined): string | null =>
+      directPlaybackSources.get(mediaPath?.trim() ?? '') ?? null,
   };
 }