feat(subtitles): use loaded subtitles to guide generation timing (#249)

launcher/commands/generate-subtitles-command.test.ts

@@ -136,25 +136,28 @@ test('current mpv generation requires an identifiable selected audio track', asy
   }
 });
 
-test('explicit local file leaves Japanese track selection to the shared generator', async () => {
+test('explicit playing file leaves audio selection to the generator but inspects subtitle references', async () => {
   const f = fixture();
   await runGenerateSubtitlesCommand(f.context, f.deps);
   assert.equal(f.generations[0]?.audioStreamIndex, undefined);
   assert.equal(
     f.commands.some((command) => command[1] === 'track-list'),
-    false,
+    true,
   );
 });
 
 test('launcher does not load generated subtitles after mpv switches files', async () => {
   const f = fixture(['generate-subs']);
-  let pathRequests = 0;
+  let changed = false;
   f.deps.mpvCommand = async (_socket, command) => {
     f.commands.push(command);
-    if (command[1] === 'path')
-      return ++pathRequests === 1 ? '/media/episode.mkv' : '/media/next.mkv';
+    if (command[1] === 'path') return changed ? '/media/next.mkv' : '/media/episode.mkv';
     return [{ type: 'audio', selected: true, 'ff-index': 2 }];
   };
+  f.deps.generate = async () => {
+    changed = true;
+    return '/media/episode.ja.srt';
+  };
   await runGenerateSubtitlesCommand(f.context, f.deps);
   assert.equal(
     f.commands.some((command) => command[0] === 'sub-add'),
@@ -163,6 +166,86 @@ test('launcher does not load generated subtitles after mpv switches files', asyn
   assert.match(f.output.join(''), /Saved Japanese subtitles/);
 });
 
+test('current mpv generation rejects tracks when playback changes during capture', async () => {
+  for (const nextMedia of ['/media/next.mkv', null]) {
+    const f = fixture(['generate-subs']);
+    let media: string | null = '/media/episode.mkv';
+    let modelChecks = 0;
+    f.deps.mpvCommand = async (_socket, command) => {
+      if (command[1] === 'path') return media;
+      if (command[1] === 'track-list') {
+        media = nextMedia;
+        return [{ type: 'audio', selected: true, 'ff-index': 99 }];
+      }
+      return undefined;
+    };
+    f.deps.resolveModel = async () => {
+      modelChecks += 1;
+      return { kind: 'external', path: '/models/model.bin' };
+    };
+    await assert.rejects(
+      runGenerateSubtitlesCommand(f.context, f.deps),
+      /mpv media.*Run generate-subs again/,
+    );
+    assert.equal(f.generations.length, 0);
+    assert.equal(modelChecks, 0);
+  }
+});
+
+test('explicit media or audio stream remains usable when the mpv snapshot changes', async () => {
+  for (const options of [['/media/episode.mkv'], ['--audio-stream', '7']]) {
+    const f = fixture(['generate-subs', ...options]);
+    let media = '/media/episode.mkv';
+    f.deps.mpvCommand = async (_socket, command) => {
+      if (command[1] === 'path') return media;
+      if (command[1] === 'track-list') {
+        media = '/media/next.mkv';
+        return [
+          { type: 'audio', selected: true, 'ff-index': 99 },
+          { type: 'sub', lang: 'eng', 'ff-index': 100 },
+        ];
+      }
+      return undefined;
+    };
+    await runGenerateSubtitlesCommand(f.context, f.deps);
+    assert.equal(f.generations[0]?.mediaPath, '/media/episode.mkv');
+    assert.equal(
+      f.generations[0]?.audioStreamIndex,
+      options[0] === '--audio-stream' ? 7 : undefined,
+    );
+    assert.deepEqual(f.generations[0]?.references, []);
+  }
+});
+
+test('launcher captures loaded external references only for the matching media', async () => {
+  for (const matching of [true, false]) {
+    const f = fixture();
+    f.deps.mpvCommand = async (_socket, command) => {
+      if (command[1] === 'path') return matching ? '/media/episode.mkv' : '/media/other.mkv';
+      if (command[1] === 'working-directory') return '/mpv';
+      if (command[1] === 'track-list')
+        return [
+          { type: 'sub', external: true, 'external-filename': 'episode.en.signs.ass' },
+          { type: 'sub', external: true, 'external-filename': 'episode.en.srt' },
+        ];
+      return undefined;
+    };
+    await runGenerateSubtitlesCommand(f.context, f.deps);
+    assert.deepEqual(
+      f.generations[0]?.references,
+      matching
+        ? [
+            {
+              label: 'episode.en.srt',
+              delaySeconds: 0,
+              source: { kind: 'external', path: '/mpv/episode.en.srt' },
+            },
+          ]
+        : [],
+    );
+  }
+});
+
 test('explicit managed model overrides external config and downloads before generation', async () => {
   const f = fixture([
     'generate-subs',
@@ -191,6 +274,69 @@ test('explicit managed model overrides external config and downloads before gene
   assert.equal(f.generations.length, 1);
 });
 
+test('audio and subtitle timing use the initial mpv snapshot across model setup', async () => {
+  for (const changeDuring of ['resolve', 'download']) {
+    const f = fixture(['generate-subs', '--download-model']);
+    let changed = false;
+    let trackReads = 0;
+    f.deps.mpvCommand = async (_socket, command) => {
+      if (command[1] === 'path') return '/media/episode.mkv';
+      if (command[1] === 'track-list') {
+        trackReads += 1;
+        return [
+          { type: 'audio', selected: true, 'ff-index': changed ? 3 : 2 },
+          { type: 'sub', id: 1, title: 'English Full', lang: 'eng', 'ff-index': changed ? 5 : 4 },
+        ];
+      }
+      if (command[1] === 'sid') return 1;
+      if (command[1] === 'sub-delay') return changed ? 9 : 1.5;
+      return undefined;
+    };
+    f.deps.resolveModel = async () => {
+      if (changeDuring === 'resolve') changed = true;
+      return { kind: 'missing', path: '/models/model.bin' };
+    };
+    f.deps.downloadModel = async () => {
+      changed = true;
+      return '/models/model.bin';
+    };
+    await runGenerateSubtitlesCommand(f.context, f.deps);
+    assert.equal(f.generations[0]?.audioStreamIndex, 2);
+    assert.deepEqual(f.generations[0]?.references, [
+      {
+        label: 'English Full',
+        delaySeconds: 1.5,
+        source: { kind: 'embedded', streamIndex: 4 },
+      },
+    ]);
+    assert.equal(trackReads, 1);
+  }
+});
+
+test('explicit audio stream bypasses mpv audio selection while retaining subtitle references', async () => {
+  const f = fixture(['generate-subs', '--audio-stream', '7']);
+  f.deps.mpvCommand = async (_socket, command) => {
+    if (command[1] === 'path') return '/media/episode.mkv';
+    if (command[1] === 'track-list')
+      return [
+        { type: 'audio', selected: true, external: true, 'ff-index': 0 },
+        { type: 'sub', id: 2, title: 'English Full', lang: 'eng', 'ff-index': 4 },
+      ];
+    if (command[1] === 'secondary-sid') return 2;
+    if (command[1] === 'secondary-sub-delay') return -0.5;
+    return undefined;
+  };
+  await runGenerateSubtitlesCommand(f.context, f.deps);
+  assert.equal(f.generations[0]?.audioStreamIndex, 7);
+  assert.deepEqual(f.generations[0]?.references, [
+    {
+      label: 'English Full',
+      delaySeconds: -0.5,
+      source: { kind: 'embedded', streamIndex: 4 },
+    },
+  ]);
+});
+
 test('generation can run standalone and never loads subtitles into another video', async () => {
   for (const playing of [null, '/media/different.mkv']) {
     const f = fixture();

launcher/commands/generate-subtitles-command.ts

@@ -8,6 +8,10 @@ import {
   resolveSubtitleGenerationTools,
 } from '../../src/core/services/subtitle-generation.js';
 import { requireSubtitleGenerationTools } from '../../src/core/services/subtitle-generation-tools.js';
+import {
+  readSubtitleGenerationReferences,
+  type SubtitleGenerationReference,
+} from '../../src/core/services/subtitle-generation-reference.js';
 import {
   resolveSubtitleGenerationConfig,
   type SubtitleGenerationProgress,
@@ -69,12 +73,9 @@ async function readMpvMedia(socketPath: string, command: GenerationCommandDeps['
   return localMediaPath(media, workingDirectory);
 }
 
-async function readMpvAudioStream(
-  socketPath: string,
-  command: GenerationCommandDeps['mpvCommand'],
-) {
-  const tracks = await command(socketPath, ['get_property', 'track-list'], 1000);
-  for (const track of Array.isArray(tracks) ? tracks : []) {
+function selectedMpvAudioStream(value: unknown) {
+  const tracks: unknown[] = Array.isArray(value) ? value : [];
+  for (const track of tracks) {
     if (
       typeof track === 'object' &&
       track !== null &&
@@ -160,11 +161,29 @@ export async function runGenerateSubtitlesCommand(
     const mediaPath = options.mediaPath ? localMediaPath(options.mediaPath) : currentMedia;
     if (!mediaPath)
       throw new Error('Pass a local video file or open one in mpv before running generate-subs.');
+    // Capture audio and subtitle timing together before model setup can yield to playback changes.
+    const matchesCurrentMedia = currentMedia !== null && sameFile(currentMedia, mediaPath);
+    const tracks = matchesCurrentMedia
+      ? await deps
+          .mpvCommand(context.mpvSocketPath, ['get_property', 'track-list'], 1000)
+          .catch(() => null)
+      : null;
+    let references: SubtitleGenerationReference[] = [];
+    if (matchesCurrentMedia) {
+      const candidates = await readSubtitleGenerationReferences(tracks, (name) =>
+        deps.mpvCommand(context.mpvSocketPath, ['get_property', name], 1000),
+      );
+      const stillPlaying = await readMpvMedia(context.mpvSocketPath, deps.mpvCommand).catch(
+        () => null,
+      );
+      if (stillPlaying && sameFile(stillPlaying, mediaPath)) references = candidates;
+      else if (!options.mediaPath && options.audioStreamIndex === undefined)
+        throw new Error(
+          'The current mpv media changed or could not be verified while reading tracks. Run generate-subs again.',
+        );
+    }
     const audioStreamIndex =
-      options.audioStreamIndex ??
-      (!options.mediaPath
-        ? await readMpvAudioStream(context.mpvSocketPath, deps.mpvCommand)
-        : undefined);
+      options.audioStreamIndex ?? (!options.mediaPath ? selectedMpvAudioStream(tracks) : undefined);
     const onProgress = createGenerationProgressReporter(write);
     // Missing executables fail here, before any model download starts.
     requireSubtitleGenerationTools(await deps.resolveTools(config));
@@ -183,6 +202,7 @@ export async function runGenerateSubtitlesCommand(
       modelDirectory,
       mediaPath,
       audioStreamIndex,
+      references,
       outputPath: options.outputPath
         ? path.resolve(resolvePathMaybe(options.outputPath))
         : undefined,