feat(subtitles): generate Japanese subtitles during Jellyfin playback (#294)

* feat(subtitles): generate Japanese subtitles during Jellyfin playback

- Read the item's original file via toJellyfinOriginalStreamUrl and write output to a temp dir
- Transcoded playback uses Jellyfin's audio index and only external references
- Show Jellyfin media by title and redact URL tokens in subprocess errors
- Update overlay copy, docs, and changelog fragment

* fix(subtitles): redact full stream tokens and reuse one Jellyfin output folder

* test(subtitles): make Jellyfin output path assertions platform-neutral

- Compare dirname/basename via node:path instead of POSIX-only regexes
- Assert the full sub-add command uses the generated output path

src/core/services/index.ts

@@ -130,6 +130,7 @@ export {
   listSubtitleTracks as listJellyfinSubtitleTracksRuntime,
   resolvePlaybackPlan as resolveJellyfinPlaybackPlanRuntime,
   ticksToSeconds as jellyfinTicksToSecondsRuntime,
+  toJellyfinOriginalStreamUrl,
 } from './jellyfin';
 export { JellyfinRemoteSessionService } from './jellyfin-remote';
 export {

src/core/services/jellyfin.test.ts

@@ -7,6 +7,7 @@ import {
   listSubtitleTracks,
   resolvePlaybackPlan,
   ticksToSeconds,
+  toJellyfinOriginalStreamUrl,
 } from './jellyfin';
 
 const clientInfo = {
@@ -862,3 +863,26 @@ test('resolvePlaybackPlan replaces a legacy api_key on the server transcoding ur
     globalThis.fetch = originalFetch;
   }
 });
+
+test('toJellyfinOriginalStreamUrl maps direct and transcoded playback to the original file', () => {
+  assert.equal(
+    toJellyfinOriginalStreamUrl(
+      'https://media.example/jf/videos/item-1/master.m3u8?ApiKey=token&MediaSourceId=src&VideoCodec=h264&StartTimeTicks=9',
+      'item-1',
+    ),
+    'https://media.example/jf/Videos/item-1/stream?static=true&ApiKey=token&MediaSourceId=src',
+  );
+  assert.equal(
+    toJellyfinOriginalStreamUrl(
+      'http://jf.local:8096/Videos/item-1/stream?static=true&api_key=legacy&AudioStreamIndex=2',
+      'item-1',
+    ),
+    'http://jf.local:8096/Videos/item-1/stream?static=true&ApiKey=legacy',
+  );
+  assert.equal(
+    toJellyfinOriginalStreamUrl('https://jf/Videos/other/stream?ApiKey=token', 'item-1'),
+    null,
+  );
+  assert.equal(toJellyfinOriginalStreamUrl('https://jf/Videos/item-1/stream', 'item-1'), null);
+  assert.equal(toJellyfinOriginalStreamUrl('/media/item-1.mkv', 'item-1'), null);
+});

src/core/services/jellyfin.ts

@@ -254,6 +254,29 @@ function createDirectPlayUrl(
   return `${session.serverUrl}/Videos/${itemId}/stream?${query.toString()}`;
 }
 
+/**
+ * Maps a playing stream URL of `itemId`, direct or transcoded, to its original file. Returns null
+ * when the URL is not that item's stream or carries no access token.
+ */
+export function toJellyfinOriginalStreamUrl(playbackUrl: string, itemId: string): string | null {
+  let url: URL;
+  try {
+    url = new URL(playbackUrl);
+  } catch {
+    return null;
+  }
+  const at = url.pathname.toLowerCase().indexOf(`/videos/${itemId.toLowerCase()}/`);
+  if (!/^https?:$/.test(url.protocol) || !itemId || at < 0) return null;
+  const param = (pattern: RegExp) =>
+    [...url.searchParams].find(([key, value]) => pattern.test(key) && value)?.[1];
+  const accessToken = param(/^api_?key$/i);
+  if (!accessToken) return null;
+  const query = new URLSearchParams({ static: 'true', ApiKey: accessToken });
+  const mediaSourceId = param(/^mediasourceid$/i);
+  if (mediaSourceId) query.set('MediaSourceId', mediaSourceId);
+  return `${url.origin}${url.pathname.slice(0, at)}/Videos/${itemId}/stream?${query.toString()}`;
+}
+
 function createTranscodeUrl(
   session: JellyfinAuthSession,
   itemId: string,

src/core/services/subtitle-generation-process.ts

@@ -3,6 +3,11 @@ import { expandSubtitleGenerationPath } from './subtitle-generation-files';
 
 const OUTPUT_LIMIT = 64 * 1024;
 
+// ffmpeg echoes input URLs in its errors; streamed media carries access tokens in the query.
+function redactUrlCredentials(text: string): string {
+  return text.replace(/([?&](?:api_?key|token|access_token)=)[^&#\s'"]+/gi, '$1<redacted>');
+}
+
 // Keep partial lines between chunks: ffmpeg and whisper both report progress on stderr.
 export function runSubtitleGenerationProcess(input: {
   command: string;
@@ -63,7 +68,11 @@ export function runSubtitleGenerationProcess(input: {
       cleanup();
       if (input.signal?.aborted) reject(new Error('Subtitle generation cancelled.'));
       else if (code !== 0) {
-        reject(new Error(`${input.command} exited with status ${code}: ${stderr.trim()}`));
+        reject(
+          new Error(
+            `${input.command} exited with status ${code}: ${redactUrlCredentials(stderr.trim())}`,
+          ),
+        );
       } else resolve(stdout);
     });
   });

src/core/services/subtitle-generation.test.ts

@@ -378,7 +378,22 @@ test('empty executable paths find tools on PATH and explicit overrides take prec
     }
   }));
 
-test('generation rejects remote media, missing models, and missing tools before starting a subprocess', () =>
+test('generation reads an HTTP stream as-is and saves to the requested output', () =>
+  fixture(async (directory) => {
+    const input = await generationFixture(directory);
+    const mediaPath = 'https://jellyfin.example/Videos/item/stream?static=true&ApiKey=secret';
+    const outputPath = path.join(directory, 'out', 'stream.ja.srt');
+    await mkdir(path.dirname(outputPath));
+    assert.equal(await generateJapaneseSubtitles({ ...input, mediaPath, outputPath }), outputPath);
+    const calls: string[][] = (await readFile(input.callsPath, 'utf8'))
+      .trim()
+      .split('\n')
+      .map((line) => JSON.parse(line));
+    assert.equal(calls[0]?.at(-1), mediaPath);
+    assert.equal(calls[1]?.[calls[1].indexOf('-i') + 1], mediaPath);
+  }));
+
+test('generation rejects remote media without an output path, missing models, and missing tools before starting a subprocess', () =>
   fixture(async (directory) => {
     const input = await generationFixture(directory);
     await assert.rejects(
@@ -473,6 +488,17 @@ test('process cancellation terminates work and bounds diagnostic output', () =>
         error.message.length < 66000 &&
         error.message.includes('status 7'),
     );
+    const unauthorized = await executable(
+      directory,
+      'unauthorized',
+      "process.stderr.write('https://host/Videos/1/stream?ApiKey=se+cr/et==&static=true: Server returned 401');process.exitCode=1;",
+    );
+    await assert.rejects(
+      runSubtitleGenerationProcess({ command: unauthorized, args: [] }),
+      (error: unknown) =>
+        error instanceof Error &&
+        error.message.includes('?ApiKey=<redacted>&static=true: Server returned 401'),
+    );
   }));
 
 test('download uses the pinned model revision and removes files that fail integrity', () =>

src/core/services/subtitle-generation.ts

@@ -191,6 +191,7 @@ export async function requireSubtitleGenerationSetup(
 export async function generateJapaneseSubtitles(input: {
   config: SubtitleGenerationConfig;
   modelDirectory: string;
+  /** A local file, or an http(s) stream when `outputPath` is set. */
   mediaPath: string;
   audioStreamIndex?: number;
   references?: readonly SubtitleGenerationReference[];
@@ -201,10 +202,12 @@ export async function generateJapaneseSubtitles(input: {
   signal?: AbortSignal;
 }): Promise<string> {
   input.signal?.throwIfAborted();
-  if (/^[a-z][a-z\d+.-]*:\/\//i.test(input.mediaPath))
+  // HTTP streams are read by ffmpeg directly but have no directory to save beside.
+  const remote = /^https?:\/\//i.test(input.mediaPath);
+  if (remote ? !input.outputPath : /^[a-z][a-z\d+.-]*:\/\//i.test(input.mediaPath))
     throw new Error('Subtitle generation requires a local media file.');
-  const mediaPath = path.resolve(input.mediaPath);
-  if (!(await stat(mediaPath)).isFile())
+  const mediaPath = remote ? input.mediaPath : path.resolve(input.mediaPath);
+  if (!remote && !(await stat(mediaPath)).isFile())
     throw new Error('Subtitle generation requires a local media file.');
   if (input.outputPath) await ensureAvailableOutput(path.resolve(input.outputPath));
   await ensureWritableDirectory(

src/main.ts

@@ -310,6 +310,7 @@ import {
   replayCurrentSubtitleRuntime,
   resolveSanitizedSubtitleSeekCommand,
   resolveJellyfinPlaybackPlanRuntime,
+  toJellyfinOriginalStreamUrl,
   runStartupBootstrapRuntime,
   saveSubtitlePosition as saveSubtitlePositionCore,
   clearYomitanParserCachesForWindow,
@@ -6723,6 +6724,17 @@ const subtitleGenerationRuntime = createSubtitleGenerationRuntime({
     }),
   resolveYoutubeSourceUrl: (mediaPath) =>
     youtubePlaybackRuntime.getYoutubeSourceUrlForStream(mediaPath),
+  resolveJellyfinMedia: (mediaPath) => {
+    const playback = activeJellyfinRemotePlayback;
+    const sourceUrl = playback && toJellyfinOriginalStreamUrl(mediaPath, playback.itemId);
+    return sourceUrl
+      ? {
+          sourceUrl,
+          transcoded: playback.playMethod === 'Transcode',
+          audioStreamIndex: playback.audioStreamIndex ?? null,
+        }
+      : null;
+  },
   getMpvClient: () => appState.mpvClient,
   onProgress: (progress) => {
     for (const window of [overlayManager.getMainWindow(), overlayManager.getModalWindow()]) {

src/main/runtime/subtitle-generation-runtime.test.ts

@@ -1,8 +1,10 @@
 import assert from 'node:assert/strict';
+import path from 'node:path';
 import test from 'node:test';
 import { DEFAULT_SUBTITLE_GENERATION_CONFIG } from '../../shared/subtitle-generation';
 import {
   createSubtitleGenerationRuntime,
+  type SubtitleGenerationJellyfinMedia,
   type SubtitleGenerationRuntimeDeps,
 } from './subtitle-generation-runtime';
 
@@ -506,3 +508,101 @@ test('switching to another resolved YouTube stream cancels the modal job', async
 
   assert.deepEqual(await job, { ok: false, message: 'Cancelled.' });
 });
+
+function jellyfinFixture(
+  jellyfin: Omit<SubtitleGenerationJellyfinMedia, 'sourceUrl'>,
+  overrides: Partial<SubtitleGenerationRuntimeDeps> = {},
+) {
+  const playing = 'https://jf.example/Videos/item/master.m3u8?ApiKey=secret';
+  const subject = fixture({
+    resolveJellyfinMedia: (mediaPath) =>
+      mediaPath === playing
+        ? {
+            sourceUrl: 'https://jf.example/Videos/item/stream?static=true&ApiKey=secret',
+            ...jellyfin,
+          }
+        : null,
+    createStreamOutputDirectory: async () => '/tmp/generated',
+    ...overrides,
+  });
+  subject.client.requestProperty = async (name) =>
+    name === 'path'
+      ? playing
+      : name === 'media-title'
+        ? 'Show S01E02'
+        : name === 'track-list'
+          ? [
+              { type: 'audio', selected: true, 'ff-index': 0 },
+              { type: 'sub', lang: 'eng', title: 'English Full', 'ff-index': 4 },
+              {
+                type: 'sub',
+                lang: 'eng',
+                title: 'English',
+                external: true,
+                'external-filename': '/tmp/jellyfin-subs/track-5.srt',
+              },
+            ]
+          : null;
+  return subject;
+}
+
+test('a direct-play Jellyfin stream generates from its original file and shows by title', async () => {
+  let outputPath = '';
+  const subject = jellyfinFixture(
+    { transcoded: false, audioStreamIndex: 2 },
+    {
+      generate: async (input) => {
+        assert.equal(
+          input.mediaPath,
+          'https://jf.example/Videos/item/stream?static=true&ApiKey=secret',
+        );
+        outputPath = input.outputPath ?? '';
+        assert.equal(path.dirname(outputPath), path.join('/tmp', 'generated'));
+        assert.match(path.basename(outputPath), /^jellyfin-\d+\.ja\.generated\.srt$/);
+        assert.equal(input.audioStreamIndex, 0);
+        assert.deepEqual(
+          input.references?.map((reference) => reference.source.kind),
+          ['embedded', 'external'],
+        );
+        return outputPath;
+      },
+    },
+  );
+  assert.equal((await subject.runtime.getStatus()).mediaPath, 'Jellyfin: Show S01E02');
+  assert.equal((await subject.runtime.start()).ok, true);
+  assert.deepEqual(subject.commands[0], [
+    'sub-add',
+    outputPath,
+    'select',
+    'Generated Japanese',
+    'ja',
+  ]);
+});
+
+test('a transcoded Jellyfin stream uses the Jellyfin audio index and only external references', async () => {
+  const subject = jellyfinFixture(
+    { transcoded: true, audioStreamIndex: 2 },
+    {
+      generate: async (input) => {
+        assert.equal(input.audioStreamIndex, 2);
+        assert.deepEqual(
+          input.references?.map((reference) => reference.source.kind),
+          ['external'],
+        );
+        return '/tmp/generated/jellyfin.ja.generated.srt';
+      },
+    },
+  );
+  assert.equal((await subject.runtime.start()).ok, true);
+});
+
+test('unrecognized streams cannot generate', async () => {
+  const subject = jellyfinFixture(
+    { transcoded: false, audioStreamIndex: null },
+    { resolveJellyfinMedia: () => null },
+  );
+  assert.equal((await subject.runtime.getStatus()).mediaPath, null);
+  const result = await subject.runtime.start();
+  assert.equal(result.ok, false);
+  assert.match(result.message, /Jellyfin stream/);
+});

src/main/runtime/subtitle-generation-runtime.ts

@@ -1,3 +1,5 @@
+import { mkdir, readdir, rm } from 'node:fs/promises';
+import { homedir } from 'node:os';
 import path from 'node:path';
 import { readSubtitleGenerationReferences } from '../../core/services/subtitle-generation-reference';
 import { detectSubtitleGenerationAcceleration } from '../../core/services/subtitle-generation-acceleration';
@@ -29,6 +31,22 @@ interface GenerationMpvClient {
   request: (command: unknown[]) => Promise<{ error?: string }>;
 }
 
+/** A Jellyfin stream playing in mpv, as subtitle generation reads it. */
+export interface SubtitleGenerationJellyfinMedia {
+  /** The item's original file. It carries the access token, so it never reaches status text. */
+  sourceUrl: string;
+  /**
+   * Transcoded playback has its own stream layout, so mpv's track indexes do not address the
+   * original file. Generation then uses Jellyfin's audio index and skips embedded references.
+   */
+  transcoded: boolean;
+  audioStreamIndex: number | null;
+}
+
+type GenerationMedia =
+  | { kind: 'local'; path: string }
+  | { kind: 'jellyfin'; path: string; jellyfin: SubtitleGenerationJellyfinMedia };
+
 export interface SubtitleGenerationRuntimeDeps {
   getConfig: () => SubtitleGenerationConfig;
   getModelDirectory: () => string;
@@ -56,11 +74,27 @@ export interface SubtitleGenerationRuntimeDeps {
    * resolved direct stream in mpv, so mpv's path alone does not identify the video.
    */
   resolveYoutubeSourceUrl?: (mediaPath: string) => string | null;
+  /** Identifies mpv's path as Jellyfin playback. Without it, streams cannot generate. */
+  resolveJellyfinMedia?: (mediaPath: string) => SubtitleGenerationJellyfinMedia | null;
+  /**
+   * Directory for subtitles of streamed media, which has no folder of its own. Called once per
+   * stream generation, so it may delete what an earlier run wrote there.
+   */
+  createStreamOutputDirectory?: () => Promise<string>;
 }
 
-async function currentLocalMedia(client: GenerationMpvClient | null): Promise<string | null> {
-  if (!client?.connected) return null;
-  const media = await client.requestProperty('path');
+// One reused folder keeps stream subtitles from piling up: each run deletes earlier files.
+// mpv has already parsed a loaded track, and a file still locked (Windows) waits for a later run.
+async function reuseStreamOutputDirectory(): Promise<string> {
+  const directory = path.join(homedir(), '.cache', 'subminer', 'generated-subs');
+  await mkdir(directory, { recursive: true });
+  for (const name of await readdir(directory))
+    await rm(path.join(directory, name), { recursive: true, force: true }).catch(() => undefined);
+  return directory;
+}
+
+// Resolves mpv's `path` value to an absolute local file, or null for URLs.
+async function localMediaPath(client: GenerationMpvClient, media: unknown): Promise<string | null> {
   if (typeof media !== 'string' || !media || /^[a-z][a-z\d+.-]*:\/\//i.test(media)) return null;
   if (path.isAbsolute(media)) return path.normalize(media);
   const directory = await client.requestProperty('working-directory');
@@ -139,6 +173,22 @@ export function createSubtitleGenerationRuntime(deps: SubtitleGenerationRuntimeD
   function youtubeSource(mediaPath: string | null | undefined): string | null | undefined {
     return (mediaPath && deps.resolveYoutubeSourceUrl?.(mediaPath)) || mediaPath;
   }
+  async function currentMedia(client: GenerationMpvClient | null): Promise<GenerationMedia | null> {
+    if (!client?.connected) return null;
+    const media = await client.requestProperty('path');
+    if (typeof media === 'string' && /^https?:\/\//i.test(media)) {
+      const jellyfin = deps.resolveJellyfinMedia?.(media);
+      return jellyfin ? { kind: 'jellyfin', path: media, jellyfin } : null;
+    }
+    const local = await localMediaPath(client, media);
+    return local ? { kind: 'local', path: local } : null;
+  }
+  // Stream URLs carry access tokens, so Jellyfin media shows by title.
+  async function describeMedia(client: GenerationMpvClient, media: GenerationMedia) {
+    if (media.kind === 'local') return media.path;
+    const title = await client.requestProperty('media-title').catch(() => null);
+    return `Jellyfin: ${typeof title === 'string' && title.trim() ? title.trim() : 'current item'}`;
+  }
   async function currentYoutubeMedia(client: GenerationMpvClient | null): Promise<string | null> {
     if (!client?.connected) return null;
     const media = await client.requestProperty('path');
@@ -196,9 +246,10 @@ export function createSubtitleGenerationRuntime(deps: SubtitleGenerationRuntimeD
       deps.getModelDirectory(),
     );
     const client = deps.getMpvClient();
+    const media = client ? await currentMedia(client).catch(() => null) : null;
     const mediaPath =
       youtubeJobUrl ??
-      (await currentLocalMedia(client).catch(() => null)) ??
+      (client && media ? await describeMedia(client, media) : null) ??
       (deps.generateYoutube ? await currentYoutubeMedia(client).catch(() => null) : null);
     return {
       model,
@@ -350,21 +401,33 @@ export function createSubtitleGenerationRuntime(deps: SubtitleGenerationRuntimeD
         const youtubeUrl = deps.generateYoutube ? await currentYoutubeMedia(client) : null;
         if (client && youtubeUrl && deps.generateYoutube)
           return await startYoutube(client, youtubeUrl, config, signal, deps.generateYoutube);
-        const mediaPath = await currentLocalMedia(client);
-        if (!client || !mediaPath)
-          throw new Error('Open a local video or audio file, or a YouTube video, in mpv first.');
+        const media = await currentMedia(client);
+        if (!client || !media)
+          throw new Error(
+            'Open a local video or audio file, a Jellyfin stream, or a YouTube video in mpv first.',
+          );
         const tracks = await client.requestProperty('track-list');
-        const audioStreamIndex = selectedAudioIndex(tracks);
-        const references = await readSubtitleGenerationReferences(tracks, (name) =>
-          client.requestProperty(name),
-        );
-        if ((await currentLocalMedia(client)) !== mediaPath)
+        const transcoded = media.kind === 'jellyfin' && media.jellyfin.transcoded;
+        const audioStreamIndex = transcoded
+          ? (media.jellyfin.audioStreamIndex ?? undefined)
+          : selectedAudioIndex(tracks);
+        const references = (
+          await readSubtitleGenerationReferences(tracks, (name) => client.requestProperty(name))
+        ).filter((reference) => !transcoded || reference.source.kind === 'external');
+        if ((await currentMedia(client))?.path !== media.path)
           throw new Error('The current media changed. Start generation again.');
         signal.throwIfAborted();
         const outputPath = await (deps.generate ?? generateJapaneseSubtitles)({
           config,
           modelDirectory: deps.getModelDirectory(),
-          mediaPath,
+          mediaPath: media.kind === 'jellyfin' ? media.jellyfin.sourceUrl : media.path,
+          outputPath:
+            media.kind === 'jellyfin'
+              ? path.join(
+                  await (deps.createStreamOutputDirectory ?? reuseStreamOutputDirectory)(),
+                  `jellyfin-${Date.now()}.ja.generated.srt`,
+                )
+              : undefined,
           audioStreamIndex,
           references,
           onProgress: report,
@@ -372,8 +435,8 @@ export function createSubtitleGenerationRuntime(deps: SubtitleGenerationRuntimeD
         });
         // Saving succeeds even if playback changes or disconnects during the job.
         try {
-          const playingMedia = await currentLocalMedia(client);
-          if (!signal.aborted && deps.getMpvClient() === client && playingMedia === mediaPath) {
+          const playingMedia = (await currentMedia(client))?.path;
+          if (!signal.aborted && deps.getMpvClient() === client && playingMedia === media.path) {
             await loadGeneratedSubtitles(client, outputPath);
             return {
               ok: true,

src/renderer/modals/subtitle-generation.ts

@@ -88,7 +88,7 @@ export function createSubtitleGenerationModal(
     const vad = snapshot ? describeGenerationVad(snapshot.vad) : null;
     const tools = snapshot ? describeGenerationTools(snapshot.tools) : null;
     const readyMessage = !snapshot?.mediaPath
-      ? 'Open a local file or a YouTube video to generate subtitles.'
+      ? 'Open a local file, a Jellyfin stream, or a YouTube video to generate subtitles.'
       : !tools?.ready
         ? 'Install the missing tools or set their paths in Settings, then click Check again.'
         : !model?.ready
@@ -97,7 +97,8 @@ export function createSubtitleGenerationModal(
             ? 'Download the speech detection model or uncheck Focus on spoken dialogue.'
             : 'Ready when you are.';
     dom.media.textContent =
-      snapshot?.mediaPath ?? 'Open a local file or a YouTube video in the player first.';
+      snapshot?.mediaPath ??
+      'Open a local file, a Jellyfin stream, or a YouTube video in the player first.';
     dom.tools.textContent = tools?.text ?? 'Checking local tools...';
     dom.model.textContent = model?.text ?? 'Checking local models...';
     dom.modelPicker.classList.toggle('hidden', !snapshot || Boolean(snapshot.externalModelPath));