feat(subtitles): add local Japanese subtitle generation (#240)

src/renderer/modals/session-help-sections.ts

@@ -225,6 +225,8 @@ function describeSessionAction(
       return 'Open jimaku';
     case 'openTsukihime':
       return 'Open TsukiHime';
+    case 'openSubtitleGeneration':
+      return 'Generate Japanese subtitles';
     case 'openYoutubePicker':
       return 'Open YouTube subtitle picker';
     case 'openPlaylistBrowser':
@@ -266,6 +268,7 @@ function sectionForSessionBinding(binding: CompiledSessionBinding): string {
     case 'openJimaku':
     case 'openTsukihime':
     case 'openCharacterDictionaryManager':
+    case 'openSubtitleGeneration':
     case 'openControllerSelect':
     case 'openControllerDebug':
     case 'openYoutubePicker':

src/renderer/modals/subtitle-generation-view.test.ts

@@ -0,0 +1,99 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import {
+  describeGenerationModel,
+  describeGenerationProgress,
+  describeGenerationTools,
+  describeGenerationVad,
+} from './subtitle-generation-view';
+
+test('missing tools block generation and list every install instruction', () => {
+  const found = { kind: 'found', path: '/usr/bin/tool' } as const;
+  assert.deepEqual(
+    describeGenerationTools({ ffmpeg: found, ffprobe: found, whisper: found, vad: null }),
+    {
+      ready: true,
+      text: 'whisper.cpp and FFmpeg are installed.',
+    },
+  );
+  assert.deepEqual(
+    describeGenerationTools({
+      ffmpeg: { kind: 'missing', message: 'ffmpeg was not found on PATH.' },
+      ffprobe: found,
+      whisper: found,
+      vad: { kind: 'missing', message: 'whisper-vad-speech-segments was not found on PATH.' },
+    }),
+    {
+      ready: false,
+      text: 'ffmpeg was not found on PATH. whisper-vad-speech-segments was not found on PATH.',
+    },
+  );
+});
+
+test('only a missing managed model offers a download', () => {
+  assert.deepEqual(describeGenerationModel({ kind: 'missing', path: '/models/small.bin' }), {
+    ready: false,
+    download: true,
+    text: 'Download a speech model to get started.',
+  });
+  for (const kind of ['managed', 'external'] as const) {
+    const model = describeGenerationModel({ kind, path: '/models/ggml-small.bin' });
+    assert.equal(model.ready, true);
+    assert.equal(model.download, false);
+  }
+  assert.deepEqual(
+    describeGenerationModel({
+      kind: 'invalid',
+      path: '/missing/model.bin',
+      message: 'Configured model does not exist.',
+    }),
+    {
+      ready: false,
+      download: false,
+      text: 'Configured model does not exist.',
+    },
+  );
+});
+
+test('generation progress distinguishes measured work from indeterminate stages', () => {
+  assert.deepEqual(
+    describeGenerationProgress({ stage: 'transcribe', percent: 42.8, message: 'Transcribing' }),
+    {
+      stage: 'Recognizing Japanese speech',
+      percent: 42.8,
+      label: '42%',
+    },
+  );
+  assert.deepEqual(describeGenerationProgress({ stage: 'extract', message: 'Extracting audio' }), {
+    stage: 'Preparing audio',
+    percent: null,
+    label: 'Working...',
+  });
+  assert.equal(
+    describeGenerationProgress({ stage: 'download', percent: 0, message: '' }).label,
+    '0%',
+  );
+  assert.equal(
+    describeGenerationProgress({ stage: 'download', percent: Number.NaN, message: '' }).percent,
+    null,
+  );
+  assert.equal(
+    describeGenerationProgress({ stage: 'write', percent: 120, message: '' }).percent,
+    100,
+  );
+});
+
+test('optional speech detection only gates generation when selected', () => {
+  const missing = { kind: 'missing', path: '/vad.bin' } as const;
+  assert.equal(describeGenerationVad({ enabled: false, model: missing }).ready, true);
+  assert.equal(describeGenerationVad({ enabled: false, model: missing }).download, false);
+  assert.equal(describeGenerationVad({ enabled: true, model: missing }).ready, false);
+  assert.equal(describeGenerationVad({ enabled: true, model: missing }).download, true);
+  assert.equal(
+    describeGenerationVad({ enabled: true, model: { kind: 'managed', path: '/vad.bin' } }).ready,
+    true,
+  );
+  const invalid = { kind: 'invalid', path: '/vad.bin', message: 'Cannot read model' } as const;
+  assert.equal(describeGenerationVad({ enabled: true, model: invalid }).download, false);
+  assert.equal(describeGenerationVad({ enabled: false, model: invalid }).ready, true);
+});

src/renderer/modals/subtitle-generation-view.ts

@@ -0,0 +1,70 @@
+import {
+  missingSubtitleGenerationTools,
+  type SubtitleGenerationModelStatus,
+  type SubtitleGenerationProgress,
+  type SubtitleGenerationTools,
+} from '../../shared/subtitle-generation';
+import type { SubtitleGenerationStatus } from '../../shared/subtitle-generation-ipc';
+
+export function describeGenerationTools(tools: SubtitleGenerationTools) {
+  const missing = missingSubtitleGenerationTools(tools);
+  if (missing.length > 0) return { ready: false, text: missing.join(' ') };
+  return {
+    ready: true,
+    text: tools.vad
+      ? 'whisper.cpp, its speech detector, and FFmpeg are installed.'
+      : 'whisper.cpp and FFmpeg are installed.',
+  };
+}
+
+export function describeGenerationVad(vad: SubtitleGenerationStatus['vad']) {
+  const model = describeGenerationModel(vad.model);
+  return {
+    ready: !vad.enabled || model.ready,
+    download: vad.enabled && model.download,
+    text: !vad.enabled
+      ? 'Optional. Generate from the full audio when unchecked.'
+      : vad.model.kind === 'missing'
+        ? 'Download the speech detection model to focus on spoken dialogue.'
+        : vad.model.kind === 'invalid'
+          ? vad.model.message
+          : vad.model.kind === 'external'
+            ? `Your speech detection model: ${vad.model.path}`
+            : 'Silero speech detection model installed.',
+  };
+}
+
+export function describeGenerationModel(model: SubtitleGenerationModelStatus) {
+  switch (model.kind) {
+    case 'external':
+      return { ready: true, download: false, text: `Your model: ${model.path}` };
+    case 'managed':
+      return { ready: true, download: false, text: `SubMiner model: ${model.path}` };
+    case 'missing':
+      return { ready: false, download: true, text: 'Download a speech model to get started.' };
+    case 'invalid':
+      return { ready: false, download: false, text: model.message };
+    default: {
+      const exhaustive: never = model;
+      return exhaustive;
+    }
+  }
+}
+
+const STAGE_LABELS = {
+  download: 'Downloading speech model',
+  extract: 'Preparing audio',
+  transcribe: 'Recognizing Japanese speech',
+  write: 'Saving subtitles',
+} satisfies Record<SubtitleGenerationProgress['stage'], string>;
+
+export function describeGenerationProgress(progress: SubtitleGenerationProgress | null) {
+  const raw = progress?.percent;
+  const percent =
+    raw !== undefined && Number.isFinite(raw) ? Math.max(0, Math.min(100, raw)) : null;
+  return {
+    stage: progress ? STAGE_LABELS[progress.stage] : 'Preparing',
+    percent,
+    label: percent === null ? 'Working...' : `${Math.floor(percent)}%`,
+  };
+}

src/renderer/modals/subtitle-generation.ts

@@ -0,0 +1,332 @@
+import type { SubtitleGenerationProgress } from '../../shared/subtitle-generation';
+import {
+  SUBTITLE_GENERATION_MODELS,
+  RECOMMENDED_SUBTITLE_GENERATION_MODEL,
+  formatSubtitleGenerationModelSize,
+  getSubtitleGenerationModel,
+  isSubtitleGenerationModelId,
+} from '../../shared/subtitle-generation-model-catalog';
+import type {
+  SubtitleGenerationResult,
+  SubtitleGenerationStatus,
+} from '../../shared/subtitle-generation-ipc';
+import type { ModalStateReader, RendererContext } from '../context';
+import { syncOverlayMouseIgnoreState } from '../overlay-mouse-ignore';
+import { createModalFocusGuard } from './modal-focus-guard';
+import {
+  describeGenerationModel,
+  describeGenerationProgress,
+  describeGenerationTools,
+  describeGenerationVad,
+} from './subtitle-generation-view';
+import { SUBTITLE_GENERATION_VAD_MODEL } from '../../shared/subtitle-generation-vad-model';
+
+function element<T extends HTMLElement>(id: string, constructor: new () => T): T {
+  const node = document.getElementById(id);
+  if (!(node instanceof constructor)) throw new Error(`Missing subtitle generation element: ${id}`);
+  return node;
+}
+
+export function createSubtitleGenerationModal(
+  ctx: RendererContext,
+  options: {
+    modalStateReader: Pick<ModalStateReader, 'isAnyModalOpen'>;
+    syncSettingsModalSubtitleSuppression: () => void;
+  },
+) {
+  const dom = {
+    modal: element('subtitleGenerationModal', HTMLDivElement),
+    close: element('subtitleGenerationClose', HTMLButtonElement),
+    open: element('subtitleGenerationOpen', HTMLButtonElement),
+    media: element('subtitleGenerationMedia', HTMLDivElement),
+    tools: element('subtitleGenerationTools', HTMLDivElement),
+    model: element('subtitleGenerationModel', HTMLDivElement),
+    modelPicker: element('subtitleGenerationModelPicker', HTMLDivElement),
+    modelSelect: element('subtitleGenerationModelSelect', HTMLSelectElement),
+    modelDescription: element('subtitleGenerationModelDescription', HTMLParagraphElement),
+    download: element('subtitleGenerationDownload', HTMLButtonElement),
+    vadEnabled: element('subtitleGenerationVadEnabled', HTMLInputElement),
+    vadModel: element('subtitleGenerationVadModel', HTMLDivElement),
+    vadDownload: element('subtitleGenerationVadDownload', HTMLButtonElement),
+    activity: element('subtitleGenerationActivity', HTMLDivElement),
+    stage: element('subtitleGenerationStage', HTMLSpanElement),
+    percent: element('subtitleGenerationPercent', HTMLSpanElement),
+    progress: element('subtitleGenerationProgress', HTMLProgressElement),
+    status: element('subtitleGenerationStatus', HTMLDivElement),
+    refresh: element('subtitleGenerationRefresh', HTMLButtonElement),
+    cancel: element('subtitleGenerationCancel', HTMLButtonElement),
+    start: element('subtitleGenerationStart', HTMLButtonElement),
+  };
+  let snapshot: SubtitleGenerationStatus | null = null;
+  let progress: SubtitleGenerationProgress | null = null;
+  let result: SubtitleGenerationResult | null = null;
+  let pending = false;
+  let cancelling = false;
+  let checking = false;
+  let error: string | null = null;
+  let priorFocus: Element | null = null;
+  let poll: ReturnType<typeof setTimeout> | null = null;
+  let unsubscribe: (() => void) | null = null;
+  for (const model of SUBTITLE_GENERATION_MODELS) {
+    const option = document.createElement('option');
+    option.value = model.id;
+    const recommended = model.id === RECOMMENDED_SUBTITLE_GENERATION_MODEL ? ' (recommended)' : '';
+    option.textContent = `${model.id}${recommended} · ${formatSubtitleGenerationModelSize(model.size)}`;
+    dom.modelSelect.append(option);
+  }
+
+  const focus = createModalFocusGuard({
+    isOpen: () => ctx.state.subtitleGenerationModalOpen,
+    getModalRoot: () => dom.modal,
+    getPreferredFocusTargets: () => [dom.modelSelect, dom.download, dom.start],
+    getFallbackFocusTarget: () => dom.close,
+    isModalLayer: ctx.platform.isModalLayer,
+  });
+
+  function render(): void {
+    const busy = pending || Boolean(snapshot?.running);
+    const model = snapshot ? describeGenerationModel(snapshot.model) : null;
+    const vad = snapshot ? describeGenerationVad(snapshot.vad) : null;
+    const tools = snapshot ? describeGenerationTools(snapshot.tools) : null;
+    const readyMessage = !snapshot?.mediaPath
+      ? 'Open local media to generate subtitles.'
+      : !tools?.ready
+        ? 'Install the missing tools or set their paths in Settings, then click Check again.'
+        : !model?.ready
+          ? 'Set up a speech model to continue.'
+          : !vad?.ready
+            ? 'Download the speech detection model or uncheck Focus on spoken dialogue.'
+            : 'Ready when you are.';
+    dom.media.textContent = snapshot?.mediaPath ?? 'Open a local media file in the player first.';
+    dom.tools.textContent = tools?.text ?? 'Checking local tools...';
+    dom.model.textContent = model?.text ?? 'Checking local models...';
+    dom.modelPicker.classList.toggle('hidden', !snapshot || Boolean(snapshot.externalModelPath));
+    dom.modelSelect.disabled = busy || checking || !snapshot || Boolean(snapshot.externalModelPath);
+    if (snapshot) {
+      dom.modelSelect.value = snapshot.managedModel;
+      dom.modelDescription.textContent = getSubtitleGenerationModel(
+        snapshot.managedModel,
+      ).description;
+    }
+    dom.download.classList.toggle('hidden', !model?.download);
+    dom.download.textContent = snapshot
+      ? `Download ${snapshot.managedModel} model`
+      : 'Download model';
+    dom.download.disabled = busy || checking;
+    if (!checking) dom.vadEnabled.checked = snapshot?.vad.enabled ?? false;
+    dom.vadEnabled.disabled = busy || checking || !snapshot;
+    dom.vadModel.textContent = vad?.text ?? 'Checking speech detection...';
+    dom.vadDownload.classList.toggle('hidden', !vad?.download);
+    dom.vadDownload.textContent = `Download speech detection model · ${formatSubtitleGenerationModelSize(SUBTITLE_GENERATION_VAD_MODEL.size)}`;
+    dom.vadDownload.disabled = busy || checking;
+    dom.start.disabled =
+      busy || checking || !tools?.ready || !model?.ready || !vad?.ready || !snapshot?.mediaPath;
+    dom.refresh.disabled = busy || checking;
+    dom.cancel.classList.toggle('hidden', !busy);
+    dom.cancel.disabled = cancelling;
+    dom.cancel.textContent = cancelling ? 'Cancelling...' : 'Cancel';
+    dom.activity.classList.toggle('hidden', !busy);
+    const activity = describeGenerationProgress(progress);
+    dom.stage.textContent = activity.stage;
+    dom.percent.textContent = activity.label;
+    if (activity.percent === null) dom.progress.removeAttribute('value');
+    else dom.progress.value = activity.percent;
+    dom.status.classList.toggle('error', Boolean(error || (!busy && result && !result.ok)));
+    dom.status.textContent =
+      error ??
+      (busy
+        ? cancelling
+          ? 'Stopping the current operation...'
+          : (progress?.message ?? 'Starting...')
+        : (result?.message ?? (checking ? 'Checking local setup...' : readyMessage)));
+  }
+
+  function stopPolling(): void {
+    if (poll !== null) clearTimeout(poll);
+    poll = null;
+  }
+
+  async function refresh(): Promise<void> {
+    if (checking) return;
+    checking = true;
+    error = null;
+    render();
+    try {
+      snapshot = await window.electronAPI.getSubtitleGenerationStatus();
+      if (!pending) {
+        progress = snapshot.progress;
+        result = snapshot.lastResult ?? result;
+      }
+    } catch (cause) {
+      error = cause instanceof Error ? cause.message : 'Could not check subtitle generation setup.';
+    } finally {
+      checking = false;
+      render();
+      stopPolling();
+      if (ctx.state.subtitleGenerationModalOpen && snapshot?.running && !pending) {
+        poll = setTimeout(() => void refresh(), 1500);
+      }
+    }
+  }
+
+  async function run(action: 'download' | 'download-vad' | 'generate'): Promise<void> {
+    if (pending || snapshot?.running || checking || !snapshot) return;
+    const model = describeGenerationModel(snapshot.model);
+    const vad = describeGenerationVad(snapshot.vad);
+    const tools = describeGenerationTools(snapshot.tools);
+    if (
+      action === 'download'
+        ? !model.download
+        : action === 'download-vad'
+          ? !vad.download
+          : !tools.ready || !model.ready || !vad.ready || !snapshot.mediaPath
+    )
+      return;
+    pending = true;
+    result = null;
+    error = null;
+    progress = { stage: action === 'generate' ? 'extract' : 'download', message: 'Starting...' };
+    render();
+    try {
+      result = await (action === 'download'
+        ? window.electronAPI.downloadSubtitleGenerationModel()
+        : action === 'download-vad'
+          ? window.electronAPI.downloadSubtitleGenerationVadModel()
+          : window.electronAPI.startSubtitleGeneration());
+    } catch (cause) {
+      result = {
+        ok: false,
+        message: cause instanceof Error ? cause.message : 'Subtitle generation failed.',
+      };
+    } finally {
+      pending = false;
+      cancelling = false;
+      // Recheck model availability after downloads and recover controls after errors.
+      await refresh();
+    }
+  }
+
+  async function selectModel(): Promise<void> {
+    const model = dom.modelSelect.value;
+    if (pending || snapshot?.running || checking || !isSubtitleGenerationModelId(model)) return;
+    checking = true;
+    error = null;
+    render();
+    try {
+      snapshot = await window.electronAPI.selectSubtitleGenerationModel(model);
+      result = snapshot.lastResult;
+      progress = snapshot.progress;
+    } catch (cause) {
+      error = cause instanceof Error ? cause.message : 'Could not select the model.';
+    } finally {
+      checking = false;
+      render();
+    }
+  }
+
+  async function selectVad(): Promise<void> {
+    if (pending || snapshot?.running || checking || !snapshot) return;
+    const enabled = dom.vadEnabled.checked;
+    checking = true;
+    error = null;
+    render();
+    try {
+      snapshot = await window.electronAPI.setSubtitleGenerationVadEnabled(enabled);
+      result = snapshot.lastResult;
+      progress = snapshot.progress;
+    } catch (cause) {
+      error = cause instanceof Error ? cause.message : 'Could not change speech detection.';
+    } finally {
+      checking = false;
+      render();
+    }
+  }
+
+  async function cancel(): Promise<void> {
+    if (cancelling) return;
+    cancelling = true;
+    render();
+    try {
+      await window.electronAPI.cancelSubtitleGeneration();
+      await refresh();
+    } catch (cause) {
+      error = cause instanceof Error ? cause.message : 'Could not cancel the operation.';
+    } finally {
+      cancelling = false;
+      render();
+    }
+  }
+
+  function open(): void {
+    if (ctx.state.subtitleGenerationModalOpen) return;
+    priorFocus = document.activeElement;
+    ctx.state.subtitleGenerationModalOpen = true;
+    options.syncSettingsModalSubtitleSuppression();
+    dom.modal.classList.remove('hidden');
+    dom.modal.setAttribute('aria-hidden', 'false');
+    syncOverlayMouseIgnoreState(ctx);
+    focus.attach();
+    focus.focusFallbackTarget();
+    window.electronAPI.notifyOverlayModalOpened('subtitle-generation');
+    void refresh();
+  }
+
+  function close(): void {
+    if (!ctx.state.subtitleGenerationModalOpen) return;
+    ctx.state.subtitleGenerationModalOpen = false;
+    options.syncSettingsModalSubtitleSuppression();
+    dom.modal.classList.add('hidden');
+    dom.modal.setAttribute('aria-hidden', 'true');
+    focus.detach();
+    stopPolling();
+    window.electronAPI.notifyOverlayModalClosed('subtitle-generation');
+    if (priorFocus instanceof HTMLElement && priorFocus.isConnected)
+      priorFocus.focus({ preventScroll: true });
+    if (!options.modalStateReader.isAnyModalOpen()) syncOverlayMouseIgnoreState(ctx);
+  }
+
+  function handleKeydown(event: KeyboardEvent): boolean {
+    if (!ctx.state.subtitleGenerationModalOpen) return false;
+    if (event.key === 'Escape') {
+      event.preventDefault();
+      close();
+    }
+    return true;
+  }
+
+  function wireDomEvents(): void {
+    dom.close.addEventListener('click', close);
+    dom.open.addEventListener('click', () => {
+      void window.electronAPI
+        .requestSubtitleGenerationOpen()
+        .then((opened) => {
+          if (!opened)
+            ctx.dom.subtitleSidebarStatus.textContent = 'Could not open subtitle generation.';
+        })
+        .catch((cause: unknown) => {
+          ctx.dom.subtitleSidebarStatus.textContent =
+            cause instanceof Error ? cause.message : 'Could not open subtitle generation.';
+        });
+    });
+    dom.download.addEventListener('click', () => void run('download'));
+    dom.vadDownload.addEventListener('click', () => void run('download-vad'));
+    dom.vadEnabled.addEventListener('change', () => void selectVad());
+    dom.modelSelect.addEventListener('change', () => void selectModel());
+    dom.start.addEventListener('click', () => void run('generate'));
+    dom.cancel.addEventListener('click', () => void cancel());
+    dom.refresh.addEventListener('click', () => void refresh());
+    unsubscribe = window.electronAPI.onSubtitleGenerationProgress((update) => {
+      progress = update;
+      render();
+    });
+  }
+
+  function dispose(): void {
+    stopPolling();
+    focus.detach();
+    unsubscribe?.();
+    unsubscribe = null;
+  }
+
+  return { open, close, handleKeydown, wireDomEvents, dispose };
+}

src/renderer/modals/subtitle-sidebar.test.ts

@@ -113,6 +113,20 @@ test('findActiveSubtitleCueIndex prefers current subtitle timing over near-futur
   assert.equal(findActiveSubtitleCueIndex(cues, { text: 'previous', startTime: 231 }, 233, 0), 0);
 });
 
+test('findActiveSubtitleCueIndex follows playback through empty subtitle gaps', () => {
+  const cues = [
+    { startTime: 0, endTime: 2, text: 'first' },
+    { startTime: 100, endTime: 102, text: 'later' },
+    { startTime: 105, endTime: 107, text: 'next' },
+  ];
+
+  assert.equal(findActiveSubtitleCueIndex(cues, { text: 'later', startTime: 100 }, 101, 1), 1);
+  assert.equal(findActiveSubtitleCueIndex(cues, { text: '', startTime: 0 }, 103, 1), 2);
+  assert.equal(findActiveSubtitleCueIndex(cues, { text: 'next', startTime: 105 }, 105, 2), 2);
+  assert.equal(findActiveSubtitleCueIndex(cues, { text: '', startTime: 0 }, 108, 2), -1);
+  assert.equal(findActiveSubtitleCueIndex(cues, { text: 'first', startTime: 0 }, 0, 2), 0);
+});
+
 test('subtitle sidebar mining context resolves selected row cue timing', () => {
   const globals = globalThis as typeof globalThis & {
     Element?: unknown;

src/renderer/modals/subtitle-sidebar.ts

@@ -120,8 +120,12 @@ export function findActiveSubtitleCueIndex(
     return -1;
   }
 
+  // The mpv client maps cleared sub-start to zero. Empty text has no active cue timing.
   const hasCurrentTiming =
-    typeof current?.startTime === 'number' && Number.isFinite(current.startTime);
+    current !== null &&
+    normalizeCueText(current.text).length > 0 &&
+    typeof current.startTime === 'number' &&
+    Number.isFinite(current.startTime);
 
   if (hasCurrentTiming) {
     const timingMatch = cues.findIndex(