feat(subtitles): add local Japanese subtitle generation

- Generate Japanese SRTs through the overlay modal or launcher
- Support managed whisper.cpp models and optional dialogue detection
This commit is contained in:
2026-09-07 21:43:53 -07:00
parent c14c690875
commit 2d623838d5
82 changed files with 4001 additions and 54 deletions
@@ -0,0 +1,245 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import path from 'node:path';
import { parseArgs } from '../config.js';
import {
createGenerationProgressReporter,
runGenerateSubtitlesCommand,
} from './generate-subtitles-command.js';
type Deps = NonNullable<Parameters<typeof runGenerateSubtitlesCommand>[1]>;
function fixture(argv: string[] = ['generate-subs', '/media/episode.mkv']) {
const output: string[] = [];
const commands: unknown[][] = [];
const generations: Parameters<NonNullable<Deps['generate']>>[0][] = [];
let exitCode: number | undefined;
let interrupted: (() => void) | undefined;
let detached = false;
const context = {
args: parseArgs(argv, 'subminer', {}),
mpvSocketPath: '/tmp/test-subminer-socket',
processAdapter: {
writeStdout: (text: string) => {
output.push(text);
},
setExitCode: (code: number) => {
exitCode = code;
},
},
};
const deps: Deps = {
readConfig: () => ({ subtitleGeneration: { modelPath: '/models/external.bin' } }),
configPath: () => '/settings/SubMiner/config.jsonc',
resolveModel: async () => ({ kind: 'external', path: '/models/external.bin' }),
downloadModel: async () => {
throw new Error('Unexpected model download');
},
generate: async (input) => {
generations.push(input);
input.onProgress?.({
stage: 'transcribe',
percent: 50,
message: 'Transcribing Japanese audio',
});
return '/media/episode.ja.srt';
},
mpvCommand: async (_socket, command) => {
commands.push(command);
if (command[1] === 'path') return '/media/episode.mkv';
if (command[1] === 'track-list') return [{ type: 'audio', selected: true, 'ff-index': 2 }];
return undefined;
},
onInterrupt: (handler) => {
interrupted = handler;
return () => {
detached = true;
};
},
};
return {
context,
deps,
output,
commands,
generations,
exitCode: () => exitCode,
interrupt: () => interrupted?.(),
detached: () => detached,
};
}
test('launcher uses the shared core and selected mpv audio then loads the generated file', async () => {
const f = fixture(['generate-subs']);
assert.equal(await runGenerateSubtitlesCommand(f.context, f.deps), true);
assert.equal(f.generations[0]?.mediaPath, '/media/episode.mkv');
assert.equal(f.generations[0]?.audioStreamIndex, 2);
assert.equal(
f.generations[0]?.modelDirectory,
path.join('/settings/SubMiner', 'models', 'whisper'),
);
assert.equal(f.generations[0]?.config.modelPath, '/models/external.bin');
assert.deepEqual(f.commands.at(-2), [
'sub-add',
'/media/episode.ja.srt',
'select',
'Japanese (generated)',
'ja',
]);
assert.deepEqual(f.commands.at(-1), ['set_property', 'sub-delay', 0]);
assert.match(f.output.join(''), /50%/);
assert.match(f.output.join(''), /Saved Japanese subtitles/);
assert.equal(f.detached(), true);
});
test('launcher never downloads a model without the explicit option', async () => {
const f = fixture();
f.deps.resolveModel = async () => ({ kind: 'missing', path: '/models/missing.bin' });
await assert.rejects(runGenerateSubtitlesCommand(f.context, f.deps), /--download-model/);
assert.equal(f.generations.length, 0);
assert.equal(f.detached(), true);
});
test('current mpv generation requires an identifiable selected audio track', async () => {
for (const tracks of [
[],
[{ type: 'audio', selected: true }],
[{ type: 'audio', selected: true, external: true, 'ff-index': 0 }],
]) {
const f = fixture(['generate-subs']);
f.deps.mpvCommand = async (_socket, command) =>
command[1] === 'path' ? '/media/episode.mkv' : tracks;
await assert.rejects(runGenerateSubtitlesCommand(f.context, f.deps), /audio track/);
assert.equal(f.generations.length, 0);
}
});
test('explicit local file leaves Japanese track selection to the shared generator', async () => {
const f = fixture();
await runGenerateSubtitlesCommand(f.context, f.deps);
assert.equal(f.generations[0]?.audioStreamIndex, undefined);
assert.equal(
f.commands.some((command) => command[1] === 'track-list'),
false,
);
});
test('launcher does not load generated subtitles after mpv switches files', async () => {
const f = fixture(['generate-subs']);
let pathRequests = 0;
f.deps.mpvCommand = async (_socket, command) => {
f.commands.push(command);
if (command[1] === 'path')
return ++pathRequests === 1 ? '/media/episode.mkv' : '/media/next.mkv';
return [{ type: 'audio', selected: true, 'ff-index': 2 }];
};
await runGenerateSubtitlesCommand(f.context, f.deps);
assert.equal(
f.commands.some((command) => command[0] === 'sub-add'),
false,
);
assert.match(f.output.join(''), /Saved Japanese subtitles/);
});
test('explicit managed model overrides external config and downloads before generation', async () => {
const f = fixture([
'generate-subs',
'/media/episode.mkv',
'--model',
'medium',
'--download-model',
]);
f.deps.resolveModel = async (config) => {
assert.equal(config.modelPath, '');
assert.equal(config.managedModel, 'medium');
return { kind: 'missing', path: '/models/medium.bin' };
};
let downloaded = false;
f.deps.downloadModel = async () => {
downloaded = true;
return '/models/medium.bin';
};
const generate = f.deps.generate;
f.deps.generate = async (input) => {
assert.equal(downloaded, true);
if (!generate) throw new Error('Missing fixture generator');
return generate(input);
};
await runGenerateSubtitlesCommand(f.context, f.deps);
assert.equal(f.generations.length, 1);
});
test('generation can run standalone and never loads subtitles into another video', async () => {
for (const playing of [null, '/media/different.mkv']) {
const f = fixture();
f.deps.mpvCommand = async (_socket, command) => {
f.commands.push(command);
if (playing === null) throw new Error('mpv is not running');
return playing;
};
await runGenerateSubtitlesCommand(f.context, f.deps);
assert.equal(f.generations.length, 1);
assert.equal(
f.commands.some((command) => command[0] === 'sub-add'),
false,
);
}
});
test('launcher preserves the saved path when loading into mpv fails', async () => {
const f = fixture();
const mpv = f.deps.mpvCommand;
f.deps.mpvCommand = async (socket, command, timeout) => {
if (command[0] === 'sub-add') throw new Error('load failed');
return mpv?.(socket, command, timeout);
};
await runGenerateSubtitlesCommand(f.context, f.deps);
assert.match(f.output.join(''), /Saved Japanese subtitles: \/media\/episode.ja.srt/);
assert.match(f.output.join(''), /mpv could not load them: load failed/);
assert.equal(f.exitCode(), 1);
});
test('SIGINT cancels shared generation and unregisters its handler', async () => {
const f = fixture();
f.deps.generate = async (input) => {
f.interrupt();
assert.equal(input.signal?.aborted, true);
throw new Error('Aborted');
};
await runGenerateSubtitlesCommand(f.context, f.deps);
assert.equal(f.exitCode(), 130);
assert.equal(f.detached(), true);
assert.match(f.output.join(''), /cancelled/);
});
test('cancellation after generation preserves the saved path and skips mpv loading', async () => {
const f = fixture();
f.deps.generate = async () => {
f.interrupt();
return '/media/episode.ja.srt';
};
await runGenerateSubtitlesCommand(f.context, f.deps);
assert.equal(
f.commands.some((command) => command[0] === 'sub-add'),
false,
);
assert.match(f.output.join(''), /Saved Japanese subtitles: \/media\/episode.ja.srt/);
assert.equal(f.exitCode(), 130);
assert.equal(f.detached(), true);
});
test('progress throttles repeated updates but always reports stage changes and completion', () => {
const output: string[] = [];
let time = 0;
const progress = createGenerationProgressReporter(
(text) => output.push(text),
() => time,
);
progress({ stage: 'download', percent: 0, message: 'Downloading' });
progress({ stage: 'download', percent: 1, message: 'Downloading' });
time = 1000;
progress({ stage: 'download', percent: 50, message: 'Downloading' });
progress({ stage: 'download', percent: 100, message: 'Downloading' });
progress({ stage: 'extract', message: 'Extracting audio' });
assert.equal(output.length, 4);
});
@@ -0,0 +1,219 @@
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import {
downloadSubtitleGenerationModel,
generateJapaneseSubtitles,
resolveSubtitleGenerationModel,
} from '../../src/core/services/subtitle-generation.js';
import {
resolveSubtitleGenerationConfig,
type SubtitleGenerationProgress,
} from '../../src/shared/subtitle-generation.js';
import {
readLauncherMainConfigObject,
resolveLauncherMainConfigPath,
} from '../config/shared-config-reader.js';
import { sendMpvCommandWithResponse } from '../mpv.js';
import { resolvePathMaybe } from '../util.js';
import type { LauncherCommandContext } from './context.js';
type GenerationCommandContext = Pick<LauncherCommandContext, 'args' | 'mpvSocketPath'> & {
processAdapter: Pick<LauncherCommandContext['processAdapter'], 'writeStdout' | 'setExitCode'>;
};
interface GenerationCommandDeps {
readConfig: typeof readLauncherMainConfigObject;
configPath: typeof resolveLauncherMainConfigPath;
resolveModel: typeof resolveSubtitleGenerationModel;
downloadModel: typeof downloadSubtitleGenerationModel;
generate: typeof generateJapaneseSubtitles;
mpvCommand: typeof sendMpvCommandWithResponse;
onInterrupt: (handler: () => void) => () => void;
}
const defaultDeps: GenerationCommandDeps = {
readConfig: readLauncherMainConfigObject,
configPath: resolveLauncherMainConfigPath,
resolveModel: resolveSubtitleGenerationModel,
downloadModel: downloadSubtitleGenerationModel,
generate: generateJapaneseSubtitles,
mpvCommand: sendMpvCommandWithResponse,
onInterrupt: (handler) => {
process.on('SIGINT', handler);
return () => process.off('SIGINT', handler);
},
};
function localMediaPath(value: string, workingDirectory = process.cwd()): string {
if (value.startsWith('file://')) return fileURLToPath(value);
if (/^[a-z][a-z\d+.-]*:\/\//i.test(value)) {
throw new Error('Japanese subtitle generation requires a local media file.');
}
return path.resolve(workingDirectory, resolvePathMaybe(value));
}
async function readMpvMedia(socketPath: string, command: GenerationCommandDeps['mpvCommand']) {
const media = await command(socketPath, ['get_property', 'path'], 1000);
if (typeof media !== 'string' || !media.trim()) return null;
let workingDirectory: string | undefined;
if (!path.isAbsolute(media) && !media.startsWith('file://')) {
const directory = await command(socketPath, ['get_property', 'working-directory'], 1000);
if (typeof directory !== 'string') return null;
workingDirectory = directory;
}
return localMediaPath(media, workingDirectory);
}
async function readMpvAudioStream(
socketPath: string,
command: GenerationCommandDeps['mpvCommand'],
) {
const tracks = await command(socketPath, ['get_property', 'track-list'], 1000);
for (const track of Array.isArray(tracks) ? tracks : []) {
if (
typeof track === 'object' &&
track !== null &&
'type' in track &&
track.type === 'audio' &&
'selected' in track &&
track.selected === true
) {
if ('external' in track && track.external === true) {
throw new Error(
'The selected mpv audio track is external. Pass its local file to generate-subs.',
);
}
if (
'ff-index' in track &&
typeof track['ff-index'] === 'number' &&
Number.isSafeInteger(track['ff-index']) &&
track['ff-index'] >= 0
) {
return track['ff-index'];
}
}
}
throw new Error(
'Select an audio track in mpv, or pass --audio-stream with its absolute stream index.',
);
}
function sameFile(left: string, right: string): boolean {
try {
return fs.realpathSync(left) === fs.realpathSync(right);
} catch {
return path.resolve(left) === path.resolve(right);
}
}
/** Keep progress readable in terminals and redirected logs, even for large model downloads. */
export function createGenerationProgressReporter(write: (text: string) => void, now = Date.now) {
let previousStage: SubtitleGenerationProgress['stage'] | undefined;
let previousTime = -Infinity;
let previousLine = '';
return (progress: SubtitleGenerationProgress): void => {
const percent =
typeof progress.percent === 'number' && Number.isFinite(progress.percent)
? Math.floor(Math.max(0, Math.min(100, progress.percent)))
: undefined;
const line = `[${progress.stage}] ${percent === undefined ? '' : `${percent}% `}${progress.message}\n`;
const timestamp = now();
if (
line === previousLine ||
(progress.stage === previousStage && timestamp - previousTime < 1000 && percent !== 100)
)
return;
write(line);
previousStage = progress.stage;
previousTime = timestamp;
previousLine = line;
};
}
export async function runGenerateSubtitlesCommand(
context: GenerationCommandContext,
overrides: Partial<GenerationCommandDeps> = {},
): Promise<boolean> {
const options = context.args.generateSubtitles;
if (!options) return false;
const deps = { ...defaultDeps, ...overrides };
const write = (text: string) => context.processAdapter.writeStdout(text);
const controller = new AbortController();
const removeInterrupt = deps.onInterrupt(() => controller.abort());
try {
const config = resolveSubtitleGenerationConfig(deps.readConfig()?.subtitleGeneration);
if (options.managedModel) {
config.managedModel = options.managedModel;
config.modelPath = '';
}
if (options.modelPath !== undefined)
config.modelPath = path.resolve(resolvePathMaybe(options.modelPath));
const modelDirectory = path.join(path.dirname(deps.configPath()), 'models', 'whisper');
const currentMedia = await readMpvMedia(context.mpvSocketPath, deps.mpvCommand).catch(
() => null,
);
const mediaPath = options.mediaPath ? localMediaPath(options.mediaPath) : currentMedia;
if (!mediaPath)
throw new Error('Pass a local video file or open one in mpv before running generate-subs.');
const audioStreamIndex =
options.audioStreamIndex ??
(!options.mediaPath
? await readMpvAudioStream(context.mpvSocketPath, deps.mpvCommand)
: undefined);
const onProgress = createGenerationProgressReporter(write);
const model = await deps.resolveModel(config, modelDirectory);
if (model.kind === 'invalid') throw new Error(model.message);
if (model.kind === 'missing') {
if (!options.downloadModel) {
throw new Error(
'No Whisper model found. Run again with --download-model, or set subtitleGeneration.modelPath / --model-path.',
);
}
await deps.downloadModel({ config, modelDirectory, onProgress, signal: controller.signal });
}
const outputPath = await deps.generate({
config,
modelDirectory,
mediaPath,
audioStreamIndex,
outputPath: options.outputPath
? path.resolve(resolvePathMaybe(options.outputPath))
: undefined,
onProgress,
signal: controller.signal,
});
write(`Saved Japanese subtitles: ${outputPath}\n`);
controller.signal.throwIfAborted();
const playingMedia = await readMpvMedia(context.mpvSocketPath, deps.mpvCommand).catch(
() => null,
);
controller.signal.throwIfAborted();
if (playingMedia && sameFile(playingMedia, mediaPath)) {
try {
await deps.mpvCommand(context.mpvSocketPath, [
'sub-add',
outputPath,
'select',
'Japanese (generated)',
'ja',
]);
await deps.mpvCommand(context.mpvSocketPath, ['set_property', 'sub-delay', 0]);
write('Loaded Japanese subtitles into mpv.\n');
} catch (error) {
write(
`Subtitles are saved, but mpv could not load them: ${error instanceof Error ? error.message : String(error)}\n`,
);
context.processAdapter.setExitCode(1);
}
}
return true;
} catch (error) {
if (!controller.signal.aborted) throw error;
write('Subtitle generation cancelled.\n');
context.processAdapter.setExitCode(130);
return true;
} finally {
removeInterrupt();
}
}
+1
View File
@@ -248,6 +248,7 @@ export function applyRootOptionsToArgs(
}
export function applyInvocationsToArgs(parsed: Args, invocations: CliInvocations): void {
if (invocations.generateSubtitles) parsed.generateSubtitles = invocations.generateSubtitles;
if (invocations.dictionaryTriggered) parsed.dictionary = true;
if (invocations.dictionaryCandidates) parsed.dictionaryCandidates = true;
if (invocations.dictionarySelect) parsed.dictionarySelect = true;
@@ -1,6 +1,61 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { parseCliPrograms, resolveTopLevelCommand } from './cli-parser-builder.js';
import { SUBTITLE_GENERATION_MODELS } from '../../src/shared/subtitle-generation-model-catalog.js';
test('generate-subs accepts all downloadable multilingual model variants', () => {
for (const { id } of SUBTITLE_GENERATION_MODELS) {
const { invocations } = parseCliPrograms(['generate-subs', '--model', id], 'subminer');
assert.equal(invocations.generateSubtitles?.managedModel, id);
}
});
test('generate-subs parses local generation options separately from YouTube options', () => {
const result = parseCliPrograms(
[
'generate-subs',
'episode.mkv',
'--download-model',
'--model',
'medium',
'--output',
'episode.ja.srt',
'--audio-stream',
'2',
],
'subminer',
);
assert.deepEqual(result.invocations.generateSubtitles, {
mediaPath: 'episode.mkv',
downloadModel: true,
managedModel: 'medium',
modelPath: undefined,
outputPath: 'episode.ja.srt',
audioStreamIndex: 2,
});
assert.equal(
parseCliPrograms(['generate-subs'], 'subminer').invocations.generateSubtitles?.mediaPath,
undefined,
);
assert.equal(
parseCliPrograms(['generate-subs', '--model-path', '/models/ggml.bin'], 'subminer').invocations
.generateSubtitles?.modelPath,
'/models/ggml.bin',
);
});
test('generate-subs rejects conflicting models and malformed audio stream indices', () => {
for (const flags of [
['--model', 'tiny.en'],
['--model', 'small.en-q5_1'],
['--model', 'toString'],
['--audio-stream', '-1'],
['--audio-stream', '1.5'],
['--model-path', '/model.bin', '--download-model'],
['--model-path', '/model.bin', '--model', 'small'],
])
assert.throws(() => parseCliPrograms(['generate-subs', ...flags], 'subminer'), /Generation/);
});
test('resolveTopLevelCommand skips root options and finds the first command', () => {
assert.deepEqual(resolveTopLevelCommand(['--backend', 'macos', 'config', 'show']), {
+53
View File
@@ -1,4 +1,9 @@
import { Command } from 'commander';
import type { Args } from '../types.js';
import {
isSubtitleGenerationModelId,
SUBTITLE_GENERATION_MODELS,
} from '../../src/shared/subtitle-generation-model-catalog.js';
export interface JellyfinInvocation {
action?: string;
@@ -20,6 +25,7 @@ export interface CommandActionInvocation {
}
export interface CliInvocations {
generateSubtitles?: Args['generateSubtitles'];
jellyfinInvocation: JellyfinInvocation | null;
configInvocation: CommandActionInvocation | null;
settingsInvocation: CommandActionInvocation | null;
@@ -118,6 +124,7 @@ function getTopLevelCommand(argv: string[]): { name: string; index: number } | n
'mpv',
'logs',
'dictionary',
'generate-subs',
'dict',
'stats',
'sync',
@@ -199,6 +206,7 @@ export function parseCliPrograms(
let texthookerOpenBrowser = false;
let doctorTriggered = false;
let texthookerTriggered = false;
let generateSubtitles: Args['generateSubtitles'];
const commandProgram = new Command();
commandProgram
@@ -223,6 +231,50 @@ export function parseCliPrograms(
.argument('[target]', 'file, directory, or URL');
applyRootOptions(rootProgram);
commandProgram
.command('generate-subs')
.description('Generate Japanese subtitles locally with whisper.cpp')
.argument('[video]', 'Local media file, or the current mpv file if omitted')
.option('--download-model', 'Download the selected managed model if missing')
.option('--model-path <path>', 'Use an existing whisper.cpp model file')
.option(
'--model <name>',
`Managed model: ${SUBTITLE_GENERATION_MODELS.map((model) => model.id).join(', ')}`,
)
.option('--output <path>', 'Save subtitles to this SRT path')
.option('--audio-stream <index>', 'Absolute audio stream index from ffprobe')
.action((mediaPath: string | undefined, options: Record<string, unknown>) => {
const model = options.model;
if (model !== undefined && !isSubtitleGenerationModelId(model)) {
throw new Error(
`Generation --model must be one of: ${SUBTITLE_GENERATION_MODELS.map((entry) => entry.id).join(', ')}.`,
);
}
if (
options.modelPath !== undefined &&
(model !== undefined || options.downloadModel === true)
) {
throw new Error(
'Generation --model-path cannot be combined with --model or --download-model.',
);
}
let audioStreamIndex: number | undefined;
if (typeof options.audioStream === 'string') {
audioStreamIndex = Number(options.audioStream);
if (!/^\d+$/.test(options.audioStream) || !Number.isSafeInteger(audioStreamIndex)) {
throw new Error('Generation --audio-stream must be a non-negative integer stream index.');
}
}
generateSubtitles = {
mediaPath,
downloadModel: options.downloadModel === true,
modelPath: typeof options.modelPath === 'string' ? options.modelPath : undefined,
managedModel: model,
outputPath: typeof options.output === 'string' ? options.output : undefined,
audioStreamIndex,
};
});
commandProgram
.command('jellyfin')
.alias('jf')
@@ -507,6 +559,7 @@ export function parseCliPrograms(
options: selectedProgram.opts<Record<string, unknown>>(),
rootTarget: rootProgram.processedArgs[0],
invocations: {
generateSubtitles,
jellyfinInvocation,
configInvocation,
settingsInvocation,
+5
View File
@@ -25,6 +25,7 @@ import { runHistorySession } from './commands/history-command.js';
import { runSyncCommand } from './commands/sync-command.js';
import { runPlaybackCommand } from './commands/playback-command.js';
import { runUpdateCommand } from './commands/update-command.js';
import { runGenerateSubtitlesCommand } from './commands/generate-subtitles-command.js';
const APP_VERSION =
typeof packageJson.version === 'string' && packageJson.version.trim()
@@ -112,6 +113,10 @@ async function main(): Promise<void> {
return;
}
if (await runGenerateSubtitlesCommand(context)) {
return;
}
const resolvedAppPath = ensureAppPath(context);
state.appPath = resolvedAppPath;
log('debug', args.logLevel, `Using SubMiner app binary: ${resolvedAppPath}`);
+9
View File
@@ -1,6 +1,7 @@
import path from 'node:path';
import os from 'node:os';
import type { MpvBackend, MpvLaunchMode } from '../src/types/config.js';
import type { SubtitleGenerationConfig } from '../src/shared/subtitle-generation.js';
import {
resolveDefaultLogFilePath,
type LogFileToggles,
@@ -88,6 +89,14 @@ export interface LauncherAiConfig {
}
export interface Args {
generateSubtitles?: {
mediaPath?: string;
downloadModel: boolean;
modelPath?: string;
managedModel?: SubtitleGenerationConfig['managedModel'];
outputPath?: string;
audioStreamIndex?: number;
};
backend: Backend;
directory: string;
recursive: boolean;