feat(subtitles): generate subtitles from anime streams

- Support finite HTTP/HTTPS streams with cached generated SRT files
- Document stream requirements and limitations
This commit is contained in:
2026-09-21 21:28:15 -07:00
parent 625f1ad238
commit 2c89bc1754
15 changed files with 369 additions and 24 deletions
@@ -0,0 +1,139 @@
import assert from 'node:assert/strict';
import { createServer } from 'node:http';
import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { DEFAULT_SUBTITLE_GENERATION_CONFIG } from '../../shared/subtitle-generation';
import { generateJapaneseSubtitles } from './subtitle-generation';
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
import { resolveSubtitleGenerationTools } from './subtitle-generation-tools';
test(
'real FFmpeg extracts a header-protected HLS episode and cleans up downloaded audio',
{
skip: process.platform === 'win32' ? 'Requires a POSIX executable fixture.' : false,
},
async (t) => {
const tools = await resolveSubtitleGenerationTools(DEFAULT_SUBTITLE_GENERATION_CONFIG);
if (tools.ffmpeg.kind !== 'found' || tools.ffprobe.kind !== 'found') {
t.skip('FFmpeg and ffprobe are required for the network extraction test.');
return;
}
const directory = await mkdtemp(path.join(tmpdir(), 'subminer-network-generation-'));
const requests: string[] = [];
const server = createServer(async (req, res) => {
if (
req.headers.referer !== 'https://anime.example/' ||
req.headers['user-agent'] !== 'SubMiner test'
) {
res.writeHead(403).end();
return;
}
const name = req.url?.slice(1) ?? '';
if (!/^(episode\.m3u8|segment\d+\.ts)$/.test(name)) {
res.writeHead(404).end();
return;
}
requests.push(name);
try {
res.end(await readFile(path.join(directory, name)));
} catch {
res.writeHead(404).end();
}
});
try {
await runSubtitleGenerationProcess({
command: tools.ffmpeg.path,
args: [
'-v',
'error',
'-f',
'lavfi',
'-i',
'sine=frequency=440:duration=3',
'-c:a',
'aac',
'-f',
'hls',
'-hls_time',
'1',
'-hls_playlist_type',
'vod',
'-hls_segment_filename',
path.join(directory, 'segment%d.ts'),
path.join(directory, 'episode.m3u8'),
],
});
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
const address = server.address();
assert.ok(address && typeof address === 'object');
const modelPath = path.join(directory, 'model.bin');
const model = Buffer.alloc(8);
model.writeUInt32LE(0x67676d6c, 0);
model.writeInt32LE(51865, 4);
await writeFile(modelPath, model);
const whisperPath = path.join(directory, 'whisper-test');
// Recognition is deterministic here; probing, HTTP requests and decoding use real FFmpeg.
await writeFile(
whisperPath,
`#!${process.execPath}
const fs = require('node:fs');
const assert = require('node:assert/strict');
const args = process.argv.slice(2);
const wav = args[args.indexOf('-f') + 1];
const bytes = fs.readFileSync(wav);
assert.equal(bytes.toString('ascii', 0, 4), 'RIFF');
assert.ok(bytes.length > 90000);
fs.writeFileSync(${JSON.stringify(path.join(directory, 'audio-path'))}, wav);
fs.writeFileSync(args[args.indexOf('-of') + 1] + '.srt', '1\\n00:00:00,500 --> 00:00:01,500\\nこんにちは\\n');
`,
{ mode: 0o755 },
);
const cacheDirectory = path.join(directory, 'cache');
const input = {
config: {
...DEFAULT_SUBTITLE_GENERATION_CONFIG,
modelPath,
whisperPath,
ffmpegPath: tools.ffmpeg.path,
ffprobePath: tools.ffprobe.path,
},
modelDirectory: directory,
mediaPath: `http://127.0.0.1:${address.port}/episode.m3u8`,
audioStreamIndex: 0,
remote: {
cacheDirectory,
httpHeaders: {
headers: { Referer: 'https://anime.example/' },
userAgent: 'SubMiner test',
},
},
};
const output = await generateJapaneseSubtitles(input);
assert.match(await readFile(output, 'utf8'), /00:00:00,500 --> 00:00:01,500\nこんにちは/);
assert.ok(requests.includes('episode.m3u8'));
assert.ok(requests.includes('segment2.ts'));
const wav = await readFile(path.join(directory, 'audio-path'), 'utf8');
await assert.rejects(readFile(wav), /ENOENT/);
await assert.rejects(
generateJapaneseSubtitles({
...input,
remote: { ...input.remote, httpHeaders: { headers: {}, userAgent: null } },
}),
/403/,
);
assert.deepEqual(await readdir(cacheDirectory), [path.basename(output)]);
} finally {
server.closeAllConnections();
await new Promise<void>((resolve, reject) =>
server.close((error) =>
error && (!('code' in error) || error.code !== 'ERR_SERVER_NOT_RUNNING')
? reject(error)
: resolve(),
),
);
await rm(directory, { recursive: true, force: true });
}
},
);
@@ -4,6 +4,8 @@ import { fileURLToPath } from 'node:url';
import type { SubtitleGenerationProgress } from '../../shared/subtitle-generation';
import { parseSrtCues } from './subtitle-cue-parser';
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
import type { ResolvedMpvHttpHeaders } from './mpv-http-headers';
import { subtitleGenerationHttpArgs } from './subtitle-generation-source';
export type SubtitleGenerationReference = {
label: string;
@@ -119,6 +121,7 @@ export async function loadSubtitleGenerationReference(input: {
ffmpegPath: string;
directory: string;
audioOffset: number;
httpHeaders?: ResolvedMpvHttpHeaders;
onProgress?: (progress: SubtitleGenerationProgress) => void;
signal?: AbortSignal;
}): Promise<number[]> {
@@ -135,6 +138,7 @@ export async function loadSubtitleGenerationReference(input: {
'-loglevel',
'error',
...(embedded ? ['-copyts', '-start_at_zero'] : []),
...(embedded && input.httpHeaders ? subtitleGenerationHttpArgs(input.httpHeaders) : []),
'-i',
reference.source.kind === 'external' ? reference.source.path : input.mediaPath,
'-map',
@@ -0,0 +1,17 @@
import { toFfmpegInputHttpArgs, type ResolvedMpvHttpHeaders } from './mpv-http-headers';
export interface SubtitleGenerationRemoteSource {
httpHeaders: ResolvedMpvHttpHeaders;
cacheDirectory: string;
}
/** Apply only to network inputs, including embedded subtitle reference extraction. */
export function subtitleGenerationHttpArgs(headers: ResolvedMpvHttpHeaders): string[] {
return [
'-protocol_whitelist',
'http,https,tcp,tls,crypto',
'-rw_timeout',
'15000000',
...toFfmpegInputHttpArgs(headers),
];
}
@@ -228,6 +228,65 @@ test('explicit audio stream and output path are respected without overwriting ex
);
}));
test('remote generation forwards HTTP options, restores timestamps, and preserves cached subtitles', () =>
fixture(async (directory) => {
const input = await generationFixture(directory);
const mediaPath = 'https://anime.example/episode.m3u8?token=private';
const cacheDirectory = path.join(directory, 'cache');
const remote = {
cacheDirectory,
httpHeaders: { headers: { Referer: 'https://anime.example/' }, userAgent: 'SubMiner test' },
};
const first = await generateJapaneseSubtitles({ ...input, mediaPath, remote });
const second = await generateJapaneseSubtitles({ ...input, mediaPath, remote });
assert.equal(path.dirname(first), cacheDirectory);
assert.match(path.basename(first), /^[a-f0-9]{24}\.ja\.generated\.srt$/);
assert.equal(second, first.replace('.srt', '.1.srt'));
assert.match(await readFile(first, 'utf8'), /00:00:03,500 --> 00:00:04,500/);
const calls: string[][] = (await readFile(input.callsPath, 'utf8'))
.trim()
.split('\n')
.map((line) => JSON.parse(line));
for (const args of calls.filter((args) => args.includes(mediaPath))) {
assert.equal(args[args.indexOf('-user_agent') + 1], 'SubMiner test');
assert.equal(args[args.indexOf('-headers') + 1], 'Referer: https://anime.example/\r\n');
assert.ok(args.indexOf('-headers') < args.indexOf(mediaPath));
assert.equal(args[args.indexOf('-protocol_whitelist') + 1], 'http,https,tcp,tls,crypto');
}
assert.deepEqual(
(await readdir(cacheDirectory)).filter((name) => name.startsWith('.')),
[],
);
}));
test('remote generation refuses live streams and unsupported protocols before extraction', () =>
fixture(async (directory) => {
const input = await generationFixture(directory);
const remote = {
cacheDirectory: path.join(directory, 'cache'),
httpHeaders: { headers: {}, userAgent: null },
};
await assert.rejects(
generateJapaneseSubtitles({ ...input, mediaPath: 'ftp://anime.example/episode', remote }),
/supported HTTP stream/,
);
const ffprobePath = await executable(
directory,
'live-probe',
"process.stdout.write(JSON.stringify({ streams: [{ index: 0, codec_type: 'audio' }], format: {} }));",
);
await assert.rejects(
generateJapaneseSubtitles({
...input,
config: { ...input.config, ffprobePath },
mediaPath: 'https://anime.example/live.m3u8',
remote,
}),
/finite episode duration/,
);
await assert.rejects(readFile(input.callsPath), /ENOENT/);
}));
test('dialogue generation isolates Whisper state between passages and preserves media timing', () =>
fixture(async (directory) => {
const input = await generationFixture(directory);
+34 -7
View File
@@ -1,5 +1,6 @@
import { access, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { access, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { constants } from 'node:fs';
import { createHash } from 'node:crypto';
import { tmpdir } from 'node:os';
import path from 'node:path';
import type {
@@ -11,6 +12,10 @@ import { runSubtitleGenerationProcess } from './subtitle-generation-process';
import { publishSubtitleGenerationFile } from './subtitle-generation-files';
import { formatTimestamp } from './subtitle-generation-srt';
import { transcribeSubtitleDialogue } from './subtitle-generation-dialogue';
import {
subtitleGenerationHttpArgs,
type SubtitleGenerationRemoteSource,
} from './subtitle-generation-source';
import {
loadSubtitleGenerationReference,
type SubtitleGenerationReference,
@@ -178,21 +183,35 @@ export async function generateJapaneseSubtitles(input: {
modelDirectory: string;
mediaPath: string;
audioStreamIndex?: number;
remote?: SubtitleGenerationRemoteSource;
references?: readonly SubtitleGenerationReference[];
outputPath?: string;
onProgress?: (progress: SubtitleGenerationProgress) => void;
signal?: AbortSignal;
}): Promise<string> {
input.signal?.throwIfAborted();
if (/^[a-z][a-z\d+.-]*:\/\//i.test(input.mediaPath))
throw new Error('Subtitle generation requires a local media file.');
const mediaPath = path.resolve(input.mediaPath);
if (!(await stat(mediaPath)).isFile())
const isRemote = /^[a-z][a-z\d+.-]*:\/\//i.test(input.mediaPath);
if (isRemote && (!input.remote || !/^https?:\/\//i.test(input.mediaPath)))
throw new Error('Subtitle generation requires a local media file or a supported HTTP stream.');
if (!isRemote && input.remote) throw new Error('Expected an HTTP stream for remote generation.');
const mediaPath = isRemote ? new URL(input.mediaPath).href : path.resolve(input.mediaPath);
if (!isRemote && !(await stat(mediaPath)).isFile())
throw new Error('Subtitle generation requires a local media file.');
// URLs may contain credentials or expiring tokens. Keep them out of cache filenames.
const destinationMediaPath = input.remote
? path.join(
input.remote.cacheDirectory,
createHash('sha256').update(mediaPath).digest('hex').slice(0, 24),
)
: mediaPath;
if (input.remote) await mkdir(input.remote.cacheDirectory, { recursive: true });
if (input.outputPath) await ensureAvailableOutput(path.resolve(input.outputPath));
await ensureWritableDirectory(
input.outputPath ? path.dirname(path.resolve(input.outputPath)) : path.dirname(mediaPath),
input.outputPath
? path.dirname(path.resolve(input.outputPath))
: path.dirname(destinationMediaPath),
);
const httpArgs = input.remote ? subtitleGenerationHttpArgs(input.remote.httpHeaders) : [];
const model = await resolveSubtitleGenerationModel(input.config, input.modelDirectory);
if (model.kind === 'missing')
throw new Error(
@@ -210,11 +229,16 @@ export async function generateJapaneseSubtitles(input: {
'stream=index,codec_type,start_time,duration:stream_tags=language:format=start_time,duration',
'-of',
'json',
...httpArgs,
mediaPath,
],
signal: input.signal,
});
const audio = parseAudioProbe(probe, input.audioStreamIndex);
if (isRemote && (!audio.duration || !Number.isFinite(audio.duration) || audio.duration <= 0))
throw new Error(
'Stream generation requires a finite episode duration. Live streams are not supported.',
);
const temporaryDirectory = await mkdtemp(path.join(tmpdir(), 'subminer-whisper-'));
try {
const wavPath = path.join(temporaryDirectory, 'audio.wav');
@@ -227,10 +251,12 @@ export async function generateJapaneseSubtitles(input: {
'-hide_banner',
'-loglevel',
'error',
...httpArgs,
'-i',
mediaPath,
'-map',
`0:${audio.index}`,
...(isRemote ? ['-t', String(audio.duration)] : []),
'-vn',
'-af',
'asetpts=PTS-STARTPTS',
@@ -268,6 +294,7 @@ export async function generateJapaneseSubtitles(input: {
ffmpegPath: tools.ffmpeg,
directory: temporaryDirectory,
audioOffset: audio.offset,
httpHeaders: input.remote?.httpHeaders,
onProgress: input.onProgress,
signal: input.signal,
});
@@ -317,7 +344,7 @@ export async function generateJapaneseSubtitles(input: {
input.onProgress?.({ stage: 'write', message: 'Saving Japanese subtitles...' });
const contents = shiftSubtitleTimestamps(srt, audio.offset);
const outputPath = await writeSubtitles({
mediaPath,
mediaPath: destinationMediaPath,
outputPath: input.outputPath,
contents,
signal: input.signal,
+1
View File
@@ -6887,6 +6887,7 @@ function setOverlayVisible(visible: boolean): void {
registerIpcRuntimeHandlers();
const subtitleGenerationRuntime = createSubtitleGenerationRuntime({
getConfig: () => configService.getConfig().subtitleGeneration,
getCacheDirectory: () => path.join(USER_DATA_PATH, 'cache', 'generated-subtitles'),
getModelDirectory: () =>
path.join(path.dirname(configService.getConfigPath()), 'models', 'whisper'),
getMpvClient: () => appState.mpvClient,
@@ -21,6 +21,7 @@ function fixture(overrides: Partial<SubtitleGenerationRuntimeDeps> = {}) {
const runtime = createSubtitleGenerationRuntime({
getConfig: () => DEFAULT_SUBTITLE_GENERATION_CONFIG,
getModelDirectory: () => '/models',
getCacheDirectory: () => '/cache/generated-subtitles',
getMpvClient: () => client,
onProgress: () => {},
detectAcceleration: async () => ({ kind: 'unavailable' }),
@@ -72,6 +73,67 @@ test('generation preserves the output without attaching it to a different video'
assert.deepEqual(subject.commands, []);
});
test('stream generation snapshots mpv headers and saves in the cache before loading', async () => {
const url = 'http://127.0.0.1:7777/proxy/episode.m3u8';
const subject = fixture({
generate: async (input) => {
assert.equal(input.mediaPath, url);
assert.equal(input.audioStreamIndex, 3);
assert.deepEqual(input.remote, {
cacheDirectory: '/cache/generated-subtitles',
httpHeaders: {
headers: { Referer: 'https://anime.example/', 'X-Stream': 'episode' },
userAgent: 'Anime Player',
},
});
return '/cache/generated-subtitles/episode.ja.generated.srt';
},
});
const request = subject.client.requestProperty;
subject.client.requestProperty = async (name) => {
if (name === 'path') return url;
if (name === 'file-local-options/http-header-fields')
return ['Referer: https://anime.example/', 'X-Stream: episode'];
if (name === 'file-local-options/user-agent') return 'Anime Player';
return request(name);
};
assert.equal((await subject.runtime.getStatus()).mediaPath, url);
assert.equal((await subject.runtime.start()).ok, true);
assert.deepEqual(subject.commands[0], [
'sub-add',
'/cache/generated-subtitles/episode.ja.generated.srt',
'select',
'Generated Japanese',
'ja',
]);
});
test('stream changes during header capture stop generation; changes during transcription keep the saved result', async () => {
for (const duringCapture of [true, false]) {
let current = 'https://anime.example/episode.m3u8';
const next = 'https://anime.example/next.m3u8';
let generated = false;
const subject = fixture({
generate: async () => {
generated = true;
current = next;
return '/cache/generated.srt';
},
});
const request = subject.client.requestProperty;
subject.client.requestProperty = async (name) => {
if (name === 'path') return current;
if (duringCapture && name === 'file-local-options/http-header-fields') current = next;
return request(name);
};
const result = await subject.runtime.start();
assert.equal(result.ok, !duringCapture);
assert.equal(generated, !duringCapture);
assert.match(result.message, /changed/);
assert.deepEqual(subject.commands, []);
}
});
test('generation selects loaded dialogue references and excludes the signs track', async () => {
const subject = fixture({
generate: async (input) => {
@@ -1,4 +1,5 @@
import path from 'node:path';
import { resolveMpvHttpHeaders } from '../../core/services/mpv-http-headers';
import { readSubtitleGenerationReferences } from '../../core/services/subtitle-generation-reference';
import { detectSubtitleGenerationAcceleration } from '../../core/services/subtitle-generation-acceleration';
import { SUBTITLE_GENERATION_VAD_MODEL } from '../../shared/subtitle-generation-vad-model';
@@ -31,6 +32,7 @@ interface GenerationMpvClient {
export interface SubtitleGenerationRuntimeDeps {
getConfig: () => SubtitleGenerationConfig;
getModelDirectory: () => string;
getCacheDirectory: () => string;
getMpvClient: () => GenerationMpvClient | null;
onProgress: (progress: SubtitleGenerationProgress) => void;
generate?: typeof generateJapaneseSubtitles;
@@ -42,10 +44,12 @@ export interface SubtitleGenerationRuntimeDeps {
resolveVadModel?: typeof resolveSubtitleGenerationVadModel;
}
async function currentLocalMedia(client: GenerationMpvClient | null): Promise<string | null> {
async function currentMedia(client: GenerationMpvClient | null): Promise<string | null> {
if (!client?.connected) return null;
const media = await client.requestProperty('path');
if (typeof media !== 'string' || !media || /^[a-z][a-z\d+.-]*:\/\//i.test(media)) return null;
if (typeof media !== 'string' || !media) return null;
if (/^https?:\/\//i.test(media)) return media;
if (/^[a-z][a-z\d+.-]*:\/\//i.test(media)) return null;
if (path.isAbsolute(media)) return path.normalize(media);
const directory = await client.requestProperty('working-directory');
return typeof directory === 'string' ? path.resolve(directory, media) : null;
@@ -64,7 +68,9 @@ function selectedAudioIndex(tracks: unknown): number {
)
continue;
if ('external' in track && track.external === true)
throw new Error('Select an audio track inside the local video before generating subtitles.');
throw new Error(
'Select an audio track inside the video or stream before generating subtitles.',
);
if (
'ff-index' in track &&
typeof track['ff-index'] === 'number' &&
@@ -155,7 +161,7 @@ export function createSubtitleGenerationRuntime(deps: SubtitleGenerationRuntimeD
config,
deps.getModelDirectory(),
);
const mediaPath = await currentLocalMedia(deps.getMpvClient()).catch(() => null);
const mediaPath = await currentMedia(deps.getMpvClient()).catch(() => null);
return {
model,
vad: {
@@ -236,15 +242,21 @@ export function createSubtitleGenerationRuntime(deps: SubtitleGenerationRuntimeD
if (vad.kind === 'invalid') throw new Error(vad.message);
}
const client = deps.getMpvClient();
const mediaPath = await currentLocalMedia(client);
const mediaPath = await currentMedia(client);
if (!client || !mediaPath)
throw new Error('Open a local video or audio file in mpv first.');
throw new Error('Open a local media file or HTTP stream in mpv first.');
const tracks = await client.requestProperty('track-list');
const audioStreamIndex = selectedAudioIndex(tracks);
const remote = /^https?:\/\//i.test(mediaPath)
? {
httpHeaders: await resolveMpvHttpHeaders(client),
cacheDirectory: deps.getCacheDirectory(),
}
: undefined;
const references = await readSubtitleGenerationReferences(tracks, (name) =>
client.requestProperty(name),
);
if ((await currentLocalMedia(client)) !== mediaPath)
if ((await currentMedia(client)) !== mediaPath)
throw new Error('The current media changed. Start generation again.');
signal.throwIfAborted();
const outputPath = await (deps.generate ?? generateJapaneseSubtitles)({
@@ -252,13 +264,14 @@ export function createSubtitleGenerationRuntime(deps: SubtitleGenerationRuntimeD
modelDirectory: deps.getModelDirectory(),
mediaPath,
audioStreamIndex,
remote,
references,
onProgress: report,
signal,
});
// Saving succeeds even if playback changes or disconnects during the job.
try {
const playingMedia = await currentLocalMedia(client);
const playingMedia = await currentMedia(client);
if (!signal.aborted && deps.getMpvClient() === client && playingMedia === mediaPath) {
const loaded = await client.request([
'sub-add',
+3 -2
View File
@@ -88,7 +88,7 @@ export function createSubtitleGenerationModal(
const vad = snapshot ? describeGenerationVad(snapshot.vad) : null;
const tools = snapshot ? describeGenerationTools(snapshot.tools) : null;
const readyMessage = !snapshot?.mediaPath
? 'Open local media to generate subtitles.'
? 'Open a local file or anime stream to generate subtitles.'
: !tools?.ready
? 'Install the missing tools or set their paths in Settings, then click Check again.'
: !model?.ready
@@ -96,7 +96,8 @@ export function createSubtitleGenerationModal(
: !vad?.ready
? 'Download the speech detection model or uncheck Focus on spoken dialogue.'
: 'Ready when you are.';
dom.media.textContent = snapshot?.mediaPath ?? 'Open a local media file in the player first.';
dom.media.textContent =
snapshot?.mediaPath ?? 'Open a local file or anime stream in the player first.';
dom.tools.textContent = tools?.text ?? 'Checking local tools...';
dom.model.textContent = model?.text ?? 'Checking local models...';
dom.modelPicker.classList.toggle('hidden', !snapshot || Boolean(snapshot.externalModelPath));