mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-09-19 05:16:27 -07:00
feat(subtitles): use loaded subtitles to guide generation timing (#249)
This commit is contained in:
@@ -0,0 +1,120 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { detectSubtitleGenerationAcceleration } from './subtitle-generation-acceleration';
|
||||
|
||||
async function fixture(run: (directory: string) => Promise<void>) {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'subtitle-acceleration-test-'));
|
||||
try {
|
||||
await run(directory);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function executable(directory: string, name: string, body: string) {
|
||||
const file = path.join(directory, name);
|
||||
await writeFile(file, `#!${process.execPath}\n${body}`, { mode: 0o755 });
|
||||
return file;
|
||||
}
|
||||
|
||||
const cudaOutput =
|
||||
'ggml_cuda_init: found 1 CUDA devices:\nwhisper_model_load: invalid model data (bad magic)\n';
|
||||
|
||||
test('NVIDIA and CUDA discovery work without downloading or loading a model', () =>
|
||||
fixture(async (directory) => {
|
||||
await executable(directory, 'nvidia-smi', 'console.log("NVIDIA Test GPU");');
|
||||
const whisper = await executable(
|
||||
directory,
|
||||
'whisper-cli',
|
||||
`const fs = require('node:fs');
|
||||
const model = process.argv[process.argv.indexOf('-m') + 1];
|
||||
if (fs.readFileSync(model).length !== 4) process.exit(1);
|
||||
fs.writeFileSync(${JSON.stringify(path.join(directory, 'probe-path'))}, model);
|
||||
process.stderr.write(${JSON.stringify(cudaOutput)}); process.exit(3);`,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await detectSubtitleGenerationAcceleration(
|
||||
{ kind: 'found', path: whisper },
|
||||
{ PATH: directory },
|
||||
),
|
||||
{ kind: 'nvidia-cuda', gpuName: 'NVIDIA Test GPU' },
|
||||
);
|
||||
const model = await readFile(path.join(directory, 'probe-path'), 'utf8');
|
||||
await assert.rejects(readdir(path.dirname(model)), { code: 'ENOENT' });
|
||||
}));
|
||||
|
||||
test('CPU-only, Vulkan-only, hidden CUDA devices and incomplete probes fall back safely', () =>
|
||||
fixture(async (directory) => {
|
||||
await executable(directory, 'nvidia-smi', 'console.log("NVIDIA Test GPU");');
|
||||
for (const output of [
|
||||
'usage: --no-gpu disable GPU\ninvalid model data (bad magic)',
|
||||
'ggml_vulkan: Found 1 Vulkan devices\ninvalid model data (bad magic)',
|
||||
'ggml_cuda_init: found 0 CUDA devices\ninvalid model data (bad magic)',
|
||||
'ggml_cuda_init: found 1 CUDA devices\nCUDA error: driver initialization failed',
|
||||
]) {
|
||||
const whisper = await executable(
|
||||
directory,
|
||||
'whisper-cli',
|
||||
`process.stderr.write(${JSON.stringify(output)}); process.exit(3);`,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await detectSubtitleGenerationAcceleration(
|
||||
{ kind: 'found', path: whisper },
|
||||
{ PATH: directory },
|
||||
),
|
||||
{ kind: 'unavailable' },
|
||||
);
|
||||
}
|
||||
}));
|
||||
|
||||
test('missing tools, missing NVIDIA devices and driver errors do not recommend turbo', () =>
|
||||
fixture(async (directory) => {
|
||||
const whisper = await executable(
|
||||
directory,
|
||||
'whisper-cli',
|
||||
`process.stderr.write(${JSON.stringify(cudaOutput)}); process.exit(3);`,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await detectSubtitleGenerationAcceleration(
|
||||
{ kind: 'missing', message: 'Not installed' },
|
||||
{ PATH: directory },
|
||||
),
|
||||
{ kind: 'unavailable' },
|
||||
);
|
||||
for (const driver of [
|
||||
null,
|
||||
'process.exit(0);',
|
||||
'console.log("NVIDIA GPU"); process.exit(1);',
|
||||
]) {
|
||||
if (driver !== null) await executable(directory, 'nvidia-smi', driver);
|
||||
assert.deepEqual(
|
||||
await detectSubtitleGenerationAcceleration(
|
||||
{ kind: 'found', path: whisper },
|
||||
{ PATH: directory },
|
||||
),
|
||||
{ kind: 'unavailable' },
|
||||
);
|
||||
}
|
||||
}));
|
||||
|
||||
test('a hung Whisper probe times out even if it printed a CUDA device', () =>
|
||||
fixture(async (directory) => {
|
||||
await executable(directory, 'nvidia-smi', 'console.log("NVIDIA Test GPU");');
|
||||
const whisper = await executable(
|
||||
directory,
|
||||
'whisper-cli',
|
||||
`process.stderr.write(${JSON.stringify(cudaOutput)}); setInterval(() => {}, 1000);`,
|
||||
);
|
||||
const started = Date.now();
|
||||
assert.deepEqual(
|
||||
await detectSubtitleGenerationAcceleration(
|
||||
{ kind: 'found', path: whisper },
|
||||
{ PATH: directory },
|
||||
),
|
||||
{ kind: 'unavailable' },
|
||||
);
|
||||
assert.ok(Date.now() - started < 6000);
|
||||
}));
|
||||
@@ -0,0 +1,62 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import type {
|
||||
SubtitleGenerationAcceleration,
|
||||
SubtitleGenerationToolStatus,
|
||||
} from '../../shared/subtitle-generation';
|
||||
|
||||
function probe(command: string, args: string[], env: NodeJS.ProcessEnv) {
|
||||
return new Promise<{ code: number; stdout: string; stderr: string } | null>((resolve) => {
|
||||
execFile(
|
||||
command,
|
||||
args,
|
||||
{ env, timeout: 3000, killSignal: 'SIGKILL', maxBuffer: 64 * 1024, windowsHide: true },
|
||||
(error, stdout, stderr) => {
|
||||
const code = error?.code ?? 0;
|
||||
if (typeof code !== 'number' || error?.killed || error?.signal) {
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
resolve({ code, stdout, stderr });
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** Require both a working NVIDIA driver and CUDA device discovery in the selected Whisper binary. */
|
||||
export async function detectSubtitleGenerationAcceleration(
|
||||
whisper: SubtitleGenerationToolStatus,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<SubtitleGenerationAcceleration> {
|
||||
const unavailable: SubtitleGenerationAcceleration = { kind: 'unavailable' };
|
||||
if (whisper.kind === 'missing') return unavailable;
|
||||
let directory: string | undefined;
|
||||
try {
|
||||
const nvidia = await probe('nvidia-smi', ['--query-gpu=name', '--format=csv,noheader'], env);
|
||||
const gpuName = nvidia?.stdout.trim().split(/\r?\n/)[0]?.trim();
|
||||
if (nvidia?.code !== 0 || !gpuName) return unavailable;
|
||||
|
||||
directory = await mkdtemp(path.join(tmpdir(), 'subminer-cuda-check-'));
|
||||
const model = path.join(directory, 'probe.bin');
|
||||
await writeFile(model, Buffer.alloc(4));
|
||||
// whisper.cpp discovers backends before checking model magic. This deliberately invalid
|
||||
// local file stops before allocating a model or decoding audio, even on a fresh install.
|
||||
const result = await probe(whisper.path, ['-m', model, '-f', model], env);
|
||||
if (
|
||||
result &&
|
||||
result.code !== 0 &&
|
||||
/ggml_cuda_init:\s+found\s+[1-9]\d*\s+CUDA devices?\b/i.test(result.stderr) &&
|
||||
/invalid model data \(bad magic\)/i.test(result.stderr)
|
||||
) {
|
||||
return { kind: 'nvidia-cuda', gpuName };
|
||||
}
|
||||
return unavailable;
|
||||
} catch {
|
||||
// Detection is advisory; unsupported builds and driver failures must not block generation.
|
||||
return unavailable;
|
||||
} finally {
|
||||
if (directory) await rm(directory, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,24 @@ import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { appendSpeechChunkCues, splitSpeechPassages } from './subtitle-generation-chunks';
|
||||
|
||||
test('reference starts guide long cuts ahead of VAD without dropping unreferenced audio', () => {
|
||||
const chunks = splitSpeechPassages(
|
||||
[{ startSeconds: 0, endSeconds: 70 }],
|
||||
[18],
|
||||
[19],
|
||||
[22, 43, 200],
|
||||
);
|
||||
assert.deepEqual(chunks, [
|
||||
{ startSeconds: 0, endSeconds: 22.25 },
|
||||
{ startSeconds: 21.75, endSeconds: 43.25 },
|
||||
{ startSeconds: 42.75, endSeconds: 63.25 },
|
||||
{ startSeconds: 62.75, endSeconds: 70 },
|
||||
]);
|
||||
assert.deepEqual(splitSpeechPassages([{ startSeconds: 0, endSeconds: 25 }], [], [], [10, 20]), [
|
||||
{ startSeconds: 0, endSeconds: 25 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('long coverage cuts at nearby speech starts instead of leaving a quiet lead-in', () => {
|
||||
const chunks = splitSpeechPassages(
|
||||
[{ startSeconds: 544.418, endSeconds: 581.581 }],
|
||||
|
||||
@@ -5,11 +5,12 @@ const CHUNK_CONTEXT_SECONDS = 0.25;
|
||||
const WHISPER_WINDOW_SECONDS = 30;
|
||||
const PAUSE_SEARCH_SECONDS = 5;
|
||||
|
||||
// Prefer detected speech starts, then quiet pauses. Context stays inside retained audio.
|
||||
// Prefer subtitle timing hints, then detected speech starts and quiet pauses.
|
||||
export function splitSpeechPassages(
|
||||
passages: readonly SpeechPassage[],
|
||||
pauses: readonly number[] = [],
|
||||
speechStarts: readonly number[] = [],
|
||||
referenceStarts: readonly number[] = [],
|
||||
): SpeechPassage[] {
|
||||
return passages.flatMap((passage) => {
|
||||
if (passage.endSeconds - passage.startSeconds <= WHISPER_WINDOW_SECONDS)
|
||||
@@ -22,17 +23,19 @@ export function splitSpeechPassages(
|
||||
if (target < passage.endSeconds) {
|
||||
// Starting in a long quiet lead-in can make Whisper place the next line
|
||||
// several seconds early. A nearby VAD start gives the next chunk an anchor.
|
||||
let nearestSpeechStart: number | undefined;
|
||||
for (const time of speechStarts) {
|
||||
if (
|
||||
time >= target - PAUSE_SEARCH_SECONDS &&
|
||||
time <= target + PAUSE_SEARCH_SECONDS &&
|
||||
time < passage.endSeconds &&
|
||||
(nearestSpeechStart === undefined ||
|
||||
Math.abs(time - target) < Math.abs(nearestSpeechStart - target))
|
||||
)
|
||||
nearestSpeechStart = time;
|
||||
}
|
||||
const nearestStart = (starts: readonly number[]): number | undefined => {
|
||||
let nearest: number | undefined;
|
||||
for (const time of starts) {
|
||||
if (
|
||||
time >= target - PAUSE_SEARCH_SECONDS &&
|
||||
time <= target + PAUSE_SEARCH_SECONDS &&
|
||||
time < passage.endSeconds &&
|
||||
(nearest === undefined || Math.abs(time - target) < Math.abs(nearest - target))
|
||||
)
|
||||
nearest = time;
|
||||
}
|
||||
return nearest;
|
||||
};
|
||||
let latestPause: number | undefined;
|
||||
for (const time of pauses) {
|
||||
if (
|
||||
@@ -42,7 +45,7 @@ export function splitSpeechPassages(
|
||||
)
|
||||
latestPause = time;
|
||||
}
|
||||
end = nearestSpeechStart ?? latestPause ?? end;
|
||||
end = nearestStart(referenceStarts) ?? nearestStart(speechStarts) ?? latestPause ?? end;
|
||||
}
|
||||
chunks.push({
|
||||
startSeconds: Math.max(passage.startSeconds, boundary - CHUNK_CONTEXT_SECONDS),
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
parseSpeechPassages,
|
||||
speechPassageCues,
|
||||
SPEECH_PASSAGE_SECONDS,
|
||||
type SpeechPassage,
|
||||
} from './subtitle-generation-speech';
|
||||
import type { SubtitleCue } from './subtitle-cue-parser';
|
||||
import { appendSpeechChunkCues, splitSpeechPassages } from './subtitle-generation-chunks';
|
||||
@@ -21,46 +22,50 @@ import { findAudiblePassages, mergeSpeechPassages } from './subtitle-generation-
|
||||
|
||||
export async function transcribeSubtitleDialogue(input: {
|
||||
config: SubtitleGenerationConfig;
|
||||
tools: SubtitleGenerationToolPaths & { vad: string };
|
||||
tools: SubtitleGenerationToolPaths;
|
||||
referenceStarts?: readonly number[];
|
||||
modelPath: string;
|
||||
wavPath: string;
|
||||
directory: string;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
const vadModelPath = expandSubtitleGenerationPath(input.config.vadModelPath);
|
||||
await access(vadModelPath, constants.R_OK);
|
||||
input.onProgress?.({ stage: 'transcribe', percent: 0, message: 'Finding spoken dialogue...' });
|
||||
const segmentLines: string[] = [];
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.tools.vad,
|
||||
args: [
|
||||
'-f',
|
||||
input.wavPath,
|
||||
'-vm',
|
||||
vadModelPath,
|
||||
'-t',
|
||||
String(input.config.threads),
|
||||
'-vt',
|
||||
'0.3',
|
||||
'--vad-min-speech-duration-ms',
|
||||
'100',
|
||||
'--vad-min-silence-duration-ms',
|
||||
'500',
|
||||
'-vp',
|
||||
'350',
|
||||
'-vmsd',
|
||||
String(SPEECH_PASSAGE_SECONDS),
|
||||
'-np',
|
||||
],
|
||||
signal: input.signal,
|
||||
// Capture structured result lines separately from the bounded process log.
|
||||
onLine: (line) => {
|
||||
if (line.startsWith('Detected ') || line.startsWith('Speech segment '))
|
||||
segmentLines.push(line);
|
||||
},
|
||||
});
|
||||
const speech = parseSpeechPassages(segmentLines.join('\n'));
|
||||
let speech: SpeechPassage[] = [];
|
||||
if (input.tools.vad !== null) {
|
||||
const vadModelPath = expandSubtitleGenerationPath(input.config.vadModelPath);
|
||||
await access(vadModelPath, constants.R_OK);
|
||||
input.onProgress?.({ stage: 'transcribe', percent: 0, message: 'Finding spoken dialogue...' });
|
||||
const segmentLines: string[] = [];
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.tools.vad,
|
||||
args: [
|
||||
'-f',
|
||||
input.wavPath,
|
||||
'-vm',
|
||||
vadModelPath,
|
||||
'-t',
|
||||
String(input.config.threads),
|
||||
'-vt',
|
||||
'0.3',
|
||||
'--vad-min-speech-duration-ms',
|
||||
'100',
|
||||
'--vad-min-silence-duration-ms',
|
||||
'500',
|
||||
'-vp',
|
||||
'350',
|
||||
'-vmsd',
|
||||
String(SPEECH_PASSAGE_SECONDS),
|
||||
'-np',
|
||||
],
|
||||
signal: input.signal,
|
||||
// Capture structured result lines separately from the bounded process log.
|
||||
onLine: (line) => {
|
||||
if (line.startsWith('Detected ') || line.startsWith('Speech segment '))
|
||||
segmentLines.push(line);
|
||||
},
|
||||
});
|
||||
speech = parseSpeechPassages(segmentLines.join('\n'));
|
||||
}
|
||||
input.onProgress?.({ stage: 'transcribe', percent: 0, message: 'Checking audio coverage...' });
|
||||
const audible = await findAudiblePassages({
|
||||
ffmpegPath: input.tools.ffmpeg,
|
||||
@@ -82,6 +87,7 @@ export async function transcribeSubtitleDialogue(input: {
|
||||
detected,
|
||||
pauses,
|
||||
speech.map((passage) => passage.startSeconds),
|
||||
input.referenceStarts,
|
||||
);
|
||||
const cues: SubtitleCue[] = [];
|
||||
for (const [index, passage] of passages.entries()) {
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
loadSubtitleGenerationReference,
|
||||
readSubtitleGenerationReferences,
|
||||
subtitleGenerationReferences,
|
||||
} from './subtitle-generation-reference';
|
||||
|
||||
const embedded = { type: 'sub', codec: 'ass', 'ff-index': 2, lang: 'eng' };
|
||||
|
||||
test('references exclude signs, songs, forced, bitmap and generated tracks even when selected', () => {
|
||||
const excluded = [
|
||||
'Signs & Songs',
|
||||
'SignsSongs',
|
||||
'Signs/Songs',
|
||||
'S&S',
|
||||
'S+S',
|
||||
'Forced',
|
||||
'Karaoke',
|
||||
'OP',
|
||||
'ED',
|
||||
'English lyrics',
|
||||
'Generated Japanese',
|
||||
];
|
||||
assert.deepEqual(
|
||||
subtitleGenerationReferences([
|
||||
...excluded.map((title) => ({ ...embedded, title, selected: true })),
|
||||
{ ...embedded, forced: true },
|
||||
{ ...embedded, codec: 'hdmv_pgs_subtitle' },
|
||||
{
|
||||
...embedded,
|
||||
title: 'English',
|
||||
external: true,
|
||||
'external-filename': '/subs/show.en.signs.ass',
|
||||
},
|
||||
{ ...embedded, title: 'English Full' },
|
||||
]).map((reference) => reference.label),
|
||||
['English Full'],
|
||||
);
|
||||
});
|
||||
|
||||
test('references rank English dialogue first and resolve loaded external files against mpv cwd', () => {
|
||||
const refs = subtitleGenerationReferences(
|
||||
[
|
||||
{ ...embedded, title: 'French Full', lang: 'fra', selected: true },
|
||||
{ ...embedded, title: 'English' },
|
||||
{ type: 'sub', external: true, 'external-filename': 'subs/show.en.full.srt' },
|
||||
{ type: 'sub', external: true, 'external-filename': 'https://example.com/en.srt' },
|
||||
{ type: 'sub', 'ff-index': -1 },
|
||||
null,
|
||||
],
|
||||
'/mpv',
|
||||
);
|
||||
assert.deepEqual(
|
||||
refs.map((ref) => ref.label),
|
||||
['show.en.full.srt', 'English', 'French Full'],
|
||||
);
|
||||
assert.deepEqual(refs[0]?.source, { kind: 'external', path: '/mpv/subs/show.en.full.srt' });
|
||||
assert.deepEqual(
|
||||
subtitleGenerationReferences([
|
||||
{ type: 'sub', external: true, 'external-filename': 'relative.en.srt' },
|
||||
]),
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test('reference discovery captures primary and secondary subtitle delays', async () => {
|
||||
const properties: Record<string, unknown> = {
|
||||
'working-directory': '/mpv',
|
||||
sid: 1,
|
||||
'secondary-sid': 2,
|
||||
'sub-delay': 1.5,
|
||||
'secondary-sub-delay': -2,
|
||||
};
|
||||
const refs = await readSubtitleGenerationReferences(
|
||||
[
|
||||
{ ...embedded, id: 1 },
|
||||
{ ...embedded, id: 2 },
|
||||
{ ...embedded, id: 3 },
|
||||
],
|
||||
async (name) => properties[name],
|
||||
);
|
||||
assert.deepEqual(
|
||||
refs.map((ref) => ref.delaySeconds),
|
||||
[1.5, -2, 0],
|
||||
);
|
||||
});
|
||||
|
||||
test('reference extraction retries unreadable tracks, restores audio offset and ignores marked lyrics', async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'generation-reference-'));
|
||||
try {
|
||||
const ffmpegPath = path.join(directory, 'ffmpeg');
|
||||
await writeFile(
|
||||
ffmpegPath,
|
||||
`#!${process.execPath}
|
||||
const args = process.argv.slice(2);
|
||||
if (args[args.indexOf('-map') + 1] === '0:2') process.exit(1);
|
||||
require('node:fs').writeFileSync(args.at(-1), '1\\n00:00:10,000 --> 00:00:12,000\\nHello\\n\\n2\\n00:00:20,000 --> 00:00:22,000\\n♪ Song ♪\\n\\n3\\n00:00:30,000 --> 00:00:32,000\\nWorld\\n');
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
const refs = subtitleGenerationReferences([{ ...embedded }, { ...embedded, 'ff-index': 3 }]);
|
||||
const hints = await loadSubtitleGenerationReference({
|
||||
references: refs,
|
||||
mediaPath: '/video.mkv',
|
||||
ffmpegPath,
|
||||
directory,
|
||||
audioOffset: 2.5,
|
||||
});
|
||||
assert.deepEqual(hints, [7.5, 27.5]);
|
||||
assert.deepEqual(
|
||||
await loadSubtitleGenerationReference({
|
||||
references: refs.slice(0, 1),
|
||||
mediaPath: '/video.mkv',
|
||||
ffmpegPath,
|
||||
directory,
|
||||
audioOffset: 0,
|
||||
}),
|
||||
[],
|
||||
);
|
||||
await assert.rejects(
|
||||
loadSubtitleGenerationReference({
|
||||
references: refs,
|
||||
mediaPath: '/video.mkv',
|
||||
ffmpegPath,
|
||||
directory,
|
||||
audioOffset: 0,
|
||||
signal: AbortSignal.abort(),
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,169 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import type { SubtitleGenerationProgress } from '../../shared/subtitle-generation';
|
||||
import { parseSrtCues } from './subtitle-cue-parser';
|
||||
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
|
||||
|
||||
export type SubtitleGenerationReference = {
|
||||
label: string;
|
||||
delaySeconds: number;
|
||||
source: { kind: 'embedded'; streamIndex: number } | { kind: 'external'; path: string };
|
||||
};
|
||||
|
||||
// Check both titles and filenames: releases often tag only one of them.
|
||||
const EXCLUDED =
|
||||
/(?:^|[^\p{L}\p{N}])(?:signs?(?:songs?)?|songs?|lyrics?|karaoke|forced|s[\s&+_-]*s|op|ed|opening|ending|generated)(?=$|[^\p{L}\p{N}])|看板|歌詞/iu;
|
||||
const TEXT_CODECS = new Set(['ass', 'ssa', 'subrip', 'srt', 'webvtt', 'mov_text', 'text']);
|
||||
|
||||
/** Rank loaded dialogue tracks, preferring English, then explicitly full tracks. */
|
||||
export function subtitleGenerationReferences(
|
||||
value: unknown,
|
||||
workingDirectory?: string,
|
||||
delays: ReadonlyMap<number, number> = new Map(),
|
||||
): SubtitleGenerationReference[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
const tracks: unknown[] = value;
|
||||
return tracks
|
||||
.flatMap((track) => {
|
||||
if (typeof track !== 'object' || track === null || !('type' in track) || track.type !== 'sub')
|
||||
return [];
|
||||
const title = 'title' in track && typeof track.title === 'string' ? track.title : '';
|
||||
const filename =
|
||||
'external-filename' in track && typeof track['external-filename'] === 'string'
|
||||
? track['external-filename']
|
||||
: '';
|
||||
const name = `${title} ${path.basename(filename)}`;
|
||||
if (('forced' in track && track.forced === true) || EXCLUDED.test(name)) return [];
|
||||
if ('codec' in track && typeof track.codec === 'string' && !TEXT_CODECS.has(track.codec))
|
||||
return [];
|
||||
let source: SubtitleGenerationReference['source'];
|
||||
if ('external' in track && track.external === true) {
|
||||
let local = filename;
|
||||
if (local.startsWith('file://')) {
|
||||
try {
|
||||
local = fileURLToPath(local);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
} else if (/^[a-z][a-z\d+.-]*:\/\//i.test(local)) return [];
|
||||
if (!local || (!path.isAbsolute(local) && !workingDirectory)) return [];
|
||||
if (!/\.(?:srt|ass|ssa|vtt)$/i.test(local)) return [];
|
||||
source = { kind: 'external', path: path.resolve(workingDirectory ?? '.', local) };
|
||||
} else {
|
||||
if (
|
||||
!('ff-index' in track) ||
|
||||
typeof track['ff-index'] !== 'number' ||
|
||||
!Number.isSafeInteger(track['ff-index']) ||
|
||||
track['ff-index'] < 0
|
||||
)
|
||||
return [];
|
||||
source = { kind: 'embedded', streamIndex: track['ff-index'] };
|
||||
}
|
||||
const language = 'lang' in track && typeof track.lang === 'string' ? track.lang : '';
|
||||
const english =
|
||||
/^(?:en|eng|english)(?:[-_]|$)/i.test(language) ||
|
||||
/(?:^|[\s.\[(_-])(?:en|eng|english)(?=$|[\s.\])_-])/i.test(name);
|
||||
const full = /\b(?:full|dialogue|dialog)\b/i.test(name);
|
||||
const selected = 'selected' in track && track.selected === true;
|
||||
const preferred = 'default' in track && track.default === true;
|
||||
return [
|
||||
{
|
||||
reference: {
|
||||
label:
|
||||
title ||
|
||||
path.basename(filename) ||
|
||||
`${language || 'Subtitle'} stream ${source.kind === 'embedded' ? source.streamIndex : ''}`,
|
||||
source,
|
||||
delaySeconds:
|
||||
'id' in track && typeof track.id === 'number' ? (delays.get(track.id) ?? 0) : 0,
|
||||
},
|
||||
score:
|
||||
Number(english) * 100 + Number(full) * 20 + Number(selected) * 2 + Number(preferred),
|
||||
},
|
||||
];
|
||||
})
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.map(({ reference }) => reference);
|
||||
}
|
||||
|
||||
/** Read mpv's path base and active subtitle delays while capturing timing references. */
|
||||
export async function readSubtitleGenerationReferences(
|
||||
tracks: unknown,
|
||||
requestProperty: (name: string) => Promise<unknown>,
|
||||
): Promise<SubtitleGenerationReference[]> {
|
||||
const [directory, primary, secondary, primaryDelay, secondaryDelay] = await Promise.all(
|
||||
['working-directory', 'sid', 'secondary-sid', 'sub-delay', 'secondary-sub-delay'].map((name) =>
|
||||
requestProperty(name).catch(() => null),
|
||||
),
|
||||
);
|
||||
const delays = new Map<number, number>();
|
||||
for (const [id, delay] of [
|
||||
[primary, primaryDelay],
|
||||
[secondary, secondaryDelay],
|
||||
]) {
|
||||
if (typeof id === 'number' && typeof delay === 'number' && Number.isFinite(delay))
|
||||
delays.set(id, delay);
|
||||
}
|
||||
return subtitleGenerationReferences(
|
||||
tracks,
|
||||
typeof directory === 'string' ? directory : undefined,
|
||||
delays,
|
||||
);
|
||||
}
|
||||
|
||||
/** Reference timestamps are hints on the extracted audio timeline, never a coverage mask. */
|
||||
export async function loadSubtitleGenerationReference(input: {
|
||||
references: readonly SubtitleGenerationReference[];
|
||||
mediaPath: string;
|
||||
ffmpegPath: string;
|
||||
directory: string;
|
||||
audioOffset: number;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<number[]> {
|
||||
for (const [index, reference] of input.references.entries()) {
|
||||
input.signal?.throwIfAborted();
|
||||
try {
|
||||
const output = path.join(input.directory, `reference-${index}.srt`);
|
||||
const embedded = reference.source.kind === 'embedded';
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.ffmpegPath,
|
||||
args: [
|
||||
'-nostdin',
|
||||
'-hide_banner',
|
||||
'-loglevel',
|
||||
'error',
|
||||
...(embedded ? ['-copyts', '-start_at_zero'] : []),
|
||||
'-i',
|
||||
reference.source.kind === 'external' ? reference.source.path : input.mediaPath,
|
||||
'-map',
|
||||
reference.source.kind === 'embedded' ? `0:${reference.source.streamIndex}` : '0:s:0',
|
||||
'-c:s',
|
||||
'srt',
|
||||
output,
|
||||
],
|
||||
signal: input.signal,
|
||||
});
|
||||
const starts = parseSrtCues(await readFile(output, 'utf8'))
|
||||
.filter((cue) => cue.text.trim() && !/[♪♫]/u.test(cue.text) && cue.endTime > cue.startTime)
|
||||
.map((cue) => cue.startTime + reference.delaySeconds - input.audioOffset)
|
||||
.filter((time) => Number.isFinite(time) && time >= 0);
|
||||
if (starts.length === 0) continue;
|
||||
input.onProgress?.({
|
||||
stage: 'extract',
|
||||
message: `Using subtitle timing reference: ${reference.label}`,
|
||||
});
|
||||
return [...new Set(starts)].sort((a, b) => a - b);
|
||||
} catch {
|
||||
input.signal?.throwIfAborted();
|
||||
// An optional reference must not prevent transcription. Try the next loaded track.
|
||||
}
|
||||
}
|
||||
if (input.references.length)
|
||||
input.onProgress?.({
|
||||
stage: 'extract',
|
||||
message: 'No readable subtitle timing reference. Using audio timing.',
|
||||
});
|
||||
return [];
|
||||
}
|
||||
@@ -95,6 +95,52 @@ test('config parser accepts supported models and rejects unsafe threads and wron
|
||||
assert.deepEqual(warnings, ['whisperPath', 'threads']);
|
||||
});
|
||||
|
||||
test('generation uses reference cuts with and without VAD while preserving media offsets', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const ffmpegPath = await executable(
|
||||
directory,
|
||||
'reference-ffmpeg',
|
||||
`
|
||||
const fs = require('node:fs');
|
||||
const args = process.argv.slice(2);
|
||||
fs.appendFileSync(${JSON.stringify(input.callsPath)}, JSON.stringify(args) + '\\n');
|
||||
if (args.includes('-c:s')) {
|
||||
fs.writeFileSync(args.at(-1), '1\\n00:00:24,500 --> 00:00:26,500\\nHello\\n\\n2\\n00:00:45,500 --> 00:00:47,500\\nWorld\\n');
|
||||
} else if (args.at(-1) === '-') {
|
||||
process.stdout.write('out_time_us=70000000\\nprogress=end\\n');
|
||||
} else fs.writeFileSync(args.at(-1), 'wav');
|
||||
`,
|
||||
);
|
||||
const vadPath = await executable(
|
||||
directory,
|
||||
'reference-vad',
|
||||
"process.stdout.write('Detected 1 speech segments:\\nSpeech segment 0: start = 0.000, end = 7000.000\\n');",
|
||||
);
|
||||
const vadModelPath = path.join(directory, 'vad.bin');
|
||||
await writeFile(vadModelPath, 'model');
|
||||
for (const vad of [false, true]) {
|
||||
await writeFile(input.callsPath, '');
|
||||
const output = await generateJapaneseSubtitles({
|
||||
...input,
|
||||
config: { ...input.config, ffmpegPath, vadPath, vadModelPath: vad ? vadModelPath : '' },
|
||||
references: [
|
||||
{ label: 'English Full', delaySeconds: 0, source: { kind: 'embedded', streamIndex: 5 } },
|
||||
],
|
||||
});
|
||||
const calls: string[][] = (await readFile(input.callsPath, 'utf8'))
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((line) => JSON.parse(line));
|
||||
const clips = calls.filter((args) => args.includes('-ss'));
|
||||
assert.equal(clips[0]?.[clips[0].indexOf('-t') + 1], '22.25');
|
||||
assert.equal(clips[1]?.[clips[1].indexOf('-ss') + 1], '21.75');
|
||||
const srt = await readFile(output, 'utf8');
|
||||
assert.match(srt, /00:00:03,500 --> 00:00:04,500/);
|
||||
assert.match(srt, /00:00:25,250 --> 00:00:26,250/);
|
||||
}
|
||||
}));
|
||||
|
||||
test('external model path wins and invalid external models never fall back to download', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
|
||||
@@ -11,6 +11,10 @@ import { runSubtitleGenerationProcess } from './subtitle-generation-process';
|
||||
import { publishSubtitleGenerationFile } from './subtitle-generation-files';
|
||||
import { formatTimestamp } from './subtitle-generation-srt';
|
||||
import { transcribeSubtitleDialogue } from './subtitle-generation-dialogue';
|
||||
import {
|
||||
loadSubtitleGenerationReference,
|
||||
type SubtitleGenerationReference,
|
||||
} from './subtitle-generation-reference';
|
||||
import {
|
||||
requireSubtitleGenerationTools,
|
||||
resolveSubtitleGenerationTools,
|
||||
@@ -174,6 +178,7 @@ export async function generateJapaneseSubtitles(input: {
|
||||
modelDirectory: string;
|
||||
mediaPath: string;
|
||||
audioStreamIndex?: number;
|
||||
references?: readonly SubtitleGenerationReference[];
|
||||
outputPath?: string;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
@@ -257,11 +262,21 @@ export async function generateJapaneseSubtitles(input: {
|
||||
percent: 0,
|
||||
message: 'Generating Japanese subtitles...',
|
||||
});
|
||||
const referenceStarts = await loadSubtitleGenerationReference({
|
||||
references: input.references ?? [],
|
||||
mediaPath,
|
||||
ffmpegPath: tools.ffmpeg,
|
||||
directory: temporaryDirectory,
|
||||
audioOffset: audio.offset,
|
||||
onProgress: input.onProgress,
|
||||
signal: input.signal,
|
||||
});
|
||||
let srt: string;
|
||||
if (tools.vad !== null) {
|
||||
if (tools.vad !== null || referenceStarts.length > 0) {
|
||||
srt = await transcribeSubtitleDialogue({
|
||||
config: input.config,
|
||||
tools: { ...tools, vad: tools.vad },
|
||||
tools,
|
||||
referenceStarts,
|
||||
modelPath: model.path,
|
||||
wavPath,
|
||||
directory: temporaryDirectory,
|
||||
|
||||
Reference in New Issue
Block a user