mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-09-17 17:16:23 -07:00
feat(subtitles): add local Japanese subtitle generation (#240)
This commit is contained in:
@@ -29,6 +29,7 @@ function makeShortcuts(overrides: Partial<ConfiguredShortcuts> = {}): Configured
|
||||
openRuntimeOptions: null,
|
||||
openJimaku: null,
|
||||
openTsukihime: null,
|
||||
openSubtitleGeneration: null,
|
||||
openSessionHelp: null,
|
||||
openControllerSelect: null,
|
||||
openControllerDebug: null,
|
||||
|
||||
@@ -24,6 +24,7 @@ function createShortcuts(overrides: Partial<ConfiguredShortcuts> = {}): Configur
|
||||
openRuntimeOptions: null,
|
||||
openJimaku: null,
|
||||
openTsukihime: null,
|
||||
openSubtitleGeneration: null,
|
||||
openSessionHelp: null,
|
||||
openControllerSelect: null,
|
||||
openControllerDebug: null,
|
||||
|
||||
@@ -41,6 +41,7 @@ function createDeps(overrides: Partial<SessionActionExecutorDeps> = {}) {
|
||||
openControllerDebug: () => calls.push('controller-debug'),
|
||||
openJimaku: () => calls.push('jimaku'),
|
||||
openTsukihime: () => calls.push('tsukihime'),
|
||||
openSubtitleGeneration: () => calls.push('subtitle-generation'),
|
||||
openYoutubeTrackPicker: () => {
|
||||
calls.push('youtube');
|
||||
},
|
||||
@@ -85,3 +86,9 @@ test('dispatchSessionAction opens the character dictionary manager', async () =>
|
||||
|
||||
assert.deepEqual(calls, ['character-dictionary-manager']);
|
||||
});
|
||||
|
||||
test('dispatchSessionAction opens subtitle generation without opening the sidebar', async () => {
|
||||
const { calls, deps } = createDeps();
|
||||
await dispatchSessionAction({ actionId: 'openSubtitleGeneration' }, deps);
|
||||
assert.deepEqual(calls, ['subtitle-generation']);
|
||||
});
|
||||
|
||||
@@ -25,6 +25,7 @@ export interface SessionActionExecutorDeps {
|
||||
openControllerDebug: () => void;
|
||||
openJimaku: () => void;
|
||||
openTsukihime: () => void;
|
||||
openSubtitleGeneration: () => void;
|
||||
openYoutubeTrackPicker: () => void | Promise<void>;
|
||||
openPlaylistBrowser: () => boolean | void | Promise<boolean | void>;
|
||||
replayCurrentSubtitle: () => void;
|
||||
@@ -119,6 +120,9 @@ export async function dispatchSessionAction(
|
||||
case 'openTsukihime':
|
||||
deps.openTsukihime();
|
||||
return;
|
||||
case 'openSubtitleGeneration':
|
||||
deps.openSubtitleGeneration();
|
||||
return;
|
||||
case 'openYoutubePicker':
|
||||
await deps.openYoutubeTrackPicker();
|
||||
return;
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { ConfiguredShortcuts } from '../utils/shortcut-config';
|
||||
import { DEFAULT_CONFIG, DEFAULT_KEYBINDINGS, SPECIAL_COMMANDS } from '../../config/definitions';
|
||||
import { resolveConfiguredShortcuts } from '../utils/shortcut-config';
|
||||
import { buildPluginSessionBindingsArtifact, compileSessionBindings } from './session-bindings';
|
||||
import { parseSessionActionDispatchRequest } from '../../shared/ipc/validators';
|
||||
|
||||
function createShortcuts(overrides: Partial<ConfiguredShortcuts> = {}): ConfiguredShortcuts {
|
||||
return {
|
||||
@@ -23,6 +24,7 @@ function createShortcuts(overrides: Partial<ConfiguredShortcuts> = {}): Configur
|
||||
openRuntimeOptions: null,
|
||||
openJimaku: null,
|
||||
openTsukihime: null,
|
||||
openSubtitleGeneration: null,
|
||||
openSessionHelp: null,
|
||||
openControllerSelect: null,
|
||||
openControllerDebug: null,
|
||||
@@ -37,6 +39,50 @@ function createKeybinding(key: string, command: Keybinding['command']): Keybindi
|
||||
return { key, command };
|
||||
}
|
||||
|
||||
test('subtitle generation shortcut compiles for overlay and mpv without conflicting with field grouping', () => {
|
||||
for (const platform of ['linux', 'darwin', 'win32'] as const) {
|
||||
const result = compileSessionBindings({
|
||||
shortcuts: resolveConfiguredShortcuts(DEFAULT_CONFIG, DEFAULT_CONFIG),
|
||||
keybindings: DEFAULT_KEYBINDINGS,
|
||||
platform,
|
||||
});
|
||||
const binding = result.bindings.find(
|
||||
(entry) =>
|
||||
entry.actionType === 'session-action' && entry.actionId === 'openSubtitleGeneration',
|
||||
);
|
||||
assert.ok(binding);
|
||||
assert.deepEqual(binding.key, { code: 'KeyG', modifiers: ['ctrl', 'shift'] });
|
||||
assert.equal(
|
||||
result.warnings.some(
|
||||
(warning) =>
|
||||
warning.path === 'shortcuts.openSubtitleGeneration' ||
|
||||
warning.conflictingPaths?.includes('shortcuts.openSubtitleGeneration'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.ok(
|
||||
result.bindings.some(
|
||||
(entry) =>
|
||||
entry.actionType === 'session-action' && entry.actionId === 'triggerFieldGrouping',
|
||||
),
|
||||
);
|
||||
const artifact = buildPluginSessionBindingsArtifact({
|
||||
bindings: [binding],
|
||||
warnings: [],
|
||||
numericSelectionTimeoutMs: 3000,
|
||||
});
|
||||
const pluginBinding = artifact.bindings[0];
|
||||
assert.ok(pluginBinding?.actionType === 'session-action');
|
||||
assert.deepEqual(pluginBinding.cliArgs, [
|
||||
'--session-action',
|
||||
'{"actionId":"openSubtitleGeneration"}',
|
||||
]);
|
||||
assert.deepEqual(parseSessionActionDispatchRequest({ actionId: 'openSubtitleGeneration' }), {
|
||||
actionId: 'openSubtitleGeneration',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('compileSessionBindings merges shortcuts and keybindings into one canonical list', () => {
|
||||
const result = compileSessionBindings({
|
||||
shortcuts: createShortcuts({
|
||||
|
||||
@@ -56,6 +56,7 @@ const SESSION_SHORTCUT_ACTIONS: Array<{
|
||||
{ key: 'openRuntimeOptions', actionId: 'openRuntimeOptions' },
|
||||
{ key: 'openJimaku', actionId: 'openJimaku' },
|
||||
{ key: 'openTsukihime', actionId: 'openTsukihime' },
|
||||
{ key: 'openSubtitleGeneration', actionId: 'openSubtitleGeneration' },
|
||||
{ key: 'openSessionHelp', actionId: 'openSessionHelp' },
|
||||
{ key: 'openControllerSelect', actionId: 'openControllerSelect' },
|
||||
{ key: 'openControllerDebug', actionId: 'openControllerDebug' },
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { appendSpeechChunkCues, splitSpeechPassages } from './subtitle-generation-chunks';
|
||||
|
||||
test('long coverage cuts at nearby speech starts instead of leaving a quiet lead-in', () => {
|
||||
const chunks = splitSpeechPassages(
|
||||
[{ startSeconds: 544.418, endSeconds: 581.581 }],
|
||||
[563.928],
|
||||
[555.07, 567.23, 579.91],
|
||||
);
|
||||
assert.deepEqual(chunks, [
|
||||
{ startSeconds: 544.418, endSeconds: 567.48 },
|
||||
{ startSeconds: 566.98, endSeconds: 581.581 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('short audible passages stay intact even with several detected speech starts', () => {
|
||||
assert.deepEqual(
|
||||
splitSpeechPassages(
|
||||
[{ startSeconds: 876.897, endSeconds: 897.812 }],
|
||||
[],
|
||||
[876.9, 881.15, 893.95, 897.99],
|
||||
),
|
||||
[{ startSeconds: 876.897, endSeconds: 897.812 }],
|
||||
);
|
||||
});
|
||||
|
||||
test('speech anchors outside retained coverage cannot extend a chunk across a silent gap', () => {
|
||||
const chunks = splitSpeechPassages(
|
||||
[{ startSeconds: 100, endSeconds: 142 }],
|
||||
[118],
|
||||
[90, 142, 144],
|
||||
);
|
||||
assert.deepEqual(chunks, [
|
||||
{ startSeconds: 100, endSeconds: 118.25 },
|
||||
{ startSeconds: 117.75, endSeconds: 138.25 },
|
||||
{ startSeconds: 137.75, endSeconds: 142 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('long speech splits near a pause with context on both sides and no lost audio', () => {
|
||||
assert.deepEqual(splitSpeechPassages([{ startSeconds: 100, endSeconds: 145 }], [105, 118, 137]), [
|
||||
{ startSeconds: 100, endSeconds: 118.25 },
|
||||
{ startSeconds: 117.75, endSeconds: 137.25 },
|
||||
{ startSeconds: 136.75, endSeconds: 145 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('uninterrupted speech retains overlapping context without crossing omitted gaps', () => {
|
||||
const chunks = splitSpeechPassages([
|
||||
{ startSeconds: 0, endSeconds: 60 },
|
||||
{ startSeconds: 100, endSeconds: 100.15 },
|
||||
]);
|
||||
assert.deepEqual(chunks, [
|
||||
{ startSeconds: 0, endSeconds: 20.25 },
|
||||
{ startSeconds: 19.75, endSeconds: 40.25 },
|
||||
{ startSeconds: 39.75, endSeconds: 60 },
|
||||
{ startSeconds: 100, endSeconds: 100.15 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('speech fitting one Whisper window stays intact instead of cutting a sentence at 20 seconds', () => {
|
||||
const passage = { startSeconds: 251.71, endSeconds: 275.01 };
|
||||
assert.deepEqual(splitSpeechPassages([passage], [271.327]), [passage]);
|
||||
});
|
||||
|
||||
test('chunk stitching ignores punctuation differences without merging separate repetitions', () => {
|
||||
const cues = [{ startTime: 19.7, endTime: 21.2, text: 'ありがとう' }];
|
||||
appendSpeechChunkCues(cues, [
|
||||
{ startTime: 19.8, endTime: 21.3, text: 'ありがとう。' },
|
||||
{ startTime: 22, endTime: 23, text: 'ありがとう！' },
|
||||
]);
|
||||
assert.deepEqual(cues, [
|
||||
{ startTime: 19.7, endTime: 21.3, text: 'ありがとう' },
|
||||
{ startTime: 22, endTime: 23, text: 'ありがとう！' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('chunk stitching removes matching overlap cues but retains repeated dialogue', () => {
|
||||
const cues = [{ startTime: 19.7, endTime: 20.2, text: 'はい' }];
|
||||
appendSpeechChunkCues(cues, [
|
||||
{ startTime: 19.8, endTime: 20.3, text: 'はい' },
|
||||
{ startTime: 21, endTime: 21.5, text: 'はい' },
|
||||
{ startTime: 21.4, endTime: 22, text: 'はい' },
|
||||
]);
|
||||
assert.deepEqual(cues, [
|
||||
{ startTime: 19.7, endTime: 20.3, text: 'はい' },
|
||||
{ startTime: 21, endTime: 21.5, text: 'はい' },
|
||||
{ startTime: 21.4, endTime: 22, text: 'はい' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('chunk stitching matches repeated text to the greatest overlap without leaving a duplicate', () => {
|
||||
const cues = [
|
||||
{ startTime: 10, endTime: 14, text: 'はい' },
|
||||
{ startTime: 13, endTime: 20, text: 'はい' },
|
||||
];
|
||||
appendSpeechChunkCues(cues, [
|
||||
{ startTime: 12, endTime: 21, text: 'はい' },
|
||||
{ startTime: 22, endTime: 23, text: 'はい' },
|
||||
]);
|
||||
assert.deepEqual(cues, [
|
||||
{ startTime: 10, endTime: 14, text: 'はい' },
|
||||
{ startTime: 12, endTime: 21, text: 'はい' },
|
||||
{ startTime: 22, endTime: 23, text: 'はい' },
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,87 @@
|
||||
import type { SubtitleCue } from './subtitle-cue-parser';
|
||||
import { SPEECH_PASSAGE_SECONDS, type SpeechPassage } from './subtitle-generation-speech';
|
||||
|
||||
const CHUNK_CONTEXT_SECONDS = 0.25;
|
||||
const WHISPER_WINDOW_SECONDS = 30;
|
||||
const PAUSE_SEARCH_SECONDS = 5;
|
||||
|
||||
// Prefer detected speech starts, then quiet pauses. Context stays inside retained audio.
|
||||
export function splitSpeechPassages(
|
||||
passages: readonly SpeechPassage[],
|
||||
pauses: readonly number[] = [],
|
||||
speechStarts: readonly number[] = [],
|
||||
): SpeechPassage[] {
|
||||
return passages.flatMap((passage) => {
|
||||
if (passage.endSeconds - passage.startSeconds <= WHISPER_WINDOW_SECONDS)
|
||||
return [{ ...passage }];
|
||||
const chunks: SpeechPassage[] = [];
|
||||
let boundary = passage.startSeconds;
|
||||
while (boundary < passage.endSeconds) {
|
||||
const target = boundary + SPEECH_PASSAGE_SECONDS;
|
||||
let end = Math.min(target, passage.endSeconds);
|
||||
if (target < passage.endSeconds) {
|
||||
// Starting in a long quiet lead-in can make Whisper place the next line
|
||||
// several seconds early. A nearby VAD start gives the next chunk an anchor.
|
||||
let nearestSpeechStart: number | undefined;
|
||||
for (const time of speechStarts) {
|
||||
if (
|
||||
time >= target - PAUSE_SEARCH_SECONDS &&
|
||||
time <= target + PAUSE_SEARCH_SECONDS &&
|
||||
time < passage.endSeconds &&
|
||||
(nearestSpeechStart === undefined ||
|
||||
Math.abs(time - target) < Math.abs(nearestSpeechStart - target))
|
||||
)
|
||||
nearestSpeechStart = time;
|
||||
}
|
||||
let latestPause: number | undefined;
|
||||
for (const time of pauses) {
|
||||
if (
|
||||
time >= target - PAUSE_SEARCH_SECONDS &&
|
||||
time <= target &&
|
||||
(latestPause === undefined || time > latestPause)
|
||||
)
|
||||
latestPause = time;
|
||||
}
|
||||
end = nearestSpeechStart ?? latestPause ?? end;
|
||||
}
|
||||
chunks.push({
|
||||
startSeconds: Math.max(passage.startSeconds, boundary - CHUNK_CONTEXT_SECONDS),
|
||||
endSeconds: Math.min(passage.endSeconds, end + CHUNK_CONTEXT_SECONDS),
|
||||
});
|
||||
boundary = end;
|
||||
}
|
||||
return chunks;
|
||||
});
|
||||
}
|
||||
|
||||
// Deduplicate only matching text substantially overlapping cues from earlier chunks.
|
||||
// Repeated words within the current chunk or at separate times remain separate.
|
||||
export function appendSpeechChunkCues(cues: SubtitleCue[], incoming: readonly SubtitleCue[]): void {
|
||||
const previousCount = cues.length;
|
||||
const matched = new Set<SubtitleCue>();
|
||||
for (const cue of incoming) {
|
||||
const text = cue.text.replace(/[\s\p{P}]+/gu, '');
|
||||
let duplicate: SubtitleCue | undefined;
|
||||
let greatestOverlap = 0;
|
||||
for (const [index, previous] of cues.entries()) {
|
||||
if (index >= previousCount) break;
|
||||
if (!text || matched.has(previous) || previous.text.replace(/[\s\p{P}]+/gu, '') !== text)
|
||||
continue;
|
||||
const overlap =
|
||||
Math.min(previous.endTime, cue.endTime) - Math.max(previous.startTime, cue.startTime);
|
||||
const shorterDuration = Math.min(
|
||||
previous.endTime - previous.startTime,
|
||||
cue.endTime - cue.startTime,
|
||||
);
|
||||
if (overlap > greatestOverlap && overlap >= shorterDuration / 2) {
|
||||
duplicate = previous;
|
||||
greatestOverlap = overlap;
|
||||
}
|
||||
}
|
||||
if (duplicate) {
|
||||
duplicate.startTime = Math.min(duplicate.startTime, cue.startTime);
|
||||
duplicate.endTime = Math.max(duplicate.endTime, cue.endTime);
|
||||
matched.add(duplicate);
|
||||
} else cues.push({ ...cue });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { findAudiblePassages, mergeSpeechPassages } from './subtitle-generation-coverage';
|
||||
|
||||
async function analyze(lines: string[], progress = 'out_time_us=20000000\n') {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'subtitle-coverage-test-'));
|
||||
try {
|
||||
const ffmpegPath = path.join(directory, 'ffmpeg');
|
||||
await writeFile(
|
||||
ffmpegPath,
|
||||
`#!${process.execPath}
|
||||
process.stderr.write(${JSON.stringify(lines.join('\n') + '\n')});
|
||||
process.stdout.write(${JSON.stringify(progress)});
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
return await findAudiblePassages({ ffmpegPath, wavPath: 'audio.wav' });
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test('audible coverage retains the full timeline when there is no confident silence', async () => {
|
||||
assert.deepEqual(await analyze([]), [{ startSeconds: 0, endSeconds: 20 }]);
|
||||
});
|
||||
|
||||
test('audible coverage omits silence while padding nearby audio without exceeding the timeline', async () => {
|
||||
assert.deepEqual(
|
||||
await analyze([
|
||||
'[silencedetect] silence_start: 0',
|
||||
'[silencedetect] silence_end: 2 | silence_duration: 2',
|
||||
'[silencedetect] silence_start: 8',
|
||||
'[silencedetect] silence_end: 12 | silence_duration: 4',
|
||||
'[silencedetect] silence_start: 18',
|
||||
]),
|
||||
[
|
||||
{ startSeconds: 1.65, endSeconds: 8.35 },
|
||||
{ startSeconds: 11.65, endSeconds: 18.35 },
|
||||
],
|
||||
);
|
||||
assert.deepEqual(await analyze(['[silencedetect] silence_end: 10.5 | silence_duration: 0.5']), [
|
||||
{ startSeconds: 0, endSeconds: 20 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('entirely silent audio has no audible passages', async () => {
|
||||
assert.deepEqual(await analyze(['[silencedetect] silence_start: 0']), []);
|
||||
assert.deepEqual(await analyze(['[silencedetect] silence_end: 20 | silence_duration: 20']), []);
|
||||
});
|
||||
|
||||
test('missing analysis duration fails instead of silently dropping audio', async () => {
|
||||
await assert.rejects(analyze([], ''), /valid duration/);
|
||||
});
|
||||
|
||||
test('merging coverage preserves quiet VAD speech and does not mutate detector results', () => {
|
||||
const speech = [{ startSeconds: 10, endSeconds: 11 }];
|
||||
assert.deepEqual(
|
||||
mergeSpeechPassages([
|
||||
...speech,
|
||||
{ startSeconds: 0, endSeconds: 5 },
|
||||
{ startSeconds: 4, endSeconds: 8 },
|
||||
{ startSeconds: 11, endSeconds: 12 },
|
||||
]),
|
||||
[
|
||||
{ startSeconds: 0, endSeconds: 8 },
|
||||
{ startSeconds: 10, endSeconds: 12 },
|
||||
],
|
||||
);
|
||||
assert.deepEqual(speech, [{ startSeconds: 10, endSeconds: 11 }]);
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
|
||||
import type { SpeechPassage } from './subtitle-generation-speech';
|
||||
|
||||
const AUDIO_PADDING_SECONDS = 0.35;
|
||||
|
||||
export function mergeSpeechPassages(passages: readonly SpeechPassage[]): SpeechPassage[] {
|
||||
const merged: SpeechPassage[] = [];
|
||||
for (const passage of [...passages].sort((a, b) => a.startSeconds - b.startSeconds)) {
|
||||
const previous = merged.at(-1);
|
||||
if (previous && passage.startSeconds <= previous.endSeconds)
|
||||
previous.endSeconds = Math.max(previous.endSeconds, passage.endSeconds);
|
||||
else merged.push({ ...passage });
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
// VAD rejection is not proof of silence. Preserve audible gaps for Whisper to evaluate.
|
||||
export async function findAudiblePassages(input: {
|
||||
ffmpegPath: string;
|
||||
wavPath: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<SpeechPassage[]> {
|
||||
const silences: SpeechPassage[] = [];
|
||||
let duration = 0;
|
||||
let trailingSilence: number | undefined;
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.ffmpegPath,
|
||||
args: [
|
||||
'-nostdin',
|
||||
'-hide_banner',
|
||||
'-nostats',
|
||||
'-i',
|
||||
input.wavPath,
|
||||
'-af',
|
||||
'silencedetect=noise=-50dB:d=0.5',
|
||||
'-progress',
|
||||
'pipe:1',
|
||||
'-f',
|
||||
'null',
|
||||
'-',
|
||||
],
|
||||
signal: input.signal,
|
||||
onLine: (line) => {
|
||||
const progress = /^out_time_us=(\d+)$/.exec(line);
|
||||
if (progress) duration = Math.max(duration, Number(progress[1]) / 1_000_000);
|
||||
const start = /silence_start: (\S+)/.exec(line);
|
||||
if (start && Number.isFinite(Number(start[1]))) trailingSilence = Number(start[1]);
|
||||
const end = /silence_end: (\S+) \| silence_duration: (\S+)/.exec(line);
|
||||
if (!end) return;
|
||||
const endSeconds = Number(end[1]);
|
||||
const length = Number(end[2]);
|
||||
if (Number.isFinite(endSeconds) && Number.isFinite(length) && length > 0) {
|
||||
silences.push({ startSeconds: Math.max(0, endSeconds - length), endSeconds });
|
||||
trailingSilence = undefined;
|
||||
}
|
||||
},
|
||||
});
|
||||
if (!Number.isFinite(duration) || duration <= 0)
|
||||
throw new Error('Audio analysis did not report a valid duration.');
|
||||
if (trailingSilence !== undefined)
|
||||
silences.push({ startSeconds: trailingSilence, endSeconds: duration });
|
||||
|
||||
const audible: SpeechPassage[] = [];
|
||||
let cursor = 0;
|
||||
for (const silence of mergeSpeechPassages(silences)) {
|
||||
if (cursor >= duration) break;
|
||||
if (silence.startSeconds > cursor)
|
||||
audible.push({ startSeconds: cursor, endSeconds: Math.min(duration, silence.startSeconds) });
|
||||
cursor = Math.max(cursor, silence.endSeconds);
|
||||
}
|
||||
if (cursor < duration) audible.push({ startSeconds: cursor, endSeconds: duration });
|
||||
return mergeSpeechPassages(
|
||||
audible.map((passage) => ({
|
||||
startSeconds: Math.max(0, passage.startSeconds - AUDIO_PADDING_SECONDS),
|
||||
endSeconds: Math.min(duration, passage.endSeconds + AUDIO_PADDING_SECONDS),
|
||||
})),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import { access, readFile, rm } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type {
|
||||
SubtitleGenerationConfig,
|
||||
SubtitleGenerationProgress,
|
||||
} from '../../shared/subtitle-generation';
|
||||
import { expandSubtitleGenerationPath } from './subtitle-generation-files';
|
||||
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
|
||||
import type { SubtitleGenerationToolPaths } from './subtitle-generation-tools';
|
||||
import { formatTimestamp } from './subtitle-generation-srt';
|
||||
import {
|
||||
parseSpeechPassages,
|
||||
speechPassageCues,
|
||||
SPEECH_PASSAGE_SECONDS,
|
||||
} from './subtitle-generation-speech';
|
||||
import type { SubtitleCue } from './subtitle-cue-parser';
|
||||
import { appendSpeechChunkCues, splitSpeechPassages } from './subtitle-generation-chunks';
|
||||
import { findSpeechPauses } from './subtitle-generation-pauses';
|
||||
import { findAudiblePassages, mergeSpeechPassages } from './subtitle-generation-coverage';
|
||||
|
||||
export async function transcribeSubtitleDialogue(input: {
|
||||
config: SubtitleGenerationConfig;
|
||||
tools: SubtitleGenerationToolPaths & { vad: string };
|
||||
modelPath: string;
|
||||
wavPath: string;
|
||||
directory: string;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
const vadModelPath = expandSubtitleGenerationPath(input.config.vadModelPath);
|
||||
await access(vadModelPath, constants.R_OK);
|
||||
input.onProgress?.({ stage: 'transcribe', percent: 0, message: 'Finding spoken dialogue...' });
|
||||
const segmentLines: string[] = [];
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.tools.vad,
|
||||
args: [
|
||||
'-f',
|
||||
input.wavPath,
|
||||
'-vm',
|
||||
vadModelPath,
|
||||
'-t',
|
||||
String(input.config.threads),
|
||||
'-vt',
|
||||
'0.3',
|
||||
'--vad-min-speech-duration-ms',
|
||||
'100',
|
||||
'--vad-min-silence-duration-ms',
|
||||
'500',
|
||||
'-vp',
|
||||
'350',
|
||||
'-vmsd',
|
||||
String(SPEECH_PASSAGE_SECONDS),
|
||||
'-np',
|
||||
],
|
||||
signal: input.signal,
|
||||
// Capture structured result lines separately from the bounded process log.
|
||||
onLine: (line) => {
|
||||
if (line.startsWith('Detected ') || line.startsWith('Speech segment '))
|
||||
segmentLines.push(line);
|
||||
},
|
||||
});
|
||||
const speech = parseSpeechPassages(segmentLines.join('\n'));
|
||||
input.onProgress?.({ stage: 'transcribe', percent: 0, message: 'Checking audio coverage...' });
|
||||
const audible = await findAudiblePassages({
|
||||
ffmpegPath: input.tools.ffmpeg,
|
||||
wavPath: input.wavPath,
|
||||
signal: input.signal,
|
||||
});
|
||||
const detected = mergeSpeechPassages([...speech, ...audible]);
|
||||
if (detected.length === 0) throw new Error('No spoken dialogue detected.');
|
||||
const pauses = detected.some(
|
||||
(passage) => passage.endSeconds - passage.startSeconds > SPEECH_PASSAGE_SECONDS,
|
||||
)
|
||||
? await findSpeechPauses({
|
||||
ffmpegPath: input.tools.ffmpeg,
|
||||
wavPath: input.wavPath,
|
||||
signal: input.signal,
|
||||
})
|
||||
: [];
|
||||
const passages = splitSpeechPassages(
|
||||
detected,
|
||||
pauses,
|
||||
speech.map((passage) => passage.startSeconds),
|
||||
);
|
||||
const cues: SubtitleCue[] = [];
|
||||
for (const [index, passage] of passages.entries()) {
|
||||
const base = path.join(input.directory, `speech-${index}`);
|
||||
input.onProgress?.({
|
||||
stage: 'transcribe',
|
||||
percent: Math.floor((index / passages.length) * 100),
|
||||
message: `Transcribing dialogue passage ${index + 1} of ${passages.length}...`,
|
||||
});
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.tools.ffmpeg,
|
||||
args: [
|
||||
'-nostdin',
|
||||
'-hide_banner',
|
||||
'-loglevel',
|
||||
'error',
|
||||
'-ss',
|
||||
String(passage.startSeconds),
|
||||
'-i',
|
||||
input.wavPath,
|
||||
'-t',
|
||||
String(passage.endSeconds - passage.startSeconds),
|
||||
'-ac',
|
||||
'1',
|
||||
'-ar',
|
||||
'16000',
|
||||
'-c:a',
|
||||
'pcm_s16le',
|
||||
`${base}.wav`,
|
||||
],
|
||||
signal: input.signal,
|
||||
});
|
||||
// -mc 0 limits text context, but does not isolate decoder state across input files.
|
||||
// A fresh process prevents earlier passages from corrupting later transcriptions.
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.tools.whisper,
|
||||
args: [
|
||||
'-m',
|
||||
input.modelPath,
|
||||
'-l',
|
||||
'ja',
|
||||
'-t',
|
||||
String(input.config.threads),
|
||||
'-mc',
|
||||
'0',
|
||||
'-sns',
|
||||
'-osrt',
|
||||
'-f',
|
||||
`${base}.wav`,
|
||||
'-of',
|
||||
base,
|
||||
],
|
||||
signal: input.signal,
|
||||
});
|
||||
input.signal?.throwIfAborted();
|
||||
appendSpeechChunkCues(cues, speechPassageCues(await readFile(`${base}.srt`, 'utf8'), passage));
|
||||
await rm(`${base}.wav`);
|
||||
}
|
||||
if (cues.length === 0) throw new Error('Whisper recognized no dialogue in the detected speech.');
|
||||
return cues
|
||||
.sort((a, b) => a.startTime - b.startTime || a.endTime - b.endTime)
|
||||
.map(
|
||||
(cue, index) =>
|
||||
`${index + 1}\n${formatTimestamp(cue.startTime * 1000)} --> ${formatTimestamp(cue.endTime * 1000)}\n${cue.text}\n`,
|
||||
)
|
||||
.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { downloadSubtitleGenerationArtifact } from './subtitle-generation-download';
|
||||
import {
|
||||
downloadSubtitleGenerationVadModel,
|
||||
resolveSubtitleGenerationVadModel,
|
||||
} from './subtitle-generation-vad-model';
|
||||
import { DEFAULT_SUBTITLE_GENERATION_CONFIG } from '../../shared/subtitle-generation';
|
||||
|
||||
test('verified model publication preserves existing files and cleans up failed or cancelled downloads', async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'subtitle-model-download-'));
|
||||
const originalFetch = globalThis.fetch;
|
||||
const bytes = new TextEncoder().encode('fixture model');
|
||||
const input = {
|
||||
url: 'https://example.test/model',
|
||||
size: bytes.length,
|
||||
sha256: createHash('sha256').update(bytes).digest('hex'),
|
||||
destination: path.join(directory, 'model.bin'),
|
||||
label: 'test model',
|
||||
};
|
||||
try {
|
||||
globalThis.fetch = Object.assign(async () => new Response(bytes), originalFetch);
|
||||
await downloadSubtitleGenerationArtifact(input);
|
||||
assert.equal(await readFile(input.destination, 'utf8'), 'fixture model');
|
||||
await assert.rejects(downloadSubtitleGenerationArtifact(input), /EEXIST/);
|
||||
await assert.rejects(
|
||||
downloadSubtitleGenerationArtifact({
|
||||
...input,
|
||||
destination: path.join(directory, 'bad.bin'),
|
||||
sha256: 'wrong',
|
||||
}),
|
||||
/integrity/,
|
||||
);
|
||||
const controller = new AbortController();
|
||||
await assert.rejects(
|
||||
downloadSubtitleGenerationArtifact({
|
||||
...input,
|
||||
destination: path.join(directory, 'cancelled.bin'),
|
||||
signal: controller.signal,
|
||||
onProgress: ({ percent }) => {
|
||||
if (percent === 99) controller.abort();
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.deepEqual(await readdir(directory), ['model.bin']);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('VAD setup recognizes existing paths and never replaces an invalid external model', async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'subtitle-vad-model-'));
|
||||
try {
|
||||
const config = { ...DEFAULT_SUBTITLE_GENERATION_CONFIG };
|
||||
assert.equal((await resolveSubtitleGenerationVadModel(config, directory)).kind, 'missing');
|
||||
config.vadModelPath = path.join(directory, 'external.bin');
|
||||
await assert.rejects(
|
||||
downloadSubtitleGenerationVadModel({ config, modelDirectory: directory }),
|
||||
/Cannot read/,
|
||||
);
|
||||
await writeFile(config.vadModelPath, 'external model');
|
||||
assert.equal((await resolveSubtitleGenerationVadModel(config, directory)).kind, 'external');
|
||||
assert.equal(
|
||||
await downloadSubtitleGenerationVadModel({ config, modelDirectory: directory }),
|
||||
config.vadModelPath,
|
||||
);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdir, mkdtemp, open, rm } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { publishSubtitleGenerationFile } from './subtitle-generation-files';
|
||||
import type { SubtitleGenerationProgress } from '../../shared/subtitle-generation';
|
||||
|
||||
export async function downloadSubtitleGenerationArtifact(input: {
|
||||
url: string;
|
||||
size: number;
|
||||
sha256: string;
|
||||
destination: string;
|
||||
label: string;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
input.signal?.throwIfAborted();
|
||||
await mkdir(path.dirname(input.destination), { recursive: true });
|
||||
const temporaryDirectory = await mkdtemp(
|
||||
path.join(path.dirname(input.destination), '.download-'),
|
||||
);
|
||||
const temporaryPath = path.join(temporaryDirectory, 'model.bin');
|
||||
input.onProgress?.({ stage: 'download', percent: 0, message: `Downloading ${input.label}...` });
|
||||
try {
|
||||
const response = await fetch(input.url, { signal: input.signal });
|
||||
if (!response.ok || !response.body) {
|
||||
throw new Error(`Model download failed: HTTP ${response.status}`);
|
||||
}
|
||||
const file = await open(temporaryPath, 'wx');
|
||||
const reader = response.body.getReader();
|
||||
const digest = createHash('sha256');
|
||||
let received = 0;
|
||||
let previousPercent = -1;
|
||||
try {
|
||||
while (true) {
|
||||
input.signal?.throwIfAborted();
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
received += value.byteLength;
|
||||
if (received > input.size) throw new Error('Downloaded model exceeds expected size.');
|
||||
digest.update(value);
|
||||
await file.writeFile(value);
|
||||
const percent = Math.min(99, Math.floor((received / input.size) * 100));
|
||||
if (percent !== previousPercent) {
|
||||
previousPercent = percent;
|
||||
input.onProgress?.({
|
||||
stage: 'download',
|
||||
percent,
|
||||
message: `Downloading ${input.label}...`,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (received !== input.size || digest.digest('hex') !== input.sha256) {
|
||||
throw new Error('Downloaded model failed integrity verification. Try downloading again.');
|
||||
}
|
||||
await file.sync();
|
||||
} finally {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
await file.close();
|
||||
}
|
||||
input.signal?.throwIfAborted();
|
||||
await publishSubtitleGenerationFile(temporaryPath, input.destination);
|
||||
input.onProgress?.({ stage: 'download', percent: 100, message: `${input.label} is ready.` });
|
||||
return input.destination;
|
||||
} finally {
|
||||
await rm(temporaryDirectory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { constants } from 'node:fs';
|
||||
import { copyFile, link } from 'node:fs/promises';
|
||||
import { homedir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
export function expandSubtitleGenerationPath(value: string): string {
|
||||
if (value === '~') return homedir();
|
||||
if (value.startsWith('~/') || value.startsWith('~\\'))
|
||||
return path.join(homedir(), value.slice(2));
|
||||
return value;
|
||||
}
|
||||
|
||||
// Prefer atomic publication. Filesystems without hard links still get exclusive creation.
|
||||
export async function publishSubtitleGenerationFile(
|
||||
source: string,
|
||||
destination: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await link(source, destination);
|
||||
} catch (error) {
|
||||
if (
|
||||
!(error instanceof Error) ||
|
||||
!('code' in error) ||
|
||||
(error.code !== 'ENOTSUP' &&
|
||||
error.code !== 'EOPNOTSUPP' &&
|
||||
error.code !== 'EPERM' &&
|
||||
error.code !== 'EXDEV')
|
||||
)
|
||||
throw error;
|
||||
await copyFile(source, destination, constants.COPYFILE_EXCL);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { access, open, stat } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { getSubtitleGenerationModel } from '../../shared/subtitle-generation-model-catalog';
|
||||
import { expandSubtitleGenerationPath } from './subtitle-generation-files';
|
||||
import { downloadSubtitleGenerationArtifact } from './subtitle-generation-download';
|
||||
import type {
|
||||
SubtitleGenerationConfig,
|
||||
SubtitleGenerationModelStatus,
|
||||
SubtitleGenerationProgress,
|
||||
} from '../../shared/subtitle-generation';
|
||||
|
||||
export function isMissingFile(error: unknown): boolean {
|
||||
return error instanceof Error && 'code' in error && error.code === 'ENOENT';
|
||||
}
|
||||
|
||||
async function modelCompatibilityError(modelPath: string): Promise<string | undefined> {
|
||||
const file = await open(modelPath, 'r');
|
||||
try {
|
||||
// whisper_model_load reads GGML magic, then n_vocab. is_multilingual uses n_vocab >= 51865.
|
||||
const header = Buffer.alloc(8);
|
||||
const { bytesRead } = await file.read(header, 0, header.length, 0);
|
||||
if (
|
||||
bytesRead !== header.length ||
|
||||
header.readUInt32LE(0) !== 0x67676d6c ||
|
||||
header.readInt32LE(4) <= 0
|
||||
) {
|
||||
return 'Unsupported model format. Choose a whisper.cpp GGML .bin model.';
|
||||
}
|
||||
if (header.readInt32LE(4) < 51865) {
|
||||
return 'This Whisper model is English-only. Japanese subtitle generation requires a multilingual model.';
|
||||
}
|
||||
return undefined;
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveSubtitleGenerationModel(
|
||||
config: SubtitleGenerationConfig,
|
||||
modelDirectory: string,
|
||||
): Promise<SubtitleGenerationModelStatus> {
|
||||
const external = config.modelPath.trim();
|
||||
const modelPath = external
|
||||
? path.resolve(expandSubtitleGenerationPath(external))
|
||||
: path.resolve(modelDirectory, `ggml-${config.managedModel}.bin`);
|
||||
try {
|
||||
const info = await stat(modelPath);
|
||||
if (!info.isFile() || info.size === 0) {
|
||||
return { kind: 'invalid', path: modelPath, message: 'Model must be a nonempty file.' };
|
||||
}
|
||||
await access(modelPath, constants.R_OK);
|
||||
if (!external && info.size !== getSubtitleGenerationModel(config.managedModel).size) {
|
||||
return { kind: 'invalid', path: modelPath, message: 'Managed model has an unexpected size.' };
|
||||
}
|
||||
const compatibilityError = await modelCompatibilityError(modelPath);
|
||||
if (compatibilityError)
|
||||
return { kind: 'invalid', path: modelPath, message: compatibilityError };
|
||||
return { kind: external ? 'external' : 'managed', path: modelPath };
|
||||
} catch (error) {
|
||||
if (!external && isMissingFile(error)) return { kind: 'missing', path: modelPath };
|
||||
return {
|
||||
kind: 'invalid',
|
||||
path: modelPath,
|
||||
message: `Cannot read model: ${error instanceof Error ? error.message : String(error)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function downloadSubtitleGenerationModel(input: {
|
||||
config: SubtitleGenerationConfig;
|
||||
modelDirectory: string;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
input.signal?.throwIfAborted();
|
||||
const current = await resolveSubtitleGenerationModel(input.config, input.modelDirectory);
|
||||
if (current.kind === 'external' || current.kind === 'managed') return current.path;
|
||||
if (current.kind === 'invalid') throw new Error(current.message);
|
||||
const model = getSubtitleGenerationModel(input.config.managedModel);
|
||||
return downloadSubtitleGenerationArtifact({
|
||||
url: `https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-${input.config.managedModel}.bin`,
|
||||
size: model.size,
|
||||
sha256: model.sha256,
|
||||
destination: current.path,
|
||||
label: 'Whisper model',
|
||||
onProgress: input.onProgress,
|
||||
signal: input.signal,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
|
||||
|
||||
// Each completed silencedetect line contains both the end and duration of a quiet interval.
|
||||
export async function findSpeechPauses(input: {
|
||||
ffmpegPath: string;
|
||||
wavPath: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<number[]> {
|
||||
const pauses: number[] = [];
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.ffmpegPath,
|
||||
args: [
|
||||
'-nostdin',
|
||||
'-hide_banner',
|
||||
'-nostats',
|
||||
'-i',
|
||||
input.wavPath,
|
||||
'-af',
|
||||
'silencedetect=noise=-35dB:d=0.12',
|
||||
'-f',
|
||||
'null',
|
||||
'-',
|
||||
],
|
||||
signal: input.signal,
|
||||
onLine: (line) => {
|
||||
const match = /silence_end: (\S+) \| silence_duration: (\S+)/.exec(line);
|
||||
if (!match) return;
|
||||
const end = Number(match[1]);
|
||||
const duration = Number(match[2]);
|
||||
if (Number.isFinite(end) && Number.isFinite(duration) && duration > 0 && end >= duration)
|
||||
pauses.push(end - duration / 2);
|
||||
},
|
||||
});
|
||||
return pauses.sort((a, b) => a - b);
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { expandSubtitleGenerationPath } from './subtitle-generation-files';
|
||||
|
||||
const OUTPUT_LIMIT = 64 * 1024;
|
||||
|
||||
// Keep partial lines between chunks: ffmpeg and whisper both report progress on stderr.
|
||||
export function runSubtitleGenerationProcess(input: {
|
||||
command: string;
|
||||
args: string[];
|
||||
signal?: AbortSignal;
|
||||
onLine?: (line: string) => void;
|
||||
}): Promise<string> {
|
||||
input.signal?.throwIfAborted();
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(expandSubtitleGenerationPath(input.command), input.args, {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let killTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
const abort = () => {
|
||||
child.kill('SIGTERM');
|
||||
killTimer = setTimeout(() => child.kill('SIGKILL'), 2000);
|
||||
killTimer.unref();
|
||||
};
|
||||
input.signal?.addEventListener('abort', abort, { once: true });
|
||||
if (input.signal?.aborted) abort();
|
||||
const cleanup = () => {
|
||||
input.signal?.removeEventListener('abort', abort);
|
||||
clearTimeout(killTimer);
|
||||
};
|
||||
for (const [stream, isStdout] of [
|
||||
[child.stdout, true],
|
||||
[child.stderr, false],
|
||||
] as const) {
|
||||
let pending = '';
|
||||
stream.setEncoding('utf8');
|
||||
stream.on('data', (chunk: string) => {
|
||||
if (isStdout) stdout = (stdout + chunk).slice(-OUTPUT_LIMIT);
|
||||
else stderr = (stderr + chunk).slice(-OUTPUT_LIMIT);
|
||||
const lines = (pending + chunk).split(/[\r\n]/);
|
||||
pending = (lines.pop() ?? '').slice(-OUTPUT_LIMIT);
|
||||
for (const line of lines) input.onLine?.(line);
|
||||
});
|
||||
stream.on('end', () => {
|
||||
if (pending) input.onLine?.(pending);
|
||||
});
|
||||
}
|
||||
child.once('error', (error) => {
|
||||
cleanup();
|
||||
reject(
|
||||
new Error(
|
||||
'code' in error && error.code === 'ENOENT'
|
||||
? `${input.command} was not found. Install it or set its path under subtitleGeneration in Settings.`
|
||||
: `Could not run ${input.command}: ${error.message}`,
|
||||
),
|
||||
);
|
||||
});
|
||||
child.once('close', (code) => {
|
||||
cleanup();
|
||||
if (input.signal?.aborted) reject(new Error('Subtitle generation cancelled.'));
|
||||
else if (code !== 0) {
|
||||
reject(new Error(`${input.command} exited with status ${code}: ${stderr.trim()}`));
|
||||
} else resolve(stdout);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { parseSpeechPassages, speechPassageCues } from './subtitle-generation-speech';
|
||||
|
||||
test('speech passages convert centiseconds, group nearby speech, and retain long gaps', () => {
|
||||
assert.deepEqual(
|
||||
parseSpeechPassages(
|
||||
[
|
||||
'Detected 4 speech segments:',
|
||||
'Speech segment 0: start = 0.00, end = 300.00',
|
||||
'Speech segment 1: start = 350.00, end = 900.00',
|
||||
'Speech segment 2: start = 10000.00, end = 11900.00',
|
||||
'Speech segment 3: start = 11950.00, end = 12500.00',
|
||||
].join('\n'),
|
||||
),
|
||||
[
|
||||
{ startSeconds: 0, endSeconds: 9 },
|
||||
{ startSeconds: 100, endSeconds: 119 },
|
||||
{ startSeconds: 119.5, endSeconds: 125 },
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('speech passages retain long merged detector segments for pause-aware splitting', () => {
|
||||
assert.deepEqual(
|
||||
parseSpeechPassages(
|
||||
[
|
||||
'Detected 3 speech segments:',
|
||||
'Speech segment 0: start = 33714.00, end = 36756.00',
|
||||
'Speech segment 1: start = 40000.00, end = 46000.00',
|
||||
'Speech segment 2: start = 50000.00, end = 50100.00',
|
||||
].join('\n'),
|
||||
),
|
||||
[
|
||||
{ startSeconds: 337.14, endSeconds: 367.56 },
|
||||
{ startSeconds: 400, endSeconds: 460 },
|
||||
{ startSeconds: 500, endSeconds: 501 },
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('speech detector distinguishes no speech from missing, malformed, or truncated output', () => {
|
||||
assert.deepEqual(parseSpeechPassages('Detected 0 speech segments:'), []);
|
||||
assert.deepEqual(
|
||||
parseSpeechPassages(
|
||||
'Detected 1 speech segments:\nSpeech segment 0: start = 79896.00, end = 82218.00',
|
||||
),
|
||||
[{ startSeconds: 798.96, endSeconds: 822.18 }],
|
||||
);
|
||||
for (const output of [
|
||||
'',
|
||||
'Detected 1 speech segments:',
|
||||
'Detected 1 speech segments:\nSpeech segment 1: start = 100.00, end = 200.00',
|
||||
'Detected 1 speech segments:\nSpeech segment 0: start = 200.00, end = 100.00',
|
||||
'Detected 1 speech segments:\nSpeech segment 0: start = NaN, end = 100.00',
|
||||
'Detected 2 speech segments:\nSpeech segment 0: start = 0.00, end = 200.00\nSpeech segment 1: start = 100.00, end = 300.00',
|
||||
])
|
||||
assert.throws(() => parseSpeechPassages(output), /Speech detector/);
|
||||
});
|
||||
|
||||
test('passage cue times cannot extend into omitted audio or accumulate offsets', () => {
|
||||
const srt =
|
||||
'1\n00:00:00,000 --> 00:00:01,000\nはい\n\n2\n00:00:01,000 --> 00:01:40,000\nはい\n\n3\n00:01:41,000 --> 00:01:42,000\n幻覚\n';
|
||||
assert.deepEqual(
|
||||
speechPassageCues(srt, { startSeconds: 1200.25, endSeconds: 1203.75 }).map(
|
||||
({ startTime, endTime, text }) => ({ startTime, endTime, text }),
|
||||
),
|
||||
[
|
||||
{ startTime: 1200.25, endTime: 1201.25, text: 'はい' },
|
||||
{ startTime: 1201.25, endTime: 1203.75, text: 'はい' },
|
||||
],
|
||||
);
|
||||
assert.deepEqual(speechPassageCues('', { startSeconds: 0, endSeconds: 1 }), []);
|
||||
assert.throws(
|
||||
() => speechPassageCues('broken SRT', { startSeconds: 0, endSeconds: 1 }),
|
||||
/malformed/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
import { parseSrtCues, type SubtitleCue } from './subtitle-cue-parser';
|
||||
|
||||
export interface SpeechPassage {
|
||||
startSeconds: number;
|
||||
endSeconds: number;
|
||||
}
|
||||
|
||||
export const SPEECH_PASSAGE_SECONDS = 20;
|
||||
|
||||
// The standalone whisper.cpp detector reports centiseconds, unlike its diagnostic logs.
|
||||
export function parseSpeechPassages(output: string): SpeechPassage[] {
|
||||
const count = /^Detected (\d+) speech segments:$/m.exec(output);
|
||||
if (!count) throw new Error('Speech detector did not report its segment count.');
|
||||
const passages: SpeechPassage[] = [];
|
||||
for (const line of output.split(/\r?\n/)) {
|
||||
if (!line.startsWith('Speech segment ')) continue;
|
||||
const match = /^Speech segment (\d+): start = (\d+(?:\.\d+)?), end = (\d+(?:\.\d+)?)$/.exec(
|
||||
line,
|
||||
);
|
||||
if (!match) throw new Error('Speech detector returned a malformed segment.');
|
||||
const index = Number(match[1]);
|
||||
const startSeconds = Number(match[2]) / 100;
|
||||
const endSeconds = Number(match[3]) / 100;
|
||||
if (
|
||||
index !== passages.length ||
|
||||
!Number.isFinite(startSeconds) ||
|
||||
!Number.isFinite(endSeconds) ||
|
||||
endSeconds <= startSeconds ||
|
||||
startSeconds < (passages.at(-1)?.endSeconds ?? 0)
|
||||
) {
|
||||
throw new Error('Speech detector returned unordered or invalid segment timing.');
|
||||
}
|
||||
passages.push({ startSeconds, endSeconds });
|
||||
}
|
||||
if (passages.length !== Number(count[1]))
|
||||
throw new Error('Speech detector output is incomplete.');
|
||||
|
||||
const grouped: SpeechPassage[] = [];
|
||||
for (const passage of passages) {
|
||||
const previous = grouped.at(-1);
|
||||
if (
|
||||
previous &&
|
||||
passage.startSeconds - previous.endSeconds <= 1 &&
|
||||
passage.endSeconds - previous.startSeconds <= SPEECH_PASSAGE_SECONDS
|
||||
) {
|
||||
previous.endSeconds = passage.endSeconds;
|
||||
} else grouped.push({ ...passage });
|
||||
}
|
||||
return grouped;
|
||||
}
|
||||
|
||||
// Clamp to the audio actually supplied to Whisper. A cue cannot cross an omitted gap.
|
||||
export function speechPassageCues(srt: string, passage: SpeechPassage): SubtitleCue[] {
|
||||
const duration = passage.endSeconds - passage.startSeconds;
|
||||
const cues = parseSrtCues(srt);
|
||||
if (srt.trim() && cues.length === 0)
|
||||
throw new Error('Whisper returned malformed subtitles for a speech passage.');
|
||||
return cues.flatMap((cue) => {
|
||||
const start = Math.max(0, cue.startTime);
|
||||
const end = Math.min(duration, cue.endTime);
|
||||
if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start) return [];
|
||||
return [
|
||||
{ ...cue, startTime: passage.startSeconds + start, endTime: passage.startSeconds + end },
|
||||
];
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export function formatTimestamp(milliseconds: number): string {
|
||||
const rounded = Math.max(0, Math.round(milliseconds));
|
||||
const hours = Math.floor(rounded / 3600000);
|
||||
const minutes = Math.floor((rounded % 3600000) / 60000);
|
||||
const seconds = Math.floor((rounded % 60000) / 1000);
|
||||
return `${String(hours).padStart(2, '0')}:${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')},${String(rounded % 1000).padStart(3, '0')}`;
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { DEFAULT_SUBTITLE_GENERATION_CONFIG } from '../../shared/subtitle-generation';
|
||||
import {
|
||||
requireSubtitleGenerationTools,
|
||||
resolveSubtitleGenerationTools,
|
||||
} from './subtitle-generation-tools';
|
||||
|
||||
async function fixture(run: (directory: string) => Promise<void>) {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'subtitle-generation-tools-'));
|
||||
try {
|
||||
await run(directory);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function executable(directory: string, name: string): Promise<string> {
|
||||
const file = path.join(directory, name);
|
||||
await writeFile(file, '#!/bin/sh\n', { mode: 0o755 });
|
||||
return file;
|
||||
}
|
||||
|
||||
test('tools resolve from PATH, honor overrides, and only require the detector in dialogue mode', () =>
|
||||
fixture(async (directory) => {
|
||||
const bin = path.join(directory, 'bin');
|
||||
await mkdir(bin);
|
||||
for (const name of ['ffmpeg', 'ffprobe', 'whisper-cli']) await executable(bin, name);
|
||||
const detector = await executable(bin, 'vad-speech-segments');
|
||||
const customWhisper = await executable(directory, 'my-whisper');
|
||||
await writeFile(path.join(directory, 'not-executable'), '', { mode: 0o644 });
|
||||
const env = { PATH: bin };
|
||||
|
||||
const found = await resolveSubtitleGenerationTools(DEFAULT_SUBTITLE_GENERATION_CONFIG, env);
|
||||
assert.deepEqual(found, {
|
||||
ffmpeg: { kind: 'found', path: path.join(bin, 'ffmpeg') },
|
||||
ffprobe: { kind: 'found', path: path.join(bin, 'ffprobe') },
|
||||
whisper: { kind: 'found', path: path.join(bin, 'whisper-cli') },
|
||||
vad: null,
|
||||
});
|
||||
assert.equal(requireSubtitleGenerationTools(found).vad, null);
|
||||
|
||||
const dialogue = await resolveSubtitleGenerationTools(
|
||||
{ ...DEFAULT_SUBTITLE_GENERATION_CONFIG, vadModelPath: '/models/vad.bin' },
|
||||
env,
|
||||
);
|
||||
assert.deepEqual(dialogue.vad, { kind: 'found', path: detector });
|
||||
|
||||
const overridden = await resolveSubtitleGenerationTools(
|
||||
{
|
||||
...DEFAULT_SUBTITLE_GENERATION_CONFIG,
|
||||
whisperPath: customWhisper,
|
||||
ffmpegPath: path.join(directory, 'not-executable'),
|
||||
},
|
||||
env,
|
||||
);
|
||||
assert.deepEqual(overridden.whisper, { kind: 'found', path: customWhisper });
|
||||
assert.equal(overridden.ffmpeg.kind, 'missing');
|
||||
assert.throws(
|
||||
() => requireSubtitleGenerationTools(overridden),
|
||||
/not-executable \(subtitleGeneration\.ffmpegPath\) is not an executable file/,
|
||||
);
|
||||
}));
|
||||
|
||||
test('missing tools name the executable, the installer, and the setting', () =>
|
||||
fixture(async (directory) => {
|
||||
const tools = await resolveSubtitleGenerationTools(
|
||||
{ ...DEFAULT_SUBTITLE_GENERATION_CONFIG, vadModelPath: '/models/vad.bin' },
|
||||
{ PATH: directory },
|
||||
);
|
||||
assert.deepEqual(tools.whisper, {
|
||||
kind: 'missing',
|
||||
message:
|
||||
'whisper-cli was not found on PATH. Install whisper.cpp or set subtitleGeneration.whisperPath in Settings.',
|
||||
});
|
||||
assert.deepEqual(tools.vad, {
|
||||
kind: 'missing',
|
||||
message:
|
||||
"whisper-vad-speech-segments was not found on PATH. Install whisper.cpp's speech segment detector or set subtitleGeneration.vadPath in Settings.",
|
||||
});
|
||||
assert.throws(() => requireSubtitleGenerationTools(tools), /ffmpeg was not found on PATH/);
|
||||
}));
|
||||
@@ -0,0 +1,125 @@
|
||||
import { access, stat } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type {
|
||||
SubtitleGenerationConfig,
|
||||
SubtitleGenerationToolStatus,
|
||||
SubtitleGenerationTools,
|
||||
} from '../../shared/subtitle-generation';
|
||||
import { expandSubtitleGenerationPath } from './subtitle-generation-files';
|
||||
|
||||
/** Executable paths ready to spawn. `vad` is null when dialogue mode is off. */
|
||||
export interface SubtitleGenerationToolPaths {
|
||||
ffmpeg: string;
|
||||
ffprobe: string;
|
||||
whisper: string;
|
||||
vad: string | null;
|
||||
}
|
||||
|
||||
const TOOL_LOOKUPS = {
|
||||
ffmpeg: { setting: 'ffmpegPath', names: ['ffmpeg'], install: 'Install FFmpeg' },
|
||||
ffprobe: { setting: 'ffprobePath', names: ['ffprobe'], install: 'Install FFmpeg' },
|
||||
whisper: { setting: 'whisperPath', names: ['whisper-cli'], install: 'Install whisper.cpp' },
|
||||
vad: {
|
||||
setting: 'vadPath',
|
||||
names: ['whisper-vad-speech-segments', 'vad-speech-segments'],
|
||||
install: "Install whisper.cpp's speech segment detector",
|
||||
},
|
||||
} as const;
|
||||
|
||||
async function isExecutableFile(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
if (!(await stat(filePath)).isFile()) return false;
|
||||
await access(filePath, constants.X_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function executableNames(name: string, env: NodeJS.ProcessEnv): string[] {
|
||||
if (process.platform !== 'win32' || path.extname(name)) return [name];
|
||||
const extensions = (env.PATHEXT ?? '.EXE;.CMD;.BAT')
|
||||
.split(';')
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
return [name, ...extensions.map((extension) => `${name}${extension}`)];
|
||||
}
|
||||
|
||||
async function findOnPath(names: readonly string[], env: NodeJS.ProcessEnv): Promise<string> {
|
||||
const directories = (env.PATH ?? '')
|
||||
.split(path.delimiter)
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
for (const directory of directories) {
|
||||
for (const name of names) {
|
||||
for (const candidate of executableNames(name, env)) {
|
||||
const filePath = path.join(directory, candidate);
|
||||
if (await isExecutableFile(filePath)) return filePath;
|
||||
}
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
async function resolveTool(
|
||||
tool: keyof typeof TOOL_LOOKUPS,
|
||||
config: SubtitleGenerationConfig,
|
||||
env: NodeJS.ProcessEnv,
|
||||
): Promise<SubtitleGenerationToolStatus> {
|
||||
const lookup = TOOL_LOOKUPS[tool];
|
||||
const override = config[lookup.setting].trim();
|
||||
if (override) {
|
||||
const expanded = expandSubtitleGenerationPath(override);
|
||||
const found =
|
||||
path.dirname(expanded) === '.'
|
||||
? await findOnPath([expanded], env)
|
||||
: (await isExecutableFile(expanded))
|
||||
? path.resolve(expanded)
|
||||
: '';
|
||||
return found
|
||||
? { kind: 'found', path: found }
|
||||
: {
|
||||
kind: 'missing',
|
||||
message: `${override} (subtitleGeneration.${lookup.setting}) is not an executable file.`,
|
||||
};
|
||||
}
|
||||
const found = await findOnPath(lookup.names, env);
|
||||
return found
|
||||
? { kind: 'found', path: found }
|
||||
: {
|
||||
kind: 'missing',
|
||||
message: `${lookup.names[0]} was not found on PATH. ${lookup.install} or set subtitleGeneration.${lookup.setting} in Settings.`,
|
||||
};
|
||||
}
|
||||
|
||||
/** Locate every executable a generation run needs, before any model download or audio work. */
|
||||
export async function resolveSubtitleGenerationTools(
|
||||
config: SubtitleGenerationConfig,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<SubtitleGenerationTools> {
|
||||
const [ffmpeg, ffprobe, whisper, vad] = await Promise.all([
|
||||
resolveTool('ffmpeg', config, env),
|
||||
resolveTool('ffprobe', config, env),
|
||||
resolveTool('whisper', config, env),
|
||||
config.vadModelPath.trim() ? resolveTool('vad', config, env) : null,
|
||||
]);
|
||||
return { ffmpeg, ffprobe, whisper, vad };
|
||||
}
|
||||
|
||||
function foundPath(tool: SubtitleGenerationToolStatus): string {
|
||||
if (tool.kind === 'missing') throw new Error(tool.message);
|
||||
return tool.path;
|
||||
}
|
||||
|
||||
/** Throw the first missing tool's message, otherwise narrow to spawnable paths. */
|
||||
export function requireSubtitleGenerationTools(
|
||||
tools: SubtitleGenerationTools,
|
||||
): SubtitleGenerationToolPaths {
|
||||
return {
|
||||
ffmpeg: foundPath(tools.ffmpeg),
|
||||
ffprobe: foundPath(tools.ffprobe),
|
||||
whisper: foundPath(tools.whisper),
|
||||
vad: tools.vad ? foundPath(tools.vad) : null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { access, stat } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type {
|
||||
SubtitleGenerationConfig,
|
||||
SubtitleGenerationModelStatus,
|
||||
SubtitleGenerationProgress,
|
||||
} from '../../shared/subtitle-generation';
|
||||
import { SUBTITLE_GENERATION_VAD_MODEL } from '../../shared/subtitle-generation-vad-model';
|
||||
import { expandSubtitleGenerationPath } from './subtitle-generation-files';
|
||||
import { isMissingFile } from './subtitle-generation-models';
|
||||
import { downloadSubtitleGenerationArtifact } from './subtitle-generation-download';
|
||||
|
||||
export async function resolveSubtitleGenerationVadModel(
|
||||
config: SubtitleGenerationConfig,
|
||||
modelDirectory: string,
|
||||
): Promise<SubtitleGenerationModelStatus> {
|
||||
const external = config.vadModelPath.trim();
|
||||
const modelPath = external
|
||||
? path.resolve(expandSubtitleGenerationPath(external))
|
||||
: path.resolve(modelDirectory, SUBTITLE_GENERATION_VAD_MODEL.filename);
|
||||
try {
|
||||
const info = await stat(modelPath);
|
||||
if (
|
||||
!info.isFile() ||
|
||||
info.size === 0 ||
|
||||
(!external && info.size !== SUBTITLE_GENERATION_VAD_MODEL.size)
|
||||
)
|
||||
return {
|
||||
kind: 'invalid',
|
||||
path: modelPath,
|
||||
message: 'Speech detection model has an invalid size.',
|
||||
};
|
||||
await access(modelPath, constants.R_OK);
|
||||
return { kind: external ? 'external' : 'managed', path: modelPath };
|
||||
} catch (error) {
|
||||
if (!external && isMissingFile(error)) return { kind: 'missing', path: modelPath };
|
||||
return {
|
||||
kind: 'invalid',
|
||||
path: modelPath,
|
||||
message: `Cannot read speech detection model: ${error instanceof Error ? error.message : String(error)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function downloadSubtitleGenerationVadModel(input: {
|
||||
config: SubtitleGenerationConfig;
|
||||
modelDirectory: string;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
input.signal?.throwIfAborted();
|
||||
const current = await resolveSubtitleGenerationVadModel(input.config, input.modelDirectory);
|
||||
if (current.kind === 'invalid') throw new Error(current.message);
|
||||
if (current.kind !== 'missing') return current.path;
|
||||
return downloadSubtitleGenerationArtifact({
|
||||
...SUBTITLE_GENERATION_VAD_MODEL,
|
||||
destination: current.path,
|
||||
label: 'Silero speech detection model',
|
||||
onProgress: input.onProgress,
|
||||
signal: input.signal,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { constants } from 'node:fs';
|
||||
import { access, chmod, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
DEFAULT_SUBTITLE_GENERATION_CONFIG,
|
||||
resolveSubtitleGenerationConfig,
|
||||
type SubtitleGenerationProgress,
|
||||
} from '../../shared/subtitle-generation';
|
||||
import {
|
||||
downloadSubtitleGenerationModel,
|
||||
ensureWritableDirectory,
|
||||
generateJapaneseSubtitles,
|
||||
resolveSubtitleGenerationModel,
|
||||
} from './subtitle-generation';
|
||||
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
|
||||
|
||||
async function fixture(run: (directory: string) => Promise<void>) {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'subtitle-generation-test-'));
|
||||
try {
|
||||
await run(directory);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function executable(directory: string, name: string, body: string) {
|
||||
const file = path.join(directory, name);
|
||||
await writeFile(file, `#!${process.execPath}\n${body}`, { mode: 0o755 });
|
||||
return file;
|
||||
}
|
||||
|
||||
function modelHeader(vocabularySize = 51865): Buffer {
|
||||
const header = Buffer.alloc(8);
|
||||
header.writeUInt32LE(0x67676d6c, 0);
|
||||
header.writeInt32LE(vocabularySize, 4);
|
||||
return header;
|
||||
}
|
||||
|
||||
async function generationFixture(directory: string) {
|
||||
const modelPath = path.join(directory, 'external.bin');
|
||||
const mediaPath = path.join(directory, 'episode.mkv');
|
||||
const callsPath = path.join(directory, 'calls.jsonl');
|
||||
await writeFile(modelPath, modelHeader());
|
||||
await writeFile(mediaPath, 'local media');
|
||||
const record = `require('node:fs').appendFileSync(${JSON.stringify(callsPath)}, JSON.stringify(process.argv.slice(2)) + '\\n');`;
|
||||
const ffprobePath = await executable(
|
||||
directory,
|
||||
'ffprobe',
|
||||
`${record}\nprocess.stdout.write(JSON.stringify({streams: [{index:1,codec_type:'audio',start_time:'10',tags:{language:'eng'}},{index:3,codec_type:'audio',start_time:'12.5',duration:'20',tags:{language:'jpn'}}],format:{start_time:'10',duration:'25'}}));`,
|
||||
);
|
||||
const ffmpegPath = await executable(
|
||||
directory,
|
||||
'ffmpeg',
|
||||
`${record}
|
||||
if (process.argv.at(-1) === '-') {
|
||||
process.stderr.write('[silencedetect] silence_end: 25 | silence_duration: 25\\n');
|
||||
process.stdout.write('out_time_us=25000000\\nprogress=end\\n');
|
||||
} else {
|
||||
require('node:fs').writeFileSync(process.argv.at(-1), 'wav');
|
||||
process.stdout.write('out_time_');
|
||||
setTimeout(() => process.stdout.write('us=10000000\\nprogress=end\\n'), 10);
|
||||
}`,
|
||||
);
|
||||
const whisperPath = await executable(
|
||||
directory,
|
||||
'whisper-cli',
|
||||
`${record}\nconst args=process.argv.slice(2); require('node:fs').writeFileSync(args[args.indexOf('-of')+1]+'.srt', '1\\n00:00:01,000 --> 00:00:02,000\\nこんにちは\\n'); process.stderr.write('whisper_print_progress_callback: progress = '); setTimeout(() => process.stderr.write('55%\\n'), 10);`,
|
||||
);
|
||||
return {
|
||||
config: {
|
||||
...DEFAULT_SUBTITLE_GENERATION_CONFIG,
|
||||
modelPath,
|
||||
ffprobePath,
|
||||
ffmpegPath,
|
||||
whisperPath,
|
||||
},
|
||||
mediaPath,
|
||||
modelDirectory: path.join(directory, 'models'),
|
||||
callsPath,
|
||||
};
|
||||
}
|
||||
|
||||
test('config parser accepts supported models and rejects unsafe threads and wrong field types', () => {
|
||||
const warnings: string[] = [];
|
||||
const result = resolveSubtitleGenerationConfig(
|
||||
{ modelPath: '/tmp/whisper.bin', threads: 0, whisperPath: 42, managedModel: 'large-v3-turbo' },
|
||||
(key) => warnings.push(key),
|
||||
);
|
||||
assert.equal(result.modelPath, '/tmp/whisper.bin');
|
||||
assert.equal(result.managedModel, 'large-v3-turbo');
|
||||
assert.equal(result.threads, DEFAULT_SUBTITLE_GENERATION_CONFIG.threads);
|
||||
assert.deepEqual(warnings, ['whisperPath', 'threads']);
|
||||
});
|
||||
|
||||
test('external model path wins and invalid external models never fall back to download', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
assert.deepEqual(await resolveSubtitleGenerationModel(input.config, input.modelDirectory), {
|
||||
kind: 'external',
|
||||
path: input.config.modelPath,
|
||||
});
|
||||
const invalid = { ...input.config, modelPath: path.join(directory, 'missing.bin') };
|
||||
assert.equal(
|
||||
(await resolveSubtitleGenerationModel(invalid, input.modelDirectory)).kind,
|
||||
'invalid',
|
||||
);
|
||||
await assert.rejects(
|
||||
downloadSubtitleGenerationModel({ ...input, config: invalid }),
|
||||
/Cannot read model/,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await resolveSubtitleGenerationModel(
|
||||
{ ...input.config, modelPath: '' },
|
||||
input.modelDirectory,
|
||||
)
|
||||
).kind,
|
||||
'missing',
|
||||
);
|
||||
}));
|
||||
|
||||
test('English-only and incompatible external models are rejected before transcription', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
await writeFile(input.config.modelPath, modelHeader(51864));
|
||||
const englishOnly = await resolveSubtitleGenerationModel(input.config, input.modelDirectory);
|
||||
assert.equal(englishOnly.kind, 'invalid');
|
||||
assert.ok('message' in englishOnly);
|
||||
assert.match(englishOnly.message, /English-only/);
|
||||
await assert.rejects(generateJapaneseSubtitles(input), /requires a multilingual model/);
|
||||
await writeFile(input.config.modelPath, 'not a GGML model');
|
||||
await assert.rejects(generateJapaneseSubtitles(input), /Unsupported model format/);
|
||||
await writeFile(input.config.modelPath, modelHeader().subarray(0, 4));
|
||||
await assert.rejects(generateJapaneseSubtitles(input), /Unsupported model format/);
|
||||
await assert.rejects(readFile(input.callsPath), /ENOENT/);
|
||||
}));
|
||||
|
||||
test('generation picks Japanese audio, restores timeline offsets, reports split progress, and preserves existing output', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const existing = path.join(directory, 'episode.ja.generated.srt');
|
||||
await writeFile(existing, 'user subtitles');
|
||||
const progress: SubtitleGenerationProgress[] = [];
|
||||
const result = await generateJapaneseSubtitles({
|
||||
...input,
|
||||
onProgress: (event) => progress.push(event),
|
||||
});
|
||||
assert.equal(result, path.join(directory, 'episode.ja.generated.1.srt'));
|
||||
assert.equal(await readFile(existing, 'utf8'), 'user subtitles');
|
||||
assert.match(await readFile(result, 'utf8'), /00:00:03,500 --> 00:00:04,500\nこんにちは/);
|
||||
const calls = (await readFile(input.callsPath, 'utf8'))
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((line): unknown => JSON.parse(line));
|
||||
assert.ok(Array.isArray(calls[1]));
|
||||
assert.ok(calls[1].includes('0:3'));
|
||||
assert.ok(Array.isArray(calls[2]));
|
||||
assert.ok(calls[2].includes('ja'));
|
||||
assert.ok(calls[2].includes('-osrt'));
|
||||
assert.ok(progress.some((event) => event.stage === 'extract' && event.percent === 50));
|
||||
assert.ok(progress.some((event) => event.stage === 'transcribe' && event.percent === 55));
|
||||
assert.deepEqual(
|
||||
(await readdir(directory)).filter((file) => file.startsWith('.subminer-')),
|
||||
[],
|
||||
);
|
||||
}));
|
||||
|
||||
test('explicit audio stream and output path are respected without overwriting existing files', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const outputPath = path.join(directory, 'chosen.srt');
|
||||
const result = await generateJapaneseSubtitles({ ...input, audioStreamIndex: 1, outputPath });
|
||||
assert.equal(result, outputPath);
|
||||
assert.match(await readFile(result, 'utf8'), /00:00:01,000 --> 00:00:02,000/);
|
||||
await assert.rejects(generateJapaneseSubtitles({ ...input, outputPath }), /already exists/);
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({ ...input, audioStreamIndex: 99 }),
|
||||
/stream 99 was not found/,
|
||||
);
|
||||
}));
|
||||
|
||||
test('dialogue generation isolates Whisper state between passages and preserves media timing', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const vadModelPath = path.join(directory, 'vad.bin');
|
||||
await writeFile(vadModelPath, 'speech detector model');
|
||||
const vadPath = await executable(
|
||||
directory,
|
||||
'vad',
|
||||
"process.stdout.write('Detected 2 speech segments:\\nSpeech segment 0: start = 1000.00, end = 1100.00\\nSpeech segment 1: start = 10000.00, end = 10100.00\\n');",
|
||||
);
|
||||
const whisperPath = await executable(
|
||||
directory,
|
||||
'dialogue-whisper',
|
||||
`const args = process.argv.slice(2);
|
||||
let files = 0;
|
||||
for (let i = 0; i < args.length; i++) if (args[i] === '-of') {
|
||||
// Reproduce a decoder that degenerates when reused for another audio file.
|
||||
const text = files++ === 0 ? 'はい' : 'お' + 'ぉ'.repeat(40) + 'ぇ'.repeat(178);
|
||||
require('node:fs').writeFileSync(args[i + 1] + '.srt', '1\\n00:00:00,000 --> 00:01:39,000\\n' + text + '\\n');
|
||||
}`,
|
||||
);
|
||||
const output = await generateJapaneseSubtitles({
|
||||
...input,
|
||||
config: { ...input.config, vadModelPath, vadPath, whisperPath },
|
||||
});
|
||||
assert.equal(
|
||||
await readFile(output, 'utf8'),
|
||||
'1\n00:00:12,500 --> 00:00:13,500\nはい\n\n2\n00:01:42,500 --> 00:01:43,500\nはい\n',
|
||||
);
|
||||
}));
|
||||
|
||||
test('dialogue generation uses quiet pauses and stitches overlapping chunks on the media timeline', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const vadModelPath = path.join(directory, 'vad.bin');
|
||||
await writeFile(vadModelPath, 'speech detector model');
|
||||
const vadPath = await executable(
|
||||
directory,
|
||||
'vad',
|
||||
`const assert = require('node:assert/strict');
|
||||
const args = process.argv.slice(2);
|
||||
assert.equal(args[args.indexOf('--vad-min-speech-duration-ms') + 1], '100');
|
||||
assert.equal(args[args.indexOf('-vp') + 1], '350');
|
||||
process.stdout.write('Detected 1 speech segments:\\nSpeech segment 0: start = 1000.00, end = 4500.00\\n');`,
|
||||
);
|
||||
const ffmpegPath = await executable(
|
||||
directory,
|
||||
'pause-ffmpeg',
|
||||
`const args = process.argv.slice(2);
|
||||
if (args.includes('silencedetect=noise=-50dB:d=0.5')) {
|
||||
process.stderr.write('[silencedetect] silence_end: 45 | silence_duration: 45\\n');
|
||||
process.stdout.write('out_time_us=45000000\\n');
|
||||
} else if (args.includes('-af')) {
|
||||
process.stderr.write('[silencedetect] silence_end: 28.1 | silence_duration: 0.2\\n');
|
||||
} else {
|
||||
require('node:fs').writeFileSync(args.at(-1), 'wav');
|
||||
}`,
|
||||
);
|
||||
const whisperPath = await executable(
|
||||
directory,
|
||||
'overlap-whisper',
|
||||
`const args = process.argv.slice(2);
|
||||
for (let i = 0; i < args.length; i++) if (args[i] === '-of') {
|
||||
const time = args[i + 1].endsWith('speech-0')
|
||||
? '00:00:17,800 --> 00:00:18,250'
|
||||
: '00:00:00,100 --> 00:00:00,650';
|
||||
require('node:fs').writeFileSync(args[i + 1] + '.srt', '1\\n' + time + '\\nはい\\n');
|
||||
}`,
|
||||
);
|
||||
const output = await generateJapaneseSubtitles({
|
||||
...input,
|
||||
config: { ...input.config, vadModelPath, vadPath, ffmpegPath, whisperPath },
|
||||
});
|
||||
assert.equal(await readFile(output, 'utf8'), '1\n00:00:30,300 --> 00:00:30,900\nはい\n');
|
||||
}));
|
||||
|
||||
test('silent audio with no detected speech stops generation without transcription', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const vadModelPath = path.join(directory, 'vad.bin');
|
||||
await writeFile(vadModelPath, 'speech detector model');
|
||||
const vadPath = await executable(
|
||||
directory,
|
||||
'vad',
|
||||
"process.stdout.write('Detected 0 speech segments:\\n');",
|
||||
);
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({ ...input, config: { ...input.config, vadModelPath, vadPath } }),
|
||||
/No spoken dialogue detected/,
|
||||
);
|
||||
assert.equal((await readFile(input.callsPath, 'utf8')).trim().split('\n').length, 3);
|
||||
assert.deepEqual(
|
||||
(await readdir(directory)).filter((file) => file.endsWith('.srt')),
|
||||
[],
|
||||
);
|
||||
}));
|
||||
|
||||
test('dialogue generation retains audible audio rejected by VAD', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const vadModelPath = path.join(directory, 'vad.bin');
|
||||
await writeFile(vadModelPath, 'speech detector model');
|
||||
const vadPath = await executable(
|
||||
directory,
|
||||
'vad',
|
||||
"process.stdout.write('Detected 0 speech segments:\\n');",
|
||||
);
|
||||
const ffmpegPath = await executable(
|
||||
directory,
|
||||
'audible-ffmpeg',
|
||||
`
|
||||
const args = process.argv.slice(2);
|
||||
if (args.at(-1) === '-') {
|
||||
process.stdout.write('out_time_us=19000000\\nprogress=end\\n');
|
||||
} else {
|
||||
require('node:fs').writeFileSync(args.at(-1), 'wav');
|
||||
}`,
|
||||
);
|
||||
const output = await generateJapaneseSubtitles({
|
||||
...input,
|
||||
config: { ...input.config, vadModelPath, vadPath, ffmpegPath },
|
||||
});
|
||||
assert.match(await readFile(output, 'utf8'), /00:00:03,500 --> 00:00:04,500\nこんにちは/);
|
||||
}));
|
||||
|
||||
test('empty executable paths find tools on PATH and explicit overrides take precedence', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const previousPath = process.env.PATH;
|
||||
process.env.PATH = directory;
|
||||
try {
|
||||
const config = {
|
||||
...DEFAULT_SUBTITLE_GENERATION_CONFIG,
|
||||
modelPath: input.config.modelPath,
|
||||
};
|
||||
const result = await generateJapaneseSubtitles({ ...input, config });
|
||||
assert.match(await readFile(result, 'utf8'), /こんにちは/);
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({
|
||||
...input,
|
||||
config: { ...config, ffprobePath: path.join(directory, 'missing-override') },
|
||||
}),
|
||||
/missing-override \(subtitleGeneration\.ffprobePath\) is not an executable file/,
|
||||
);
|
||||
} finally {
|
||||
if (previousPath === undefined) delete process.env.PATH;
|
||||
else process.env.PATH = previousPath;
|
||||
}
|
||||
}));
|
||||
|
||||
test('generation rejects remote media, missing models, and missing tools before starting a subprocess', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({ ...input, mediaPath: 'https://example.com/movie.mkv' }),
|
||||
/local media file/,
|
||||
);
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({ ...input, config: { ...input.config, modelPath: '' } }),
|
||||
/No Whisper model found/,
|
||||
);
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({
|
||||
...input,
|
||||
config: {
|
||||
...input.config,
|
||||
vadModelPath: path.join(directory, 'vad.bin'),
|
||||
vadPath: path.join(directory, 'missing-detector'),
|
||||
},
|
||||
}),
|
||||
/missing-detector \(subtitleGeneration\.vadPath\) is not an executable file/,
|
||||
);
|
||||
await assert.rejects(readFile(input.callsPath), /ENOENT/);
|
||||
}));
|
||||
|
||||
test(
|
||||
'directory permission preflight rejects a write-only destination',
|
||||
{
|
||||
skip: process.platform === 'win32' || process.getuid?.() === 0,
|
||||
},
|
||||
() =>
|
||||
fixture(async (directory) => {
|
||||
const writeOnly = path.join(directory, 'write-only');
|
||||
await mkdir(writeOnly);
|
||||
try {
|
||||
await chmod(writeOnly, 0o200);
|
||||
await access(writeOnly, constants.W_OK);
|
||||
await assert.rejects(ensureWritableDirectory(writeOnly), /write-only is not writable/);
|
||||
} finally {
|
||||
await chmod(writeOnly, 0o755);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
test(
|
||||
'generation rejects an unwritable destination before extracting audio',
|
||||
{
|
||||
skip: process.platform === 'win32' || process.getuid?.() === 0,
|
||||
},
|
||||
() =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const readOnly = path.join(directory, 'read-only');
|
||||
await mkdir(readOnly, { mode: 0o555 });
|
||||
try {
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({ ...input, outputPath: path.join(readOnly, 'out.srt') }),
|
||||
/read-only is not writable/,
|
||||
);
|
||||
await assert.rejects(readFile(input.callsPath), /ENOENT/);
|
||||
} finally {
|
||||
await chmod(readOnly, 0o755);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
test('process cancellation terminates work and bounds diagnostic output', () =>
|
||||
fixture(async (directory) => {
|
||||
const slow = await executable(
|
||||
directory,
|
||||
'slow',
|
||||
"process.stdout.write('ready\\n');setInterval(()=>{},1000);",
|
||||
);
|
||||
const controller = new AbortController();
|
||||
await assert.rejects(
|
||||
runSubtitleGenerationProcess({
|
||||
command: slow,
|
||||
args: [],
|
||||
signal: controller.signal,
|
||||
onLine: () => controller.abort(),
|
||||
}),
|
||||
/cancelled/,
|
||||
);
|
||||
const failed = await executable(
|
||||
directory,
|
||||
'failed',
|
||||
"process.stderr.write('x'.repeat(100000));process.exitCode=7;",
|
||||
);
|
||||
await assert.rejects(
|
||||
runSubtitleGenerationProcess({ command: failed, args: [] }),
|
||||
(error: unknown) =>
|
||||
error instanceof Error &&
|
||||
error.message.length < 66000 &&
|
||||
error.message.includes('status 7'),
|
||||
);
|
||||
}));
|
||||
|
||||
test('download uses the pinned model revision and removes files that fail integrity', () =>
|
||||
fixture(async (directory) => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = Object.assign(async (request: string | URL | Request) => {
|
||||
assert.equal(
|
||||
request,
|
||||
'https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-small.bin',
|
||||
);
|
||||
return new Response('not a model');
|
||||
}, originalFetch);
|
||||
try {
|
||||
await assert.rejects(
|
||||
downloadSubtitleGenerationModel({
|
||||
config: DEFAULT_SUBTITLE_GENERATION_CONFIG,
|
||||
modelDirectory: directory,
|
||||
}),
|
||||
/integrity verification/,
|
||||
);
|
||||
assert.deepEqual(await readdir(directory), []);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
}));
|
||||
@@ -0,0 +1,315 @@
|
||||
import { access, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import type {
|
||||
SubtitleGenerationConfig,
|
||||
SubtitleGenerationProgress,
|
||||
} from '../../shared/subtitle-generation';
|
||||
import { isMissingFile, resolveSubtitleGenerationModel } from './subtitle-generation-models';
|
||||
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
|
||||
import { publishSubtitleGenerationFile } from './subtitle-generation-files';
|
||||
import { formatTimestamp } from './subtitle-generation-srt';
|
||||
import { transcribeSubtitleDialogue } from './subtitle-generation-dialogue';
|
||||
import {
|
||||
requireSubtitleGenerationTools,
|
||||
resolveSubtitleGenerationTools,
|
||||
} from './subtitle-generation-tools';
|
||||
|
||||
export {
|
||||
downloadSubtitleGenerationModel,
|
||||
resolveSubtitleGenerationModel,
|
||||
} from './subtitle-generation-models';
|
||||
export { resolveSubtitleGenerationTools } from './subtitle-generation-tools';
|
||||
|
||||
function numericTime(value: unknown): number | undefined {
|
||||
if (typeof value !== 'number' && typeof value !== 'string') return undefined;
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) ? number : undefined;
|
||||
}
|
||||
|
||||
function parseAudioProbe(raw: string, selectedIndex: number | undefined) {
|
||||
const value: unknown = JSON.parse(raw);
|
||||
if (
|
||||
typeof value !== 'object' ||
|
||||
value === null ||
|
||||
!('streams' in value) ||
|
||||
!Array.isArray(value.streams)
|
||||
) {
|
||||
throw new Error('ffprobe did not return media streams.');
|
||||
}
|
||||
const streams = value.streams.flatMap((stream: unknown) => {
|
||||
if (
|
||||
typeof stream !== 'object' ||
|
||||
stream === null ||
|
||||
!('codec_type' in stream) ||
|
||||
stream.codec_type !== 'audio' ||
|
||||
!('index' in stream) ||
|
||||
typeof stream.index !== 'number' ||
|
||||
!Number.isInteger(stream.index) ||
|
||||
stream.index < 0
|
||||
)
|
||||
return [];
|
||||
const tags = 'tags' in stream ? stream.tags : undefined;
|
||||
const language =
|
||||
typeof tags === 'object' && tags !== null && 'language' in tags ? tags.language : undefined;
|
||||
return [
|
||||
{
|
||||
index: stream.index,
|
||||
start: 'start_time' in stream ? numericTime(stream.start_time) : undefined,
|
||||
duration: 'duration' in stream ? numericTime(stream.duration) : undefined,
|
||||
japanese: language === 'ja' || language === 'jpn',
|
||||
},
|
||||
];
|
||||
});
|
||||
const selected =
|
||||
selectedIndex === undefined
|
||||
? (streams.find((stream) => stream.japanese) ?? streams[0])
|
||||
: streams.find((stream) => stream.index === selectedIndex);
|
||||
if (!selected)
|
||||
throw new Error(
|
||||
selectedIndex === undefined
|
||||
? 'No audio track found.'
|
||||
: `Audio stream ${selectedIndex} was not found.`,
|
||||
);
|
||||
const format = 'format' in value ? value.format : undefined;
|
||||
const formatStart =
|
||||
typeof format === 'object' && format !== null && 'start_time' in format
|
||||
? (numericTime(format.start_time) ?? 0)
|
||||
: 0;
|
||||
const duration =
|
||||
selected.duration ??
|
||||
(typeof format === 'object' && format !== null && 'duration' in format
|
||||
? numericTime(format.duration)
|
||||
: undefined);
|
||||
// mpv rebases media timestamps to the container start. Extraction rebases the selected audio.
|
||||
return { index: selected.index, offset: (selected.start ?? formatStart) - formatStart, duration };
|
||||
}
|
||||
|
||||
function shiftSubtitleTimestamps(srt: string, offsetSeconds: number): string {
|
||||
let cueCount = 0;
|
||||
const result = srt.replace(
|
||||
/(\d{2,}):(\d{2}):(\d{2}),(\d{3}) --> (\d{2,}):(\d{2}):(\d{2}),(\d{3})/g,
|
||||
(
|
||||
_match,
|
||||
sh: string,
|
||||
sm: string,
|
||||
ss: string,
|
||||
sms: string,
|
||||
eh: string,
|
||||
em: string,
|
||||
es: string,
|
||||
ems: string,
|
||||
) => {
|
||||
cueCount += 1;
|
||||
const start = Number(sh) * 3600000 + Number(sm) * 60000 + Number(ss) * 1000 + Number(sms);
|
||||
const end = Number(eh) * 3600000 + Number(em) * 60000 + Number(es) * 1000 + Number(ems);
|
||||
return `${formatTimestamp(start + offsetSeconds * 1000)} --> ${formatTimestamp(end + offsetSeconds * 1000)}`;
|
||||
},
|
||||
);
|
||||
if (cueCount === 0)
|
||||
throw new Error(
|
||||
'Whisper produced no subtitle cues. The audio may contain no recognized speech.',
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function ensureAvailableOutput(outputPath: string): Promise<void> {
|
||||
try {
|
||||
await stat(outputPath);
|
||||
} catch (error) {
|
||||
if (isMissingFile(error)) return;
|
||||
throw error;
|
||||
}
|
||||
throw new Error(`Subtitle output already exists: ${outputPath}`);
|
||||
}
|
||||
|
||||
// Fail before extraction and transcription when the destination cannot take the file.
|
||||
export async function ensureWritableDirectory(directory: string): Promise<void> {
|
||||
try {
|
||||
await access(directory, constants.W_OK | constants.X_OK);
|
||||
} catch {
|
||||
throw new Error(`Cannot save subtitles: ${directory} is not writable.`);
|
||||
}
|
||||
}
|
||||
|
||||
async function writeSubtitles(input: {
|
||||
mediaPath: string;
|
||||
outputPath?: string;
|
||||
contents: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
const parsed = path.parse(input.mediaPath);
|
||||
const directory = input.outputPath ? path.dirname(path.resolve(input.outputPath)) : parsed.dir;
|
||||
const temporaryDirectory = await mkdtemp(path.join(directory, '.subminer-subtitles-'));
|
||||
try {
|
||||
const staged = path.join(temporaryDirectory, 'subtitles.srt');
|
||||
await writeFile(staged, input.contents, { flag: 'wx' });
|
||||
for (let suffix = 0; ; suffix += 1) {
|
||||
input.signal?.throwIfAborted();
|
||||
const destination = input.outputPath
|
||||
? path.resolve(input.outputPath)
|
||||
: path.join(directory, `${parsed.name}.ja.generated${suffix ? `.${suffix}` : ''}.srt`);
|
||||
try {
|
||||
await publishSubtitleGenerationFile(staged, destination);
|
||||
return destination;
|
||||
} catch (error) {
|
||||
if (
|
||||
!input.outputPath &&
|
||||
error instanceof Error &&
|
||||
'code' in error &&
|
||||
error.code === 'EEXIST'
|
||||
)
|
||||
continue;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await rm(temporaryDirectory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateJapaneseSubtitles(input: {
|
||||
config: SubtitleGenerationConfig;
|
||||
modelDirectory: string;
|
||||
mediaPath: string;
|
||||
audioStreamIndex?: number;
|
||||
outputPath?: string;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
input.signal?.throwIfAborted();
|
||||
if (/^[a-z][a-z\d+.-]*:\/\//i.test(input.mediaPath))
|
||||
throw new Error('Subtitle generation requires a local media file.');
|
||||
const mediaPath = path.resolve(input.mediaPath);
|
||||
if (!(await stat(mediaPath)).isFile())
|
||||
throw new Error('Subtitle generation requires a local media file.');
|
||||
if (input.outputPath) await ensureAvailableOutput(path.resolve(input.outputPath));
|
||||
await ensureWritableDirectory(
|
||||
input.outputPath ? path.dirname(path.resolve(input.outputPath)) : path.dirname(mediaPath),
|
||||
);
|
||||
const model = await resolveSubtitleGenerationModel(input.config, input.modelDirectory);
|
||||
if (model.kind === 'missing')
|
||||
throw new Error(
|
||||
'No Whisper model found. Download a model or configure an existing model path.',
|
||||
);
|
||||
if (model.kind === 'invalid') throw new Error(model.message);
|
||||
const tools = requireSubtitleGenerationTools(await resolveSubtitleGenerationTools(input.config));
|
||||
input.onProgress?.({ stage: 'extract', message: 'Inspecting audio tracks...' });
|
||||
const probe = await runSubtitleGenerationProcess({
|
||||
command: tools.ffprobe,
|
||||
args: [
|
||||
'-v',
|
||||
'error',
|
||||
'-show_entries',
|
||||
'stream=index,codec_type,start_time,duration:stream_tags=language:format=start_time,duration',
|
||||
'-of',
|
||||
'json',
|
||||
mediaPath,
|
||||
],
|
||||
signal: input.signal,
|
||||
});
|
||||
const audio = parseAudioProbe(probe, input.audioStreamIndex);
|
||||
const temporaryDirectory = await mkdtemp(path.join(tmpdir(), 'subminer-whisper-'));
|
||||
try {
|
||||
const wavPath = path.join(temporaryDirectory, 'audio.wav');
|
||||
const subtitleBase = path.join(temporaryDirectory, 'subtitles');
|
||||
input.onProgress?.({ stage: 'extract', percent: 0, message: 'Extracting audio...' });
|
||||
await runSubtitleGenerationProcess({
|
||||
command: tools.ffmpeg,
|
||||
args: [
|
||||
'-nostdin',
|
||||
'-hide_banner',
|
||||
'-loglevel',
|
||||
'error',
|
||||
'-i',
|
||||
mediaPath,
|
||||
'-map',
|
||||
`0:${audio.index}`,
|
||||
'-vn',
|
||||
'-af',
|
||||
'asetpts=PTS-STARTPTS',
|
||||
'-ac',
|
||||
'1',
|
||||
'-ar',
|
||||
'16000',
|
||||
'-c:a',
|
||||
'pcm_s16le',
|
||||
'-progress',
|
||||
'pipe:1',
|
||||
'-nostats',
|
||||
wavPath,
|
||||
],
|
||||
signal: input.signal,
|
||||
onLine: (line) => {
|
||||
const match = /^out_time_us=(\d+)$/.exec(line);
|
||||
if (match && audio.duration && audio.duration > 0) {
|
||||
input.onProgress?.({
|
||||
stage: 'extract',
|
||||
percent: Math.min(100, Math.floor(Number(match[1]) / 10000 / audio.duration)),
|
||||
message: 'Extracting audio...',
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
input.onProgress?.({
|
||||
stage: 'transcribe',
|
||||
percent: 0,
|
||||
message: 'Generating Japanese subtitles...',
|
||||
});
|
||||
let srt: string;
|
||||
if (tools.vad !== null) {
|
||||
srt = await transcribeSubtitleDialogue({
|
||||
config: input.config,
|
||||
tools: { ...tools, vad: tools.vad },
|
||||
modelPath: model.path,
|
||||
wavPath,
|
||||
directory: temporaryDirectory,
|
||||
signal: input.signal,
|
||||
onProgress: input.onProgress,
|
||||
});
|
||||
} else {
|
||||
await runSubtitleGenerationProcess({
|
||||
command: tools.whisper,
|
||||
args: [
|
||||
'-m',
|
||||
model.path,
|
||||
'-f',
|
||||
wavPath,
|
||||
'-l',
|
||||
'ja',
|
||||
'-t',
|
||||
String(input.config.threads),
|
||||
'-osrt',
|
||||
'-of',
|
||||
subtitleBase,
|
||||
'-pp',
|
||||
],
|
||||
signal: input.signal,
|
||||
onLine: (line) => {
|
||||
const match = /progress\s*=\s*(\d+(?:\.\d+)?)%/.exec(line);
|
||||
if (match)
|
||||
input.onProgress?.({
|
||||
stage: 'transcribe',
|
||||
percent: Math.min(100, Number(match[1])),
|
||||
message: 'Generating Japanese subtitles...',
|
||||
});
|
||||
},
|
||||
});
|
||||
srt = await readFile(`${subtitleBase}.srt`, 'utf8');
|
||||
}
|
||||
input.signal?.throwIfAborted();
|
||||
input.onProgress?.({ stage: 'write', message: 'Saving Japanese subtitles...' });
|
||||
const contents = shiftSubtitleTimestamps(srt, audio.offset);
|
||||
const outputPath = await writeSubtitles({
|
||||
mediaPath,
|
||||
outputPath: input.outputPath,
|
||||
contents,
|
||||
signal: input.signal,
|
||||
});
|
||||
input.onProgress?.({ stage: 'write', percent: 100, message: 'Japanese subtitles are ready.' });
|
||||
return outputPath;
|
||||
} finally {
|
||||
await rm(temporaryDirectory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ export interface ConfiguredShortcuts {
|
||||
openRuntimeOptions: string | null | undefined;
|
||||
openJimaku: string | null | undefined;
|
||||
openTsukihime: string | null | undefined;
|
||||
openSubtitleGeneration: string | null | undefined;
|
||||
openSessionHelp: string | null | undefined;
|
||||
openControllerSelect: string | null | undefined;
|
||||
openControllerDebug: string | null | undefined;
|
||||
@@ -67,6 +68,7 @@ export function resolveConfiguredShortcuts(
|
||||
openRuntimeOptions: normalizeShortcut(shortcutValue('openRuntimeOptions')),
|
||||
openJimaku: normalizeShortcut(shortcutValue('openJimaku')),
|
||||
openTsukihime: normalizeShortcut(shortcutValue('openTsukihime')),
|
||||
openSubtitleGeneration: normalizeShortcut(shortcutValue('openSubtitleGeneration')),
|
||||
openSessionHelp: normalizeShortcut(shortcutValue('openSessionHelp')),
|
||||
openControllerSelect: normalizeShortcut(shortcutValue('openControllerSelect')),
|
||||
openControllerDebug: normalizeShortcut(shortcutValue('openControllerDebug')),
|
||||
|
||||
Reference in New Issue
Block a user