mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-09-19 05:16:27 -07:00
feat(subtitles): add local Japanese subtitle generation (#240)
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import { RawConfig, ResolvedConfig } from '../types/config';
|
||||
import { DEFAULT_SUBTITLE_GENERATION_CONFIG } from '../shared/subtitle-generation';
|
||||
import { CORE_DEFAULT_CONFIG } from './definitions/defaults-core';
|
||||
import { IMMERSION_DEFAULT_CONFIG } from './definitions/defaults-immersion';
|
||||
import { INTEGRATIONS_DEFAULT_CONFIG } from './definitions/defaults-integrations';
|
||||
@@ -54,6 +55,7 @@ const { immersionTracking } = IMMERSION_DEFAULT_CONFIG;
|
||||
const { stats } = STATS_DEFAULT_CONFIG;
|
||||
|
||||
export const DEFAULT_CONFIG: ResolvedConfig = {
|
||||
subtitleGeneration: { ...DEFAULT_SUBTITLE_GENERATION_CONFIG },
|
||||
subtitlePosition,
|
||||
keybindings,
|
||||
websocket,
|
||||
|
||||
@@ -99,6 +99,7 @@ export const CORE_DEFAULT_CONFIG: Pick<
|
||||
openRuntimeOptions: 'CommandOrControl+Shift+O',
|
||||
openJimaku: 'Ctrl+Shift+J',
|
||||
openTsukihime: 'Ctrl+Shift+T',
|
||||
openSubtitleGeneration: 'Ctrl+Shift+G',
|
||||
openSessionHelp: 'CommandOrControl+Slash',
|
||||
openControllerSelect: 'Alt+C',
|
||||
openControllerDebug: 'Alt+Shift+C',
|
||||
|
||||
@@ -628,6 +628,12 @@ export function buildCoreConfigOptionRegistry(
|
||||
defaultValue: defaultConfig.shortcuts.openSessionHelp,
|
||||
description: 'Accelerator that opens the session help / keybinding cheatsheet.',
|
||||
},
|
||||
{
|
||||
path: 'shortcuts.openSubtitleGeneration',
|
||||
kind: 'string',
|
||||
defaultValue: defaultConfig.shortcuts.openSubtitleGeneration,
|
||||
description: 'Accelerator that opens the standalone Japanese subtitle generation modal.',
|
||||
},
|
||||
{
|
||||
path: 'shortcuts.openControllerSelect',
|
||||
kind: 'string',
|
||||
|
||||
@@ -1,10 +1,46 @@
|
||||
import { ResolvedConfig } from '../../types/config';
|
||||
import { ConfigOptionRegistryEntry } from './shared';
|
||||
import { SUBTITLE_GENERATION_MODELS } from '../../shared/subtitle-generation-model-catalog';
|
||||
|
||||
export function buildSubtitleConfigOptionRegistry(
|
||||
defaultConfig: ResolvedConfig,
|
||||
): ConfigOptionRegistryEntry[] {
|
||||
return [
|
||||
...(
|
||||
['whisperPath', 'modelPath', 'ffmpegPath', 'ffprobePath', 'vadModelPath', 'vadPath'] as const
|
||||
).map((key) => ({
|
||||
path: `subtitleGeneration.${key}`,
|
||||
kind: 'string' as const,
|
||||
defaultValue: defaultConfig.subtitleGeneration[key],
|
||||
description: {
|
||||
whisperPath:
|
||||
'Optional path override for whisper.cpp. Leave empty to find whisper-cli on PATH.',
|
||||
modelPath:
|
||||
'Path to an existing multilingual whisper.cpp GGML model. Leave empty to use a SubMiner-managed model. A configured path always takes precedence.',
|
||||
ffmpegPath:
|
||||
'Optional FFmpeg path override for audio extraction. Leave empty to find ffmpeg on PATH.',
|
||||
ffprobePath:
|
||||
'Optional FFprobe path override for audio tracks and timing. Leave empty to find ffprobe on PATH.',
|
||||
vadModelPath:
|
||||
'Path to a whisper.cpp Silero VAD model. Enables dialogue-focused generation while retaining uncertain audible sections, which may include songs. Leave empty to transcribe the full audio.',
|
||||
vadPath:
|
||||
'Optional speech detector executable override. With vadModelPath configured, leave empty to find whisper-vad-speech-segments or vad-speech-segments on PATH.',
|
||||
}[key],
|
||||
})),
|
||||
{
|
||||
path: 'subtitleGeneration.managedModel',
|
||||
kind: 'enum',
|
||||
enumValues: SUBTITLE_GENERATION_MODELS.map((model) => model.id),
|
||||
defaultValue: defaultConfig.subtitleGeneration.managedModel,
|
||||
description:
|
||||
'Multilingual whisper.cpp model to use when modelPath is empty. Download it explicitly from the generation modal or launcher.',
|
||||
},
|
||||
{
|
||||
path: 'subtitleGeneration.threads',
|
||||
kind: 'number',
|
||||
defaultValue: defaultConfig.subtitleGeneration.threads,
|
||||
description: 'Positive integer CPU thread count for whisper.cpp Japanese transcription.',
|
||||
},
|
||||
{
|
||||
path: 'subtitleStyle.primaryDefaultMode',
|
||||
kind: 'enum',
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
import { ConfigTemplateSection } from './shared';
|
||||
|
||||
const CORE_TEMPLATE_SECTIONS: ConfigTemplateSection[] = [
|
||||
{
|
||||
title: 'Japanese Subtitle Generation',
|
||||
description: [
|
||||
'Generate timed Japanese subtitles from local audio using whisper.cpp.',
|
||||
'Configure an existing GGML model path or explicitly download a SubMiner-managed model.',
|
||||
],
|
||||
notes: ['Hot-reload: settings apply to the next generation or model download.'],
|
||||
key: 'subtitleGeneration',
|
||||
},
|
||||
{
|
||||
title: 'Visible Overlay Auto-Start',
|
||||
description: [
|
||||
|
||||
@@ -237,6 +237,7 @@ export function applyCoreDomainConfig(context: ResolveContext): void {
|
||||
'openRuntimeOptions',
|
||||
'openJimaku',
|
||||
'openTsukihime',
|
||||
'openSubtitleGeneration',
|
||||
'openSessionHelp',
|
||||
'openControllerSelect',
|
||||
'openControllerDebug',
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { ResolvedConfig } from '../../types/config';
|
||||
import { ResolveContext } from './context';
|
||||
import { resolveSubtitleGenerationConfig } from '../../shared/subtitle-generation';
|
||||
import {
|
||||
asBoolean,
|
||||
asColor,
|
||||
@@ -46,6 +47,17 @@ function applySubtitleHoverTokenCssCompatibility(
|
||||
|
||||
export function applySubtitleDomainConfig(context: ResolveContext): void {
|
||||
const { src, resolved, warn } = context;
|
||||
resolved.subtitleGeneration = resolveSubtitleGenerationConfig(
|
||||
src.subtitleGeneration,
|
||||
(key, value, message) => {
|
||||
const configPath = key === 'subtitleGeneration' ? key : `subtitleGeneration.${key}`;
|
||||
const fallback =
|
||||
key === 'subtitleGeneration'
|
||||
? resolved.subtitleGeneration
|
||||
: Object.entries(resolved.subtitleGeneration).find(([name]) => name === key)?.[1];
|
||||
warn(configPath, value, fallback, message);
|
||||
},
|
||||
);
|
||||
|
||||
if (isObject(src.jimaku)) {
|
||||
const apiKey = asString(src.jimaku.apiKey);
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { resolveConfig } from '../resolve';
|
||||
import { buildConfigSettingsRegistry } from '../settings/registry';
|
||||
import { SUBTITLE_GENERATION_MODELS } from '../../shared/subtitle-generation-model-catalog';
|
||||
import { resolveSubtitleGenerationConfig } from '../../shared/subtitle-generation';
|
||||
|
||||
test('every downloadable multilingual model is accepted by config and offered in settings', () => {
|
||||
const settings = buildConfigSettingsRegistry(resolveConfig({}).resolved).find(
|
||||
(entry) => entry.configPath === 'subtitleGeneration.managedModel',
|
||||
);
|
||||
assert.deepEqual(
|
||||
settings?.enumValues,
|
||||
SUBTITLE_GENERATION_MODELS.map((model) => model.id),
|
||||
);
|
||||
for (const { id } of SUBTITLE_GENERATION_MODELS) {
|
||||
const { resolved, warnings } = resolveConfig({ subtitleGeneration: { managedModel: id } });
|
||||
assert.equal(resolved.subtitleGeneration.managedModel, id);
|
||||
assert.equal(warnings.length, 0);
|
||||
}
|
||||
});
|
||||
|
||||
test('generation config rejects English-only, unknown, and prototype model names', () => {
|
||||
for (const managedModel of ['tiny.en', 'small.en-q5_1', 'unknown', 'toString', '__proto__']) {
|
||||
const warnings: string[] = [];
|
||||
const resolved = resolveSubtitleGenerationConfig({ managedModel }, (key) => warnings.push(key));
|
||||
assert.equal(resolved.managedModel, 'small');
|
||||
assert.equal(warnings.length, 1);
|
||||
}
|
||||
});
|
||||
|
||||
test('generation config is resolved and its external model path is editable without restart', () => {
|
||||
const { resolved, warnings } = resolveConfig({
|
||||
subtitleGeneration: { modelPath: '/models/japanese.bin', managedModel: 'medium', threads: 8 },
|
||||
});
|
||||
assert.equal(resolved.subtitleGeneration.modelPath, '/models/japanese.bin');
|
||||
assert.equal(resolved.subtitleGeneration.managedModel, 'medium');
|
||||
assert.equal(warnings.length, 0);
|
||||
const field = buildConfigSettingsRegistry(resolved).find(
|
||||
(entry) => entry.configPath === 'subtitleGeneration.modelPath',
|
||||
);
|
||||
assert.equal(field?.category, 'integrations');
|
||||
assert.equal(field?.restartBehavior, 'hot-reload');
|
||||
});
|
||||
|
||||
test('generation executable overrides default to empty and accept blank values without warnings', () => {
|
||||
for (const subtitleGeneration of [
|
||||
{},
|
||||
{ whisperPath: '', ffmpegPath: '', ffprobePath: '' },
|
||||
{ whisperPath: ' ', ffmpegPath: ' ', ffprobePath: ' ' },
|
||||
]) {
|
||||
const { resolved, warnings } = resolveConfig({ subtitleGeneration });
|
||||
assert.equal(resolved.subtitleGeneration.whisperPath, '');
|
||||
assert.equal(resolved.subtitleGeneration.ffmpegPath, '');
|
||||
assert.equal(resolved.subtitleGeneration.ffprobePath, '');
|
||||
assert.equal(warnings.length, 0);
|
||||
}
|
||||
});
|
||||
|
||||
test('subtitle generation shortcut can be customized or disabled', () => {
|
||||
assert.equal(resolveConfig({}).resolved.shortcuts.openSubtitleGeneration, 'Ctrl+Shift+G');
|
||||
assert.equal(
|
||||
resolveConfig({ shortcuts: { openSubtitleGeneration: 'Ctrl+Alt+G' } }).resolved.shortcuts
|
||||
.openSubtitleGeneration,
|
||||
'Ctrl+Alt+G',
|
||||
);
|
||||
assert.equal(
|
||||
resolveConfig({ shortcuts: { openSubtitleGeneration: null } }).resolved.shortcuts
|
||||
.openSubtitleGeneration,
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test('dialogue detection paths are optional, validated, and editable in Settings', () => {
|
||||
const { resolved, warnings } = resolveConfig({
|
||||
subtitleGeneration: { vadModelPath: ' /models/silero.bin ', vadPath: ' /bin/vad ' },
|
||||
});
|
||||
assert.equal(resolved.subtitleGeneration.vadModelPath, '/models/silero.bin');
|
||||
assert.equal(resolved.subtitleGeneration.vadPath, '/bin/vad');
|
||||
assert.equal(warnings.length, 0);
|
||||
for (const key of ['vadModelPath', 'vadPath'] as const) {
|
||||
const field = buildConfigSettingsRegistry(resolved).find(
|
||||
(entry) => entry.configPath === `subtitleGeneration.${key}`,
|
||||
);
|
||||
assert.equal(field?.category, 'integrations');
|
||||
assert.equal(field?.restartBehavior, 'hot-reload');
|
||||
assert.equal(resolveConfig({}).resolved.subtitleGeneration[key], '');
|
||||
assert.equal(resolveConfig({ subtitleGeneration: { [key]: false } }).warnings.length, 1);
|
||||
}
|
||||
});
|
||||
@@ -448,6 +448,9 @@ function categoryAndSection(path: string): { category: ConfigSettingsCategory; s
|
||||
if (path.startsWith('subsync.')) {
|
||||
return { category: 'integrations', section: topSection(path) };
|
||||
}
|
||||
if (path.startsWith('subtitleGeneration.')) {
|
||||
return { category: 'integrations', section: 'Japanese Subtitle Generation' };
|
||||
}
|
||||
if (path === 'stats.toggleKey' || path === 'stats.markWatchedKey') {
|
||||
return { category: 'input', section: 'Overlay Shortcuts' };
|
||||
}
|
||||
@@ -620,6 +623,7 @@ function subsectionForPath(path: string): string | undefined {
|
||||
leaf === 'openRuntimeOptions' ||
|
||||
leaf === 'openJimaku' ||
|
||||
leaf === 'openTsukihime' ||
|
||||
leaf === 'openSubtitleGeneration' ||
|
||||
leaf === 'openSessionHelp' ||
|
||||
leaf === 'openControllerSelect' ||
|
||||
leaf === 'openControllerDebug'
|
||||
@@ -728,7 +732,8 @@ function restartBehaviorForPath(path: string): ConfigSettingsRestartBehavior {
|
||||
pathStartsWith(path, 'notifications') ||
|
||||
path === 'youtube.primarySubLanguages' ||
|
||||
pathStartsWith(path, 'jimaku') ||
|
||||
pathStartsWith(path, 'subsync')
|
||||
pathStartsWith(path, 'subsync') ||
|
||||
pathStartsWith(path, 'subtitleGeneration')
|
||||
) {
|
||||
return 'hot-reload';
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ function makeShortcuts(overrides: Partial<ConfiguredShortcuts> = {}): Configured
|
||||
openRuntimeOptions: null,
|
||||
openJimaku: null,
|
||||
openTsukihime: null,
|
||||
openSubtitleGeneration: null,
|
||||
openSessionHelp: null,
|
||||
openControllerSelect: null,
|
||||
openControllerDebug: null,
|
||||
|
||||
@@ -24,6 +24,7 @@ function createShortcuts(overrides: Partial<ConfiguredShortcuts> = {}): Configur
|
||||
openRuntimeOptions: null,
|
||||
openJimaku: null,
|
||||
openTsukihime: null,
|
||||
openSubtitleGeneration: null,
|
||||
openSessionHelp: null,
|
||||
openControllerSelect: null,
|
||||
openControllerDebug: null,
|
||||
|
||||
@@ -41,6 +41,7 @@ function createDeps(overrides: Partial<SessionActionExecutorDeps> = {}) {
|
||||
openControllerDebug: () => calls.push('controller-debug'),
|
||||
openJimaku: () => calls.push('jimaku'),
|
||||
openTsukihime: () => calls.push('tsukihime'),
|
||||
openSubtitleGeneration: () => calls.push('subtitle-generation'),
|
||||
openYoutubeTrackPicker: () => {
|
||||
calls.push('youtube');
|
||||
},
|
||||
@@ -85,3 +86,9 @@ test('dispatchSessionAction opens the character dictionary manager', async () =>
|
||||
|
||||
assert.deepEqual(calls, ['character-dictionary-manager']);
|
||||
});
|
||||
|
||||
test('dispatchSessionAction opens subtitle generation without opening the sidebar', async () => {
|
||||
const { calls, deps } = createDeps();
|
||||
await dispatchSessionAction({ actionId: 'openSubtitleGeneration' }, deps);
|
||||
assert.deepEqual(calls, ['subtitle-generation']);
|
||||
});
|
||||
|
||||
@@ -25,6 +25,7 @@ export interface SessionActionExecutorDeps {
|
||||
openControllerDebug: () => void;
|
||||
openJimaku: () => void;
|
||||
openTsukihime: () => void;
|
||||
openSubtitleGeneration: () => void;
|
||||
openYoutubeTrackPicker: () => void | Promise<void>;
|
||||
openPlaylistBrowser: () => boolean | void | Promise<boolean | void>;
|
||||
replayCurrentSubtitle: () => void;
|
||||
@@ -119,6 +120,9 @@ export async function dispatchSessionAction(
|
||||
case 'openTsukihime':
|
||||
deps.openTsukihime();
|
||||
return;
|
||||
case 'openSubtitleGeneration':
|
||||
deps.openSubtitleGeneration();
|
||||
return;
|
||||
case 'openYoutubePicker':
|
||||
await deps.openYoutubeTrackPicker();
|
||||
return;
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { ConfiguredShortcuts } from '../utils/shortcut-config';
|
||||
import { DEFAULT_CONFIG, DEFAULT_KEYBINDINGS, SPECIAL_COMMANDS } from '../../config/definitions';
|
||||
import { resolveConfiguredShortcuts } from '../utils/shortcut-config';
|
||||
import { buildPluginSessionBindingsArtifact, compileSessionBindings } from './session-bindings';
|
||||
import { parseSessionActionDispatchRequest } from '../../shared/ipc/validators';
|
||||
|
||||
function createShortcuts(overrides: Partial<ConfiguredShortcuts> = {}): ConfiguredShortcuts {
|
||||
return {
|
||||
@@ -23,6 +24,7 @@ function createShortcuts(overrides: Partial<ConfiguredShortcuts> = {}): Configur
|
||||
openRuntimeOptions: null,
|
||||
openJimaku: null,
|
||||
openTsukihime: null,
|
||||
openSubtitleGeneration: null,
|
||||
openSessionHelp: null,
|
||||
openControllerSelect: null,
|
||||
openControllerDebug: null,
|
||||
@@ -37,6 +39,50 @@ function createKeybinding(key: string, command: Keybinding['command']): Keybindi
|
||||
return { key, command };
|
||||
}
|
||||
|
||||
test('subtitle generation shortcut compiles for overlay and mpv without conflicting with field grouping', () => {
|
||||
for (const platform of ['linux', 'darwin', 'win32'] as const) {
|
||||
const result = compileSessionBindings({
|
||||
shortcuts: resolveConfiguredShortcuts(DEFAULT_CONFIG, DEFAULT_CONFIG),
|
||||
keybindings: DEFAULT_KEYBINDINGS,
|
||||
platform,
|
||||
});
|
||||
const binding = result.bindings.find(
|
||||
(entry) =>
|
||||
entry.actionType === 'session-action' && entry.actionId === 'openSubtitleGeneration',
|
||||
);
|
||||
assert.ok(binding);
|
||||
assert.deepEqual(binding.key, { code: 'KeyG', modifiers: ['ctrl', 'shift'] });
|
||||
assert.equal(
|
||||
result.warnings.some(
|
||||
(warning) =>
|
||||
warning.path === 'shortcuts.openSubtitleGeneration' ||
|
||||
warning.conflictingPaths?.includes('shortcuts.openSubtitleGeneration'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.ok(
|
||||
result.bindings.some(
|
||||
(entry) =>
|
||||
entry.actionType === 'session-action' && entry.actionId === 'triggerFieldGrouping',
|
||||
),
|
||||
);
|
||||
const artifact = buildPluginSessionBindingsArtifact({
|
||||
bindings: [binding],
|
||||
warnings: [],
|
||||
numericSelectionTimeoutMs: 3000,
|
||||
});
|
||||
const pluginBinding = artifact.bindings[0];
|
||||
assert.ok(pluginBinding?.actionType === 'session-action');
|
||||
assert.deepEqual(pluginBinding.cliArgs, [
|
||||
'--session-action',
|
||||
'{"actionId":"openSubtitleGeneration"}',
|
||||
]);
|
||||
assert.deepEqual(parseSessionActionDispatchRequest({ actionId: 'openSubtitleGeneration' }), {
|
||||
actionId: 'openSubtitleGeneration',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('compileSessionBindings merges shortcuts and keybindings into one canonical list', () => {
|
||||
const result = compileSessionBindings({
|
||||
shortcuts: createShortcuts({
|
||||
|
||||
@@ -56,6 +56,7 @@ const SESSION_SHORTCUT_ACTIONS: Array<{
|
||||
{ key: 'openRuntimeOptions', actionId: 'openRuntimeOptions' },
|
||||
{ key: 'openJimaku', actionId: 'openJimaku' },
|
||||
{ key: 'openTsukihime', actionId: 'openTsukihime' },
|
||||
{ key: 'openSubtitleGeneration', actionId: 'openSubtitleGeneration' },
|
||||
{ key: 'openSessionHelp', actionId: 'openSessionHelp' },
|
||||
{ key: 'openControllerSelect', actionId: 'openControllerSelect' },
|
||||
{ key: 'openControllerDebug', actionId: 'openControllerDebug' },
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { appendSpeechChunkCues, splitSpeechPassages } from './subtitle-generation-chunks';
|
||||
|
||||
test('long coverage cuts at nearby speech starts instead of leaving a quiet lead-in', () => {
|
||||
const chunks = splitSpeechPassages(
|
||||
[{ startSeconds: 544.418, endSeconds: 581.581 }],
|
||||
[563.928],
|
||||
[555.07, 567.23, 579.91],
|
||||
);
|
||||
assert.deepEqual(chunks, [
|
||||
{ startSeconds: 544.418, endSeconds: 567.48 },
|
||||
{ startSeconds: 566.98, endSeconds: 581.581 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('short audible passages stay intact even with several detected speech starts', () => {
|
||||
assert.deepEqual(
|
||||
splitSpeechPassages(
|
||||
[{ startSeconds: 876.897, endSeconds: 897.812 }],
|
||||
[],
|
||||
[876.9, 881.15, 893.95, 897.99],
|
||||
),
|
||||
[{ startSeconds: 876.897, endSeconds: 897.812 }],
|
||||
);
|
||||
});
|
||||
|
||||
test('speech anchors outside retained coverage cannot extend a chunk across a silent gap', () => {
|
||||
const chunks = splitSpeechPassages(
|
||||
[{ startSeconds: 100, endSeconds: 142 }],
|
||||
[118],
|
||||
[90, 142, 144],
|
||||
);
|
||||
assert.deepEqual(chunks, [
|
||||
{ startSeconds: 100, endSeconds: 118.25 },
|
||||
{ startSeconds: 117.75, endSeconds: 138.25 },
|
||||
{ startSeconds: 137.75, endSeconds: 142 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('long speech splits near a pause with context on both sides and no lost audio', () => {
|
||||
assert.deepEqual(splitSpeechPassages([{ startSeconds: 100, endSeconds: 145 }], [105, 118, 137]), [
|
||||
{ startSeconds: 100, endSeconds: 118.25 },
|
||||
{ startSeconds: 117.75, endSeconds: 137.25 },
|
||||
{ startSeconds: 136.75, endSeconds: 145 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('uninterrupted speech retains overlapping context without crossing omitted gaps', () => {
|
||||
const chunks = splitSpeechPassages([
|
||||
{ startSeconds: 0, endSeconds: 60 },
|
||||
{ startSeconds: 100, endSeconds: 100.15 },
|
||||
]);
|
||||
assert.deepEqual(chunks, [
|
||||
{ startSeconds: 0, endSeconds: 20.25 },
|
||||
{ startSeconds: 19.75, endSeconds: 40.25 },
|
||||
{ startSeconds: 39.75, endSeconds: 60 },
|
||||
{ startSeconds: 100, endSeconds: 100.15 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('speech fitting one Whisper window stays intact instead of cutting a sentence at 20 seconds', () => {
|
||||
const passage = { startSeconds: 251.71, endSeconds: 275.01 };
|
||||
assert.deepEqual(splitSpeechPassages([passage], [271.327]), [passage]);
|
||||
});
|
||||
|
||||
test('chunk stitching ignores punctuation differences without merging separate repetitions', () => {
|
||||
const cues = [{ startTime: 19.7, endTime: 21.2, text: 'ありがとう' }];
|
||||
appendSpeechChunkCues(cues, [
|
||||
{ startTime: 19.8, endTime: 21.3, text: 'ありがとう。' },
|
||||
{ startTime: 22, endTime: 23, text: 'ありがとう！' },
|
||||
]);
|
||||
assert.deepEqual(cues, [
|
||||
{ startTime: 19.7, endTime: 21.3, text: 'ありがとう' },
|
||||
{ startTime: 22, endTime: 23, text: 'ありがとう！' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('chunk stitching removes matching overlap cues but retains repeated dialogue', () => {
|
||||
const cues = [{ startTime: 19.7, endTime: 20.2, text: 'はい' }];
|
||||
appendSpeechChunkCues(cues, [
|
||||
{ startTime: 19.8, endTime: 20.3, text: 'はい' },
|
||||
{ startTime: 21, endTime: 21.5, text: 'はい' },
|
||||
{ startTime: 21.4, endTime: 22, text: 'はい' },
|
||||
]);
|
||||
assert.deepEqual(cues, [
|
||||
{ startTime: 19.7, endTime: 20.3, text: 'はい' },
|
||||
{ startTime: 21, endTime: 21.5, text: 'はい' },
|
||||
{ startTime: 21.4, endTime: 22, text: 'はい' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('chunk stitching matches repeated text to the greatest overlap without leaving a duplicate', () => {
|
||||
const cues = [
|
||||
{ startTime: 10, endTime: 14, text: 'はい' },
|
||||
{ startTime: 13, endTime: 20, text: 'はい' },
|
||||
];
|
||||
appendSpeechChunkCues(cues, [
|
||||
{ startTime: 12, endTime: 21, text: 'はい' },
|
||||
{ startTime: 22, endTime: 23, text: 'はい' },
|
||||
]);
|
||||
assert.deepEqual(cues, [
|
||||
{ startTime: 10, endTime: 14, text: 'はい' },
|
||||
{ startTime: 12, endTime: 21, text: 'はい' },
|
||||
{ startTime: 22, endTime: 23, text: 'はい' },
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,87 @@
|
||||
import type { SubtitleCue } from './subtitle-cue-parser';
|
||||
import { SPEECH_PASSAGE_SECONDS, type SpeechPassage } from './subtitle-generation-speech';
|
||||
|
||||
const CHUNK_CONTEXT_SECONDS = 0.25;
|
||||
const WHISPER_WINDOW_SECONDS = 30;
|
||||
const PAUSE_SEARCH_SECONDS = 5;
|
||||
|
||||
// Prefer detected speech starts, then quiet pauses. Context stays inside retained audio.
|
||||
export function splitSpeechPassages(
|
||||
passages: readonly SpeechPassage[],
|
||||
pauses: readonly number[] = [],
|
||||
speechStarts: readonly number[] = [],
|
||||
): SpeechPassage[] {
|
||||
return passages.flatMap((passage) => {
|
||||
if (passage.endSeconds - passage.startSeconds <= WHISPER_WINDOW_SECONDS)
|
||||
return [{ ...passage }];
|
||||
const chunks: SpeechPassage[] = [];
|
||||
let boundary = passage.startSeconds;
|
||||
while (boundary < passage.endSeconds) {
|
||||
const target = boundary + SPEECH_PASSAGE_SECONDS;
|
||||
let end = Math.min(target, passage.endSeconds);
|
||||
if (target < passage.endSeconds) {
|
||||
// Starting in a long quiet lead-in can make Whisper place the next line
|
||||
// several seconds early. A nearby VAD start gives the next chunk an anchor.
|
||||
let nearestSpeechStart: number | undefined;
|
||||
for (const time of speechStarts) {
|
||||
if (
|
||||
time >= target - PAUSE_SEARCH_SECONDS &&
|
||||
time <= target + PAUSE_SEARCH_SECONDS &&
|
||||
time < passage.endSeconds &&
|
||||
(nearestSpeechStart === undefined ||
|
||||
Math.abs(time - target) < Math.abs(nearestSpeechStart - target))
|
||||
)
|
||||
nearestSpeechStart = time;
|
||||
}
|
||||
let latestPause: number | undefined;
|
||||
for (const time of pauses) {
|
||||
if (
|
||||
time >= target - PAUSE_SEARCH_SECONDS &&
|
||||
time <= target &&
|
||||
(latestPause === undefined || time > latestPause)
|
||||
)
|
||||
latestPause = time;
|
||||
}
|
||||
end = nearestSpeechStart ?? latestPause ?? end;
|
||||
}
|
||||
chunks.push({
|
||||
startSeconds: Math.max(passage.startSeconds, boundary - CHUNK_CONTEXT_SECONDS),
|
||||
endSeconds: Math.min(passage.endSeconds, end + CHUNK_CONTEXT_SECONDS),
|
||||
});
|
||||
boundary = end;
|
||||
}
|
||||
return chunks;
|
||||
});
|
||||
}
|
||||
|
||||
// Deduplicate only matching text substantially overlapping cues from earlier chunks.
|
||||
// Repeated words within the current chunk or at separate times remain separate.
|
||||
export function appendSpeechChunkCues(cues: SubtitleCue[], incoming: readonly SubtitleCue[]): void {
|
||||
const previousCount = cues.length;
|
||||
const matched = new Set<SubtitleCue>();
|
||||
for (const cue of incoming) {
|
||||
const text = cue.text.replace(/[\s\p{P}]+/gu, '');
|
||||
let duplicate: SubtitleCue | undefined;
|
||||
let greatestOverlap = 0;
|
||||
for (const [index, previous] of cues.entries()) {
|
||||
if (index >= previousCount) break;
|
||||
if (!text || matched.has(previous) || previous.text.replace(/[\s\p{P}]+/gu, '') !== text)
|
||||
continue;
|
||||
const overlap =
|
||||
Math.min(previous.endTime, cue.endTime) - Math.max(previous.startTime, cue.startTime);
|
||||
const shorterDuration = Math.min(
|
||||
previous.endTime - previous.startTime,
|
||||
cue.endTime - cue.startTime,
|
||||
);
|
||||
if (overlap > greatestOverlap && overlap >= shorterDuration / 2) {
|
||||
duplicate = previous;
|
||||
greatestOverlap = overlap;
|
||||
}
|
||||
}
|
||||
if (duplicate) {
|
||||
duplicate.startTime = Math.min(duplicate.startTime, cue.startTime);
|
||||
duplicate.endTime = Math.max(duplicate.endTime, cue.endTime);
|
||||
matched.add(duplicate);
|
||||
} else cues.push({ ...cue });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { findAudiblePassages, mergeSpeechPassages } from './subtitle-generation-coverage';
|
||||
|
||||
async function analyze(lines: string[], progress = 'out_time_us=20000000\n') {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'subtitle-coverage-test-'));
|
||||
try {
|
||||
const ffmpegPath = path.join(directory, 'ffmpeg');
|
||||
await writeFile(
|
||||
ffmpegPath,
|
||||
`#!${process.execPath}
|
||||
process.stderr.write(${JSON.stringify(lines.join('\n') + '\n')});
|
||||
process.stdout.write(${JSON.stringify(progress)});
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
return await findAudiblePassages({ ffmpegPath, wavPath: 'audio.wav' });
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test('audible coverage retains the full timeline when there is no confident silence', async () => {
|
||||
assert.deepEqual(await analyze([]), [{ startSeconds: 0, endSeconds: 20 }]);
|
||||
});
|
||||
|
||||
test('audible coverage omits silence while padding nearby audio without exceeding the timeline', async () => {
|
||||
assert.deepEqual(
|
||||
await analyze([
|
||||
'[silencedetect] silence_start: 0',
|
||||
'[silencedetect] silence_end: 2 | silence_duration: 2',
|
||||
'[silencedetect] silence_start: 8',
|
||||
'[silencedetect] silence_end: 12 | silence_duration: 4',
|
||||
'[silencedetect] silence_start: 18',
|
||||
]),
|
||||
[
|
||||
{ startSeconds: 1.65, endSeconds: 8.35 },
|
||||
{ startSeconds: 11.65, endSeconds: 18.35 },
|
||||
],
|
||||
);
|
||||
assert.deepEqual(await analyze(['[silencedetect] silence_end: 10.5 | silence_duration: 0.5']), [
|
||||
{ startSeconds: 0, endSeconds: 20 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('entirely silent audio has no audible passages', async () => {
|
||||
assert.deepEqual(await analyze(['[silencedetect] silence_start: 0']), []);
|
||||
assert.deepEqual(await analyze(['[silencedetect] silence_end: 20 | silence_duration: 20']), []);
|
||||
});
|
||||
|
||||
test('missing analysis duration fails instead of silently dropping audio', async () => {
|
||||
await assert.rejects(analyze([], ''), /valid duration/);
|
||||
});
|
||||
|
||||
test('merging coverage preserves quiet VAD speech and does not mutate detector results', () => {
|
||||
const speech = [{ startSeconds: 10, endSeconds: 11 }];
|
||||
assert.deepEqual(
|
||||
mergeSpeechPassages([
|
||||
...speech,
|
||||
{ startSeconds: 0, endSeconds: 5 },
|
||||
{ startSeconds: 4, endSeconds: 8 },
|
||||
{ startSeconds: 11, endSeconds: 12 },
|
||||
]),
|
||||
[
|
||||
{ startSeconds: 0, endSeconds: 8 },
|
||||
{ startSeconds: 10, endSeconds: 12 },
|
||||
],
|
||||
);
|
||||
assert.deepEqual(speech, [{ startSeconds: 10, endSeconds: 11 }]);
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
|
||||
import type { SpeechPassage } from './subtitle-generation-speech';
|
||||
|
||||
const AUDIO_PADDING_SECONDS = 0.35;
|
||||
|
||||
export function mergeSpeechPassages(passages: readonly SpeechPassage[]): SpeechPassage[] {
|
||||
const merged: SpeechPassage[] = [];
|
||||
for (const passage of [...passages].sort((a, b) => a.startSeconds - b.startSeconds)) {
|
||||
const previous = merged.at(-1);
|
||||
if (previous && passage.startSeconds <= previous.endSeconds)
|
||||
previous.endSeconds = Math.max(previous.endSeconds, passage.endSeconds);
|
||||
else merged.push({ ...passage });
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
// VAD rejection is not proof of silence. Preserve audible gaps for Whisper to evaluate.
|
||||
export async function findAudiblePassages(input: {
|
||||
ffmpegPath: string;
|
||||
wavPath: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<SpeechPassage[]> {
|
||||
const silences: SpeechPassage[] = [];
|
||||
let duration = 0;
|
||||
let trailingSilence: number | undefined;
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.ffmpegPath,
|
||||
args: [
|
||||
'-nostdin',
|
||||
'-hide_banner',
|
||||
'-nostats',
|
||||
'-i',
|
||||
input.wavPath,
|
||||
'-af',
|
||||
'silencedetect=noise=-50dB:d=0.5',
|
||||
'-progress',
|
||||
'pipe:1',
|
||||
'-f',
|
||||
'null',
|
||||
'-',
|
||||
],
|
||||
signal: input.signal,
|
||||
onLine: (line) => {
|
||||
const progress = /^out_time_us=(\d+)$/.exec(line);
|
||||
if (progress) duration = Math.max(duration, Number(progress[1]) / 1_000_000);
|
||||
const start = /silence_start: (\S+)/.exec(line);
|
||||
if (start && Number.isFinite(Number(start[1]))) trailingSilence = Number(start[1]);
|
||||
const end = /silence_end: (\S+) \| silence_duration: (\S+)/.exec(line);
|
||||
if (!end) return;
|
||||
const endSeconds = Number(end[1]);
|
||||
const length = Number(end[2]);
|
||||
if (Number.isFinite(endSeconds) && Number.isFinite(length) && length > 0) {
|
||||
silences.push({ startSeconds: Math.max(0, endSeconds - length), endSeconds });
|
||||
trailingSilence = undefined;
|
||||
}
|
||||
},
|
||||
});
|
||||
if (!Number.isFinite(duration) || duration <= 0)
|
||||
throw new Error('Audio analysis did not report a valid duration.');
|
||||
if (trailingSilence !== undefined)
|
||||
silences.push({ startSeconds: trailingSilence, endSeconds: duration });
|
||||
|
||||
const audible: SpeechPassage[] = [];
|
||||
let cursor = 0;
|
||||
for (const silence of mergeSpeechPassages(silences)) {
|
||||
if (cursor >= duration) break;
|
||||
if (silence.startSeconds > cursor)
|
||||
audible.push({ startSeconds: cursor, endSeconds: Math.min(duration, silence.startSeconds) });
|
||||
cursor = Math.max(cursor, silence.endSeconds);
|
||||
}
|
||||
if (cursor < duration) audible.push({ startSeconds: cursor, endSeconds: duration });
|
||||
return mergeSpeechPassages(
|
||||
audible.map((passage) => ({
|
||||
startSeconds: Math.max(0, passage.startSeconds - AUDIO_PADDING_SECONDS),
|
||||
endSeconds: Math.min(duration, passage.endSeconds + AUDIO_PADDING_SECONDS),
|
||||
})),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import { access, readFile, rm } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type {
|
||||
SubtitleGenerationConfig,
|
||||
SubtitleGenerationProgress,
|
||||
} from '../../shared/subtitle-generation';
|
||||
import { expandSubtitleGenerationPath } from './subtitle-generation-files';
|
||||
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
|
||||
import type { SubtitleGenerationToolPaths } from './subtitle-generation-tools';
|
||||
import { formatTimestamp } from './subtitle-generation-srt';
|
||||
import {
|
||||
parseSpeechPassages,
|
||||
speechPassageCues,
|
||||
SPEECH_PASSAGE_SECONDS,
|
||||
} from './subtitle-generation-speech';
|
||||
import type { SubtitleCue } from './subtitle-cue-parser';
|
||||
import { appendSpeechChunkCues, splitSpeechPassages } from './subtitle-generation-chunks';
|
||||
import { findSpeechPauses } from './subtitle-generation-pauses';
|
||||
import { findAudiblePassages, mergeSpeechPassages } from './subtitle-generation-coverage';
|
||||
|
||||
export async function transcribeSubtitleDialogue(input: {
|
||||
config: SubtitleGenerationConfig;
|
||||
tools: SubtitleGenerationToolPaths & { vad: string };
|
||||
modelPath: string;
|
||||
wavPath: string;
|
||||
directory: string;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
const vadModelPath = expandSubtitleGenerationPath(input.config.vadModelPath);
|
||||
await access(vadModelPath, constants.R_OK);
|
||||
input.onProgress?.({ stage: 'transcribe', percent: 0, message: 'Finding spoken dialogue...' });
|
||||
const segmentLines: string[] = [];
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.tools.vad,
|
||||
args: [
|
||||
'-f',
|
||||
input.wavPath,
|
||||
'-vm',
|
||||
vadModelPath,
|
||||
'-t',
|
||||
String(input.config.threads),
|
||||
'-vt',
|
||||
'0.3',
|
||||
'--vad-min-speech-duration-ms',
|
||||
'100',
|
||||
'--vad-min-silence-duration-ms',
|
||||
'500',
|
||||
'-vp',
|
||||
'350',
|
||||
'-vmsd',
|
||||
String(SPEECH_PASSAGE_SECONDS),
|
||||
'-np',
|
||||
],
|
||||
signal: input.signal,
|
||||
// Capture structured result lines separately from the bounded process log.
|
||||
onLine: (line) => {
|
||||
if (line.startsWith('Detected ') || line.startsWith('Speech segment '))
|
||||
segmentLines.push(line);
|
||||
},
|
||||
});
|
||||
const speech = parseSpeechPassages(segmentLines.join('\n'));
|
||||
input.onProgress?.({ stage: 'transcribe', percent: 0, message: 'Checking audio coverage...' });
|
||||
const audible = await findAudiblePassages({
|
||||
ffmpegPath: input.tools.ffmpeg,
|
||||
wavPath: input.wavPath,
|
||||
signal: input.signal,
|
||||
});
|
||||
const detected = mergeSpeechPassages([...speech, ...audible]);
|
||||
if (detected.length === 0) throw new Error('No spoken dialogue detected.');
|
||||
const pauses = detected.some(
|
||||
(passage) => passage.endSeconds - passage.startSeconds > SPEECH_PASSAGE_SECONDS,
|
||||
)
|
||||
? await findSpeechPauses({
|
||||
ffmpegPath: input.tools.ffmpeg,
|
||||
wavPath: input.wavPath,
|
||||
signal: input.signal,
|
||||
})
|
||||
: [];
|
||||
const passages = splitSpeechPassages(
|
||||
detected,
|
||||
pauses,
|
||||
speech.map((passage) => passage.startSeconds),
|
||||
);
|
||||
const cues: SubtitleCue[] = [];
|
||||
for (const [index, passage] of passages.entries()) {
|
||||
const base = path.join(input.directory, `speech-${index}`);
|
||||
input.onProgress?.({
|
||||
stage: 'transcribe',
|
||||
percent: Math.floor((index / passages.length) * 100),
|
||||
message: `Transcribing dialogue passage ${index + 1} of ${passages.length}...`,
|
||||
});
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.tools.ffmpeg,
|
||||
args: [
|
||||
'-nostdin',
|
||||
'-hide_banner',
|
||||
'-loglevel',
|
||||
'error',
|
||||
'-ss',
|
||||
String(passage.startSeconds),
|
||||
'-i',
|
||||
input.wavPath,
|
||||
'-t',
|
||||
String(passage.endSeconds - passage.startSeconds),
|
||||
'-ac',
|
||||
'1',
|
||||
'-ar',
|
||||
'16000',
|
||||
'-c:a',
|
||||
'pcm_s16le',
|
||||
`${base}.wav`,
|
||||
],
|
||||
signal: input.signal,
|
||||
});
|
||||
// -mc 0 limits text context, but does not isolate decoder state across input files.
|
||||
// A fresh process prevents earlier passages from corrupting later transcriptions.
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.tools.whisper,
|
||||
args: [
|
||||
'-m',
|
||||
input.modelPath,
|
||||
'-l',
|
||||
'ja',
|
||||
'-t',
|
||||
String(input.config.threads),
|
||||
'-mc',
|
||||
'0',
|
||||
'-sns',
|
||||
'-osrt',
|
||||
'-f',
|
||||
`${base}.wav`,
|
||||
'-of',
|
||||
base,
|
||||
],
|
||||
signal: input.signal,
|
||||
});
|
||||
input.signal?.throwIfAborted();
|
||||
appendSpeechChunkCues(cues, speechPassageCues(await readFile(`${base}.srt`, 'utf8'), passage));
|
||||
await rm(`${base}.wav`);
|
||||
}
|
||||
if (cues.length === 0) throw new Error('Whisper recognized no dialogue in the detected speech.');
|
||||
return cues
|
||||
.sort((a, b) => a.startTime - b.startTime || a.endTime - b.endTime)
|
||||
.map(
|
||||
(cue, index) =>
|
||||
`${index + 1}\n${formatTimestamp(cue.startTime * 1000)} --> ${formatTimestamp(cue.endTime * 1000)}\n${cue.text}\n`,
|
||||
)
|
||||
.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { downloadSubtitleGenerationArtifact } from './subtitle-generation-download';
|
||||
import {
|
||||
downloadSubtitleGenerationVadModel,
|
||||
resolveSubtitleGenerationVadModel,
|
||||
} from './subtitle-generation-vad-model';
|
||||
import { DEFAULT_SUBTITLE_GENERATION_CONFIG } from '../../shared/subtitle-generation';
|
||||
|
||||
test('verified model publication preserves existing files and cleans up failed or cancelled downloads', async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'subtitle-model-download-'));
|
||||
const originalFetch = globalThis.fetch;
|
||||
const bytes = new TextEncoder().encode('fixture model');
|
||||
const input = {
|
||||
url: 'https://example.test/model',
|
||||
size: bytes.length,
|
||||
sha256: createHash('sha256').update(bytes).digest('hex'),
|
||||
destination: path.join(directory, 'model.bin'),
|
||||
label: 'test model',
|
||||
};
|
||||
try {
|
||||
globalThis.fetch = Object.assign(async () => new Response(bytes), originalFetch);
|
||||
await downloadSubtitleGenerationArtifact(input);
|
||||
assert.equal(await readFile(input.destination, 'utf8'), 'fixture model');
|
||||
await assert.rejects(downloadSubtitleGenerationArtifact(input), /EEXIST/);
|
||||
await assert.rejects(
|
||||
downloadSubtitleGenerationArtifact({
|
||||
...input,
|
||||
destination: path.join(directory, 'bad.bin'),
|
||||
sha256: 'wrong',
|
||||
}),
|
||||
/integrity/,
|
||||
);
|
||||
const controller = new AbortController();
|
||||
await assert.rejects(
|
||||
downloadSubtitleGenerationArtifact({
|
||||
...input,
|
||||
destination: path.join(directory, 'cancelled.bin'),
|
||||
signal: controller.signal,
|
||||
onProgress: ({ percent }) => {
|
||||
if (percent === 99) controller.abort();
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.deepEqual(await readdir(directory), ['model.bin']);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('VAD setup recognizes existing paths and never replaces an invalid external model', async () => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'subtitle-vad-model-'));
|
||||
try {
|
||||
const config = { ...DEFAULT_SUBTITLE_GENERATION_CONFIG };
|
||||
assert.equal((await resolveSubtitleGenerationVadModel(config, directory)).kind, 'missing');
|
||||
config.vadModelPath = path.join(directory, 'external.bin');
|
||||
await assert.rejects(
|
||||
downloadSubtitleGenerationVadModel({ config, modelDirectory: directory }),
|
||||
/Cannot read/,
|
||||
);
|
||||
await writeFile(config.vadModelPath, 'external model');
|
||||
assert.equal((await resolveSubtitleGenerationVadModel(config, directory)).kind, 'external');
|
||||
assert.equal(
|
||||
await downloadSubtitleGenerationVadModel({ config, modelDirectory: directory }),
|
||||
config.vadModelPath,
|
||||
);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdir, mkdtemp, open, rm } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { publishSubtitleGenerationFile } from './subtitle-generation-files';
|
||||
import type { SubtitleGenerationProgress } from '../../shared/subtitle-generation';
|
||||
|
||||
export async function downloadSubtitleGenerationArtifact(input: {
|
||||
url: string;
|
||||
size: number;
|
||||
sha256: string;
|
||||
destination: string;
|
||||
label: string;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
input.signal?.throwIfAborted();
|
||||
await mkdir(path.dirname(input.destination), { recursive: true });
|
||||
const temporaryDirectory = await mkdtemp(
|
||||
path.join(path.dirname(input.destination), '.download-'),
|
||||
);
|
||||
const temporaryPath = path.join(temporaryDirectory, 'model.bin');
|
||||
input.onProgress?.({ stage: 'download', percent: 0, message: `Downloading ${input.label}...` });
|
||||
try {
|
||||
const response = await fetch(input.url, { signal: input.signal });
|
||||
if (!response.ok || !response.body) {
|
||||
throw new Error(`Model download failed: HTTP ${response.status}`);
|
||||
}
|
||||
const file = await open(temporaryPath, 'wx');
|
||||
const reader = response.body.getReader();
|
||||
const digest = createHash('sha256');
|
||||
let received = 0;
|
||||
let previousPercent = -1;
|
||||
try {
|
||||
while (true) {
|
||||
input.signal?.throwIfAborted();
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
received += value.byteLength;
|
||||
if (received > input.size) throw new Error('Downloaded model exceeds expected size.');
|
||||
digest.update(value);
|
||||
await file.writeFile(value);
|
||||
const percent = Math.min(99, Math.floor((received / input.size) * 100));
|
||||
if (percent !== previousPercent) {
|
||||
previousPercent = percent;
|
||||
input.onProgress?.({
|
||||
stage: 'download',
|
||||
percent,
|
||||
message: `Downloading ${input.label}...`,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (received !== input.size || digest.digest('hex') !== input.sha256) {
|
||||
throw new Error('Downloaded model failed integrity verification. Try downloading again.');
|
||||
}
|
||||
await file.sync();
|
||||
} finally {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
await file.close();
|
||||
}
|
||||
input.signal?.throwIfAborted();
|
||||
await publishSubtitleGenerationFile(temporaryPath, input.destination);
|
||||
input.onProgress?.({ stage: 'download', percent: 100, message: `${input.label} is ready.` });
|
||||
return input.destination;
|
||||
} finally {
|
||||
await rm(temporaryDirectory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { constants } from 'node:fs';
|
||||
import { copyFile, link } from 'node:fs/promises';
|
||||
import { homedir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
export function expandSubtitleGenerationPath(value: string): string {
|
||||
if (value === '~') return homedir();
|
||||
if (value.startsWith('~/') || value.startsWith('~\\'))
|
||||
return path.join(homedir(), value.slice(2));
|
||||
return value;
|
||||
}
|
||||
|
||||
// Prefer atomic publication. Filesystems without hard links still get exclusive creation.
|
||||
export async function publishSubtitleGenerationFile(
|
||||
source: string,
|
||||
destination: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await link(source, destination);
|
||||
} catch (error) {
|
||||
if (
|
||||
!(error instanceof Error) ||
|
||||
!('code' in error) ||
|
||||
(error.code !== 'ENOTSUP' &&
|
||||
error.code !== 'EOPNOTSUPP' &&
|
||||
error.code !== 'EPERM' &&
|
||||
error.code !== 'EXDEV')
|
||||
)
|
||||
throw error;
|
||||
await copyFile(source, destination, constants.COPYFILE_EXCL);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { access, open, stat } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { getSubtitleGenerationModel } from '../../shared/subtitle-generation-model-catalog';
|
||||
import { expandSubtitleGenerationPath } from './subtitle-generation-files';
|
||||
import { downloadSubtitleGenerationArtifact } from './subtitle-generation-download';
|
||||
import type {
|
||||
SubtitleGenerationConfig,
|
||||
SubtitleGenerationModelStatus,
|
||||
SubtitleGenerationProgress,
|
||||
} from '../../shared/subtitle-generation';
|
||||
|
||||
export function isMissingFile(error: unknown): boolean {
|
||||
return error instanceof Error && 'code' in error && error.code === 'ENOENT';
|
||||
}
|
||||
|
||||
async function modelCompatibilityError(modelPath: string): Promise<string | undefined> {
|
||||
const file = await open(modelPath, 'r');
|
||||
try {
|
||||
// whisper_model_load reads GGML magic, then n_vocab. is_multilingual uses n_vocab >= 51865.
|
||||
const header = Buffer.alloc(8);
|
||||
const { bytesRead } = await file.read(header, 0, header.length, 0);
|
||||
if (
|
||||
bytesRead !== header.length ||
|
||||
header.readUInt32LE(0) !== 0x67676d6c ||
|
||||
header.readInt32LE(4) <= 0
|
||||
) {
|
||||
return 'Unsupported model format. Choose a whisper.cpp GGML .bin model.';
|
||||
}
|
||||
if (header.readInt32LE(4) < 51865) {
|
||||
return 'This Whisper model is English-only. Japanese subtitle generation requires a multilingual model.';
|
||||
}
|
||||
return undefined;
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveSubtitleGenerationModel(
|
||||
config: SubtitleGenerationConfig,
|
||||
modelDirectory: string,
|
||||
): Promise<SubtitleGenerationModelStatus> {
|
||||
const external = config.modelPath.trim();
|
||||
const modelPath = external
|
||||
? path.resolve(expandSubtitleGenerationPath(external))
|
||||
: path.resolve(modelDirectory, `ggml-${config.managedModel}.bin`);
|
||||
try {
|
||||
const info = await stat(modelPath);
|
||||
if (!info.isFile() || info.size === 0) {
|
||||
return { kind: 'invalid', path: modelPath, message: 'Model must be a nonempty file.' };
|
||||
}
|
||||
await access(modelPath, constants.R_OK);
|
||||
if (!external && info.size !== getSubtitleGenerationModel(config.managedModel).size) {
|
||||
return { kind: 'invalid', path: modelPath, message: 'Managed model has an unexpected size.' };
|
||||
}
|
||||
const compatibilityError = await modelCompatibilityError(modelPath);
|
||||
if (compatibilityError)
|
||||
return { kind: 'invalid', path: modelPath, message: compatibilityError };
|
||||
return { kind: external ? 'external' : 'managed', path: modelPath };
|
||||
} catch (error) {
|
||||
if (!external && isMissingFile(error)) return { kind: 'missing', path: modelPath };
|
||||
return {
|
||||
kind: 'invalid',
|
||||
path: modelPath,
|
||||
message: `Cannot read model: ${error instanceof Error ? error.message : String(error)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function downloadSubtitleGenerationModel(input: {
|
||||
config: SubtitleGenerationConfig;
|
||||
modelDirectory: string;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
input.signal?.throwIfAborted();
|
||||
const current = await resolveSubtitleGenerationModel(input.config, input.modelDirectory);
|
||||
if (current.kind === 'external' || current.kind === 'managed') return current.path;
|
||||
if (current.kind === 'invalid') throw new Error(current.message);
|
||||
const model = getSubtitleGenerationModel(input.config.managedModel);
|
||||
return downloadSubtitleGenerationArtifact({
|
||||
url: `https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-${input.config.managedModel}.bin`,
|
||||
size: model.size,
|
||||
sha256: model.sha256,
|
||||
destination: current.path,
|
||||
label: 'Whisper model',
|
||||
onProgress: input.onProgress,
|
||||
signal: input.signal,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
|
||||
|
||||
// Each completed silencedetect line contains both the end and duration of a quiet interval.
|
||||
export async function findSpeechPauses(input: {
|
||||
ffmpegPath: string;
|
||||
wavPath: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<number[]> {
|
||||
const pauses: number[] = [];
|
||||
await runSubtitleGenerationProcess({
|
||||
command: input.ffmpegPath,
|
||||
args: [
|
||||
'-nostdin',
|
||||
'-hide_banner',
|
||||
'-nostats',
|
||||
'-i',
|
||||
input.wavPath,
|
||||
'-af',
|
||||
'silencedetect=noise=-35dB:d=0.12',
|
||||
'-f',
|
||||
'null',
|
||||
'-',
|
||||
],
|
||||
signal: input.signal,
|
||||
onLine: (line) => {
|
||||
const match = /silence_end: (\S+) \| silence_duration: (\S+)/.exec(line);
|
||||
if (!match) return;
|
||||
const end = Number(match[1]);
|
||||
const duration = Number(match[2]);
|
||||
if (Number.isFinite(end) && Number.isFinite(duration) && duration > 0 && end >= duration)
|
||||
pauses.push(end - duration / 2);
|
||||
},
|
||||
});
|
||||
return pauses.sort((a, b) => a - b);
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { expandSubtitleGenerationPath } from './subtitle-generation-files';
|
||||
|
||||
const OUTPUT_LIMIT = 64 * 1024;
|
||||
|
||||
// Keep partial lines between chunks: ffmpeg and whisper both report progress on stderr.
|
||||
export function runSubtitleGenerationProcess(input: {
|
||||
command: string;
|
||||
args: string[];
|
||||
signal?: AbortSignal;
|
||||
onLine?: (line: string) => void;
|
||||
}): Promise<string> {
|
||||
input.signal?.throwIfAborted();
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(expandSubtitleGenerationPath(input.command), input.args, {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let killTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
const abort = () => {
|
||||
child.kill('SIGTERM');
|
||||
killTimer = setTimeout(() => child.kill('SIGKILL'), 2000);
|
||||
killTimer.unref();
|
||||
};
|
||||
input.signal?.addEventListener('abort', abort, { once: true });
|
||||
if (input.signal?.aborted) abort();
|
||||
const cleanup = () => {
|
||||
input.signal?.removeEventListener('abort', abort);
|
||||
clearTimeout(killTimer);
|
||||
};
|
||||
for (const [stream, isStdout] of [
|
||||
[child.stdout, true],
|
||||
[child.stderr, false],
|
||||
] as const) {
|
||||
let pending = '';
|
||||
stream.setEncoding('utf8');
|
||||
stream.on('data', (chunk: string) => {
|
||||
if (isStdout) stdout = (stdout + chunk).slice(-OUTPUT_LIMIT);
|
||||
else stderr = (stderr + chunk).slice(-OUTPUT_LIMIT);
|
||||
const lines = (pending + chunk).split(/[\r\n]/);
|
||||
pending = (lines.pop() ?? '').slice(-OUTPUT_LIMIT);
|
||||
for (const line of lines) input.onLine?.(line);
|
||||
});
|
||||
stream.on('end', () => {
|
||||
if (pending) input.onLine?.(pending);
|
||||
});
|
||||
}
|
||||
child.once('error', (error) => {
|
||||
cleanup();
|
||||
reject(
|
||||
new Error(
|
||||
'code' in error && error.code === 'ENOENT'
|
||||
? `${input.command} was not found. Install it or set its path under subtitleGeneration in Settings.`
|
||||
: `Could not run ${input.command}: ${error.message}`,
|
||||
),
|
||||
);
|
||||
});
|
||||
child.once('close', (code) => {
|
||||
cleanup();
|
||||
if (input.signal?.aborted) reject(new Error('Subtitle generation cancelled.'));
|
||||
else if (code !== 0) {
|
||||
reject(new Error(`${input.command} exited with status ${code}: ${stderr.trim()}`));
|
||||
} else resolve(stdout);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { parseSpeechPassages, speechPassageCues } from './subtitle-generation-speech';
|
||||
|
||||
test('speech passages convert centiseconds, group nearby speech, and retain long gaps', () => {
|
||||
assert.deepEqual(
|
||||
parseSpeechPassages(
|
||||
[
|
||||
'Detected 4 speech segments:',
|
||||
'Speech segment 0: start = 0.00, end = 300.00',
|
||||
'Speech segment 1: start = 350.00, end = 900.00',
|
||||
'Speech segment 2: start = 10000.00, end = 11900.00',
|
||||
'Speech segment 3: start = 11950.00, end = 12500.00',
|
||||
].join('\n'),
|
||||
),
|
||||
[
|
||||
{ startSeconds: 0, endSeconds: 9 },
|
||||
{ startSeconds: 100, endSeconds: 119 },
|
||||
{ startSeconds: 119.5, endSeconds: 125 },
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('speech passages retain long merged detector segments for pause-aware splitting', () => {
|
||||
assert.deepEqual(
|
||||
parseSpeechPassages(
|
||||
[
|
||||
'Detected 3 speech segments:',
|
||||
'Speech segment 0: start = 33714.00, end = 36756.00',
|
||||
'Speech segment 1: start = 40000.00, end = 46000.00',
|
||||
'Speech segment 2: start = 50000.00, end = 50100.00',
|
||||
].join('\n'),
|
||||
),
|
||||
[
|
||||
{ startSeconds: 337.14, endSeconds: 367.56 },
|
||||
{ startSeconds: 400, endSeconds: 460 },
|
||||
{ startSeconds: 500, endSeconds: 501 },
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('speech detector distinguishes no speech from missing, malformed, or truncated output', () => {
|
||||
assert.deepEqual(parseSpeechPassages('Detected 0 speech segments:'), []);
|
||||
assert.deepEqual(
|
||||
parseSpeechPassages(
|
||||
'Detected 1 speech segments:\nSpeech segment 0: start = 79896.00, end = 82218.00',
|
||||
),
|
||||
[{ startSeconds: 798.96, endSeconds: 822.18 }],
|
||||
);
|
||||
for (const output of [
|
||||
'',
|
||||
'Detected 1 speech segments:',
|
||||
'Detected 1 speech segments:\nSpeech segment 1: start = 100.00, end = 200.00',
|
||||
'Detected 1 speech segments:\nSpeech segment 0: start = 200.00, end = 100.00',
|
||||
'Detected 1 speech segments:\nSpeech segment 0: start = NaN, end = 100.00',
|
||||
'Detected 2 speech segments:\nSpeech segment 0: start = 0.00, end = 200.00\nSpeech segment 1: start = 100.00, end = 300.00',
|
||||
])
|
||||
assert.throws(() => parseSpeechPassages(output), /Speech detector/);
|
||||
});
|
||||
|
||||
test('passage cue times cannot extend into omitted audio or accumulate offsets', () => {
|
||||
const srt =
|
||||
'1\n00:00:00,000 --> 00:00:01,000\nはい\n\n2\n00:00:01,000 --> 00:01:40,000\nはい\n\n3\n00:01:41,000 --> 00:01:42,000\n幻覚\n';
|
||||
assert.deepEqual(
|
||||
speechPassageCues(srt, { startSeconds: 1200.25, endSeconds: 1203.75 }).map(
|
||||
({ startTime, endTime, text }) => ({ startTime, endTime, text }),
|
||||
),
|
||||
[
|
||||
{ startTime: 1200.25, endTime: 1201.25, text: 'はい' },
|
||||
{ startTime: 1201.25, endTime: 1203.75, text: 'はい' },
|
||||
],
|
||||
);
|
||||
assert.deepEqual(speechPassageCues('', { startSeconds: 0, endSeconds: 1 }), []);
|
||||
assert.throws(
|
||||
() => speechPassageCues('broken SRT', { startSeconds: 0, endSeconds: 1 }),
|
||||
/malformed/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
import { parseSrtCues, type SubtitleCue } from './subtitle-cue-parser';
|
||||
|
||||
export interface SpeechPassage {
|
||||
startSeconds: number;
|
||||
endSeconds: number;
|
||||
}
|
||||
|
||||
export const SPEECH_PASSAGE_SECONDS = 20;
|
||||
|
||||
// The standalone whisper.cpp detector reports centiseconds, unlike its diagnostic logs.
|
||||
export function parseSpeechPassages(output: string): SpeechPassage[] {
|
||||
const count = /^Detected (\d+) speech segments:$/m.exec(output);
|
||||
if (!count) throw new Error('Speech detector did not report its segment count.');
|
||||
const passages: SpeechPassage[] = [];
|
||||
for (const line of output.split(/\r?\n/)) {
|
||||
if (!line.startsWith('Speech segment ')) continue;
|
||||
const match = /^Speech segment (\d+): start = (\d+(?:\.\d+)?), end = (\d+(?:\.\d+)?)$/.exec(
|
||||
line,
|
||||
);
|
||||
if (!match) throw new Error('Speech detector returned a malformed segment.');
|
||||
const index = Number(match[1]);
|
||||
const startSeconds = Number(match[2]) / 100;
|
||||
const endSeconds = Number(match[3]) / 100;
|
||||
if (
|
||||
index !== passages.length ||
|
||||
!Number.isFinite(startSeconds) ||
|
||||
!Number.isFinite(endSeconds) ||
|
||||
endSeconds <= startSeconds ||
|
||||
startSeconds < (passages.at(-1)?.endSeconds ?? 0)
|
||||
) {
|
||||
throw new Error('Speech detector returned unordered or invalid segment timing.');
|
||||
}
|
||||
passages.push({ startSeconds, endSeconds });
|
||||
}
|
||||
if (passages.length !== Number(count[1]))
|
||||
throw new Error('Speech detector output is incomplete.');
|
||||
|
||||
const grouped: SpeechPassage[] = [];
|
||||
for (const passage of passages) {
|
||||
const previous = grouped.at(-1);
|
||||
if (
|
||||
previous &&
|
||||
passage.startSeconds - previous.endSeconds <= 1 &&
|
||||
passage.endSeconds - previous.startSeconds <= SPEECH_PASSAGE_SECONDS
|
||||
) {
|
||||
previous.endSeconds = passage.endSeconds;
|
||||
} else grouped.push({ ...passage });
|
||||
}
|
||||
return grouped;
|
||||
}
|
||||
|
||||
// Clamp to the audio actually supplied to Whisper. A cue cannot cross an omitted gap.
|
||||
export function speechPassageCues(srt: string, passage: SpeechPassage): SubtitleCue[] {
|
||||
const duration = passage.endSeconds - passage.startSeconds;
|
||||
const cues = parseSrtCues(srt);
|
||||
if (srt.trim() && cues.length === 0)
|
||||
throw new Error('Whisper returned malformed subtitles for a speech passage.');
|
||||
return cues.flatMap((cue) => {
|
||||
const start = Math.max(0, cue.startTime);
|
||||
const end = Math.min(duration, cue.endTime);
|
||||
if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start) return [];
|
||||
return [
|
||||
{ ...cue, startTime: passage.startSeconds + start, endTime: passage.startSeconds + end },
|
||||
];
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export function formatTimestamp(milliseconds: number): string {
|
||||
const rounded = Math.max(0, Math.round(milliseconds));
|
||||
const hours = Math.floor(rounded / 3600000);
|
||||
const minutes = Math.floor((rounded % 3600000) / 60000);
|
||||
const seconds = Math.floor((rounded % 60000) / 1000);
|
||||
return `${String(hours).padStart(2, '0')}:${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')},${String(rounded % 1000).padStart(3, '0')}`;
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { DEFAULT_SUBTITLE_GENERATION_CONFIG } from '../../shared/subtitle-generation';
|
||||
import {
|
||||
requireSubtitleGenerationTools,
|
||||
resolveSubtitleGenerationTools,
|
||||
} from './subtitle-generation-tools';
|
||||
|
||||
async function fixture(run: (directory: string) => Promise<void>) {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'subtitle-generation-tools-'));
|
||||
try {
|
||||
await run(directory);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function executable(directory: string, name: string): Promise<string> {
|
||||
const file = path.join(directory, name);
|
||||
await writeFile(file, '#!/bin/sh\n', { mode: 0o755 });
|
||||
return file;
|
||||
}
|
||||
|
||||
test('tools resolve from PATH, honor overrides, and only require the detector in dialogue mode', () =>
|
||||
fixture(async (directory) => {
|
||||
const bin = path.join(directory, 'bin');
|
||||
await mkdir(bin);
|
||||
for (const name of ['ffmpeg', 'ffprobe', 'whisper-cli']) await executable(bin, name);
|
||||
const detector = await executable(bin, 'vad-speech-segments');
|
||||
const customWhisper = await executable(directory, 'my-whisper');
|
||||
await writeFile(path.join(directory, 'not-executable'), '', { mode: 0o644 });
|
||||
const env = { PATH: bin };
|
||||
|
||||
const found = await resolveSubtitleGenerationTools(DEFAULT_SUBTITLE_GENERATION_CONFIG, env);
|
||||
assert.deepEqual(found, {
|
||||
ffmpeg: { kind: 'found', path: path.join(bin, 'ffmpeg') },
|
||||
ffprobe: { kind: 'found', path: path.join(bin, 'ffprobe') },
|
||||
whisper: { kind: 'found', path: path.join(bin, 'whisper-cli') },
|
||||
vad: null,
|
||||
});
|
||||
assert.equal(requireSubtitleGenerationTools(found).vad, null);
|
||||
|
||||
const dialogue = await resolveSubtitleGenerationTools(
|
||||
{ ...DEFAULT_SUBTITLE_GENERATION_CONFIG, vadModelPath: '/models/vad.bin' },
|
||||
env,
|
||||
);
|
||||
assert.deepEqual(dialogue.vad, { kind: 'found', path: detector });
|
||||
|
||||
const overridden = await resolveSubtitleGenerationTools(
|
||||
{
|
||||
...DEFAULT_SUBTITLE_GENERATION_CONFIG,
|
||||
whisperPath: customWhisper,
|
||||
ffmpegPath: path.join(directory, 'not-executable'),
|
||||
},
|
||||
env,
|
||||
);
|
||||
assert.deepEqual(overridden.whisper, { kind: 'found', path: customWhisper });
|
||||
assert.equal(overridden.ffmpeg.kind, 'missing');
|
||||
assert.throws(
|
||||
() => requireSubtitleGenerationTools(overridden),
|
||||
/not-executable \(subtitleGeneration\.ffmpegPath\) is not an executable file/,
|
||||
);
|
||||
}));
|
||||
|
||||
test('missing tools name the executable, the installer, and the setting', () =>
|
||||
fixture(async (directory) => {
|
||||
const tools = await resolveSubtitleGenerationTools(
|
||||
{ ...DEFAULT_SUBTITLE_GENERATION_CONFIG, vadModelPath: '/models/vad.bin' },
|
||||
{ PATH: directory },
|
||||
);
|
||||
assert.deepEqual(tools.whisper, {
|
||||
kind: 'missing',
|
||||
message:
|
||||
'whisper-cli was not found on PATH. Install whisper.cpp or set subtitleGeneration.whisperPath in Settings.',
|
||||
});
|
||||
assert.deepEqual(tools.vad, {
|
||||
kind: 'missing',
|
||||
message:
|
||||
"whisper-vad-speech-segments was not found on PATH. Install whisper.cpp's speech segment detector or set subtitleGeneration.vadPath in Settings.",
|
||||
});
|
||||
assert.throws(() => requireSubtitleGenerationTools(tools), /ffmpeg was not found on PATH/);
|
||||
}));
|
||||
@@ -0,0 +1,125 @@
|
||||
import { access, stat } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type {
|
||||
SubtitleGenerationConfig,
|
||||
SubtitleGenerationToolStatus,
|
||||
SubtitleGenerationTools,
|
||||
} from '../../shared/subtitle-generation';
|
||||
import { expandSubtitleGenerationPath } from './subtitle-generation-files';
|
||||
|
||||
/** Executable paths ready to spawn. `vad` is null when dialogue mode is off. */
|
||||
export interface SubtitleGenerationToolPaths {
|
||||
ffmpeg: string;
|
||||
ffprobe: string;
|
||||
whisper: string;
|
||||
vad: string | null;
|
||||
}
|
||||
|
||||
const TOOL_LOOKUPS = {
|
||||
ffmpeg: { setting: 'ffmpegPath', names: ['ffmpeg'], install: 'Install FFmpeg' },
|
||||
ffprobe: { setting: 'ffprobePath', names: ['ffprobe'], install: 'Install FFmpeg' },
|
||||
whisper: { setting: 'whisperPath', names: ['whisper-cli'], install: 'Install whisper.cpp' },
|
||||
vad: {
|
||||
setting: 'vadPath',
|
||||
names: ['whisper-vad-speech-segments', 'vad-speech-segments'],
|
||||
install: "Install whisper.cpp's speech segment detector",
|
||||
},
|
||||
} as const;
|
||||
|
||||
async function isExecutableFile(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
if (!(await stat(filePath)).isFile()) return false;
|
||||
await access(filePath, constants.X_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function executableNames(name: string, env: NodeJS.ProcessEnv): string[] {
|
||||
if (process.platform !== 'win32' || path.extname(name)) return [name];
|
||||
const extensions = (env.PATHEXT ?? '.EXE;.CMD;.BAT')
|
||||
.split(';')
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
return [name, ...extensions.map((extension) => `${name}${extension}`)];
|
||||
}
|
||||
|
||||
async function findOnPath(names: readonly string[], env: NodeJS.ProcessEnv): Promise<string> {
|
||||
const directories = (env.PATH ?? '')
|
||||
.split(path.delimiter)
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
for (const directory of directories) {
|
||||
for (const name of names) {
|
||||
for (const candidate of executableNames(name, env)) {
|
||||
const filePath = path.join(directory, candidate);
|
||||
if (await isExecutableFile(filePath)) return filePath;
|
||||
}
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
async function resolveTool(
|
||||
tool: keyof typeof TOOL_LOOKUPS,
|
||||
config: SubtitleGenerationConfig,
|
||||
env: NodeJS.ProcessEnv,
|
||||
): Promise<SubtitleGenerationToolStatus> {
|
||||
const lookup = TOOL_LOOKUPS[tool];
|
||||
const override = config[lookup.setting].trim();
|
||||
if (override) {
|
||||
const expanded = expandSubtitleGenerationPath(override);
|
||||
const found =
|
||||
path.dirname(expanded) === '.'
|
||||
? await findOnPath([expanded], env)
|
||||
: (await isExecutableFile(expanded))
|
||||
? path.resolve(expanded)
|
||||
: '';
|
||||
return found
|
||||
? { kind: 'found', path: found }
|
||||
: {
|
||||
kind: 'missing',
|
||||
message: `${override} (subtitleGeneration.${lookup.setting}) is not an executable file.`,
|
||||
};
|
||||
}
|
||||
const found = await findOnPath(lookup.names, env);
|
||||
return found
|
||||
? { kind: 'found', path: found }
|
||||
: {
|
||||
kind: 'missing',
|
||||
message: `${lookup.names[0]} was not found on PATH. ${lookup.install} or set subtitleGeneration.${lookup.setting} in Settings.`,
|
||||
};
|
||||
}
|
||||
|
||||
/** Locate every executable a generation run needs, before any model download or audio work. */
|
||||
export async function resolveSubtitleGenerationTools(
|
||||
config: SubtitleGenerationConfig,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<SubtitleGenerationTools> {
|
||||
const [ffmpeg, ffprobe, whisper, vad] = await Promise.all([
|
||||
resolveTool('ffmpeg', config, env),
|
||||
resolveTool('ffprobe', config, env),
|
||||
resolveTool('whisper', config, env),
|
||||
config.vadModelPath.trim() ? resolveTool('vad', config, env) : null,
|
||||
]);
|
||||
return { ffmpeg, ffprobe, whisper, vad };
|
||||
}
|
||||
|
||||
function foundPath(tool: SubtitleGenerationToolStatus): string {
|
||||
if (tool.kind === 'missing') throw new Error(tool.message);
|
||||
return tool.path;
|
||||
}
|
||||
|
||||
/** Throw the first missing tool's message, otherwise narrow to spawnable paths. */
|
||||
export function requireSubtitleGenerationTools(
|
||||
tools: SubtitleGenerationTools,
|
||||
): SubtitleGenerationToolPaths {
|
||||
return {
|
||||
ffmpeg: foundPath(tools.ffmpeg),
|
||||
ffprobe: foundPath(tools.ffprobe),
|
||||
whisper: foundPath(tools.whisper),
|
||||
vad: tools.vad ? foundPath(tools.vad) : null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { access, stat } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type {
|
||||
SubtitleGenerationConfig,
|
||||
SubtitleGenerationModelStatus,
|
||||
SubtitleGenerationProgress,
|
||||
} from '../../shared/subtitle-generation';
|
||||
import { SUBTITLE_GENERATION_VAD_MODEL } from '../../shared/subtitle-generation-vad-model';
|
||||
import { expandSubtitleGenerationPath } from './subtitle-generation-files';
|
||||
import { isMissingFile } from './subtitle-generation-models';
|
||||
import { downloadSubtitleGenerationArtifact } from './subtitle-generation-download';
|
||||
|
||||
export async function resolveSubtitleGenerationVadModel(
|
||||
config: SubtitleGenerationConfig,
|
||||
modelDirectory: string,
|
||||
): Promise<SubtitleGenerationModelStatus> {
|
||||
const external = config.vadModelPath.trim();
|
||||
const modelPath = external
|
||||
? path.resolve(expandSubtitleGenerationPath(external))
|
||||
: path.resolve(modelDirectory, SUBTITLE_GENERATION_VAD_MODEL.filename);
|
||||
try {
|
||||
const info = await stat(modelPath);
|
||||
if (
|
||||
!info.isFile() ||
|
||||
info.size === 0 ||
|
||||
(!external && info.size !== SUBTITLE_GENERATION_VAD_MODEL.size)
|
||||
)
|
||||
return {
|
||||
kind: 'invalid',
|
||||
path: modelPath,
|
||||
message: 'Speech detection model has an invalid size.',
|
||||
};
|
||||
await access(modelPath, constants.R_OK);
|
||||
return { kind: external ? 'external' : 'managed', path: modelPath };
|
||||
} catch (error) {
|
||||
if (!external && isMissingFile(error)) return { kind: 'missing', path: modelPath };
|
||||
return {
|
||||
kind: 'invalid',
|
||||
path: modelPath,
|
||||
message: `Cannot read speech detection model: ${error instanceof Error ? error.message : String(error)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function downloadSubtitleGenerationVadModel(input: {
|
||||
config: SubtitleGenerationConfig;
|
||||
modelDirectory: string;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
input.signal?.throwIfAborted();
|
||||
const current = await resolveSubtitleGenerationVadModel(input.config, input.modelDirectory);
|
||||
if (current.kind === 'invalid') throw new Error(current.message);
|
||||
if (current.kind !== 'missing') return current.path;
|
||||
return downloadSubtitleGenerationArtifact({
|
||||
...SUBTITLE_GENERATION_VAD_MODEL,
|
||||
destination: current.path,
|
||||
label: 'Silero speech detection model',
|
||||
onProgress: input.onProgress,
|
||||
signal: input.signal,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { constants } from 'node:fs';
|
||||
import { access, chmod, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
DEFAULT_SUBTITLE_GENERATION_CONFIG,
|
||||
resolveSubtitleGenerationConfig,
|
||||
type SubtitleGenerationProgress,
|
||||
} from '../../shared/subtitle-generation';
|
||||
import {
|
||||
downloadSubtitleGenerationModel,
|
||||
ensureWritableDirectory,
|
||||
generateJapaneseSubtitles,
|
||||
resolveSubtitleGenerationModel,
|
||||
} from './subtitle-generation';
|
||||
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
|
||||
|
||||
async function fixture(run: (directory: string) => Promise<void>) {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), 'subtitle-generation-test-'));
|
||||
try {
|
||||
await run(directory);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function executable(directory: string, name: string, body: string) {
|
||||
const file = path.join(directory, name);
|
||||
await writeFile(file, `#!${process.execPath}\n${body}`, { mode: 0o755 });
|
||||
return file;
|
||||
}
|
||||
|
||||
function modelHeader(vocabularySize = 51865): Buffer {
|
||||
const header = Buffer.alloc(8);
|
||||
header.writeUInt32LE(0x67676d6c, 0);
|
||||
header.writeInt32LE(vocabularySize, 4);
|
||||
return header;
|
||||
}
|
||||
|
||||
async function generationFixture(directory: string) {
|
||||
const modelPath = path.join(directory, 'external.bin');
|
||||
const mediaPath = path.join(directory, 'episode.mkv');
|
||||
const callsPath = path.join(directory, 'calls.jsonl');
|
||||
await writeFile(modelPath, modelHeader());
|
||||
await writeFile(mediaPath, 'local media');
|
||||
const record = `require('node:fs').appendFileSync(${JSON.stringify(callsPath)}, JSON.stringify(process.argv.slice(2)) + '\\n');`;
|
||||
const ffprobePath = await executable(
|
||||
directory,
|
||||
'ffprobe',
|
||||
`${record}\nprocess.stdout.write(JSON.stringify({streams: [{index:1,codec_type:'audio',start_time:'10',tags:{language:'eng'}},{index:3,codec_type:'audio',start_time:'12.5',duration:'20',tags:{language:'jpn'}}],format:{start_time:'10',duration:'25'}}));`,
|
||||
);
|
||||
const ffmpegPath = await executable(
|
||||
directory,
|
||||
'ffmpeg',
|
||||
`${record}
|
||||
if (process.argv.at(-1) === '-') {
|
||||
process.stderr.write('[silencedetect] silence_end: 25 | silence_duration: 25\\n');
|
||||
process.stdout.write('out_time_us=25000000\\nprogress=end\\n');
|
||||
} else {
|
||||
require('node:fs').writeFileSync(process.argv.at(-1), 'wav');
|
||||
process.stdout.write('out_time_');
|
||||
setTimeout(() => process.stdout.write('us=10000000\\nprogress=end\\n'), 10);
|
||||
}`,
|
||||
);
|
||||
const whisperPath = await executable(
|
||||
directory,
|
||||
'whisper-cli',
|
||||
`${record}\nconst args=process.argv.slice(2); require('node:fs').writeFileSync(args[args.indexOf('-of')+1]+'.srt', '1\\n00:00:01,000 --> 00:00:02,000\\nこんにちは\\n'); process.stderr.write('whisper_print_progress_callback: progress = '); setTimeout(() => process.stderr.write('55%\\n'), 10);`,
|
||||
);
|
||||
return {
|
||||
config: {
|
||||
...DEFAULT_SUBTITLE_GENERATION_CONFIG,
|
||||
modelPath,
|
||||
ffprobePath,
|
||||
ffmpegPath,
|
||||
whisperPath,
|
||||
},
|
||||
mediaPath,
|
||||
modelDirectory: path.join(directory, 'models'),
|
||||
callsPath,
|
||||
};
|
||||
}
|
||||
|
||||
test('config parser accepts supported models and rejects unsafe threads and wrong field types', () => {
|
||||
const warnings: string[] = [];
|
||||
const result = resolveSubtitleGenerationConfig(
|
||||
{ modelPath: '/tmp/whisper.bin', threads: 0, whisperPath: 42, managedModel: 'large-v3-turbo' },
|
||||
(key) => warnings.push(key),
|
||||
);
|
||||
assert.equal(result.modelPath, '/tmp/whisper.bin');
|
||||
assert.equal(result.managedModel, 'large-v3-turbo');
|
||||
assert.equal(result.threads, DEFAULT_SUBTITLE_GENERATION_CONFIG.threads);
|
||||
assert.deepEqual(warnings, ['whisperPath', 'threads']);
|
||||
});
|
||||
|
||||
test('external model path wins and invalid external models never fall back to download', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
assert.deepEqual(await resolveSubtitleGenerationModel(input.config, input.modelDirectory), {
|
||||
kind: 'external',
|
||||
path: input.config.modelPath,
|
||||
});
|
||||
const invalid = { ...input.config, modelPath: path.join(directory, 'missing.bin') };
|
||||
assert.equal(
|
||||
(await resolveSubtitleGenerationModel(invalid, input.modelDirectory)).kind,
|
||||
'invalid',
|
||||
);
|
||||
await assert.rejects(
|
||||
downloadSubtitleGenerationModel({ ...input, config: invalid }),
|
||||
/Cannot read model/,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await resolveSubtitleGenerationModel(
|
||||
{ ...input.config, modelPath: '' },
|
||||
input.modelDirectory,
|
||||
)
|
||||
).kind,
|
||||
'missing',
|
||||
);
|
||||
}));
|
||||
|
||||
test('English-only and incompatible external models are rejected before transcription', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
await writeFile(input.config.modelPath, modelHeader(51864));
|
||||
const englishOnly = await resolveSubtitleGenerationModel(input.config, input.modelDirectory);
|
||||
assert.equal(englishOnly.kind, 'invalid');
|
||||
assert.ok('message' in englishOnly);
|
||||
assert.match(englishOnly.message, /English-only/);
|
||||
await assert.rejects(generateJapaneseSubtitles(input), /requires a multilingual model/);
|
||||
await writeFile(input.config.modelPath, 'not a GGML model');
|
||||
await assert.rejects(generateJapaneseSubtitles(input), /Unsupported model format/);
|
||||
await writeFile(input.config.modelPath, modelHeader().subarray(0, 4));
|
||||
await assert.rejects(generateJapaneseSubtitles(input), /Unsupported model format/);
|
||||
await assert.rejects(readFile(input.callsPath), /ENOENT/);
|
||||
}));
|
||||
|
||||
test('generation picks Japanese audio, restores timeline offsets, reports split progress, and preserves existing output', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const existing = path.join(directory, 'episode.ja.generated.srt');
|
||||
await writeFile(existing, 'user subtitles');
|
||||
const progress: SubtitleGenerationProgress[] = [];
|
||||
const result = await generateJapaneseSubtitles({
|
||||
...input,
|
||||
onProgress: (event) => progress.push(event),
|
||||
});
|
||||
assert.equal(result, path.join(directory, 'episode.ja.generated.1.srt'));
|
||||
assert.equal(await readFile(existing, 'utf8'), 'user subtitles');
|
||||
assert.match(await readFile(result, 'utf8'), /00:00:03,500 --> 00:00:04,500\nこんにちは/);
|
||||
const calls = (await readFile(input.callsPath, 'utf8'))
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((line): unknown => JSON.parse(line));
|
||||
assert.ok(Array.isArray(calls[1]));
|
||||
assert.ok(calls[1].includes('0:3'));
|
||||
assert.ok(Array.isArray(calls[2]));
|
||||
assert.ok(calls[2].includes('ja'));
|
||||
assert.ok(calls[2].includes('-osrt'));
|
||||
assert.ok(progress.some((event) => event.stage === 'extract' && event.percent === 50));
|
||||
assert.ok(progress.some((event) => event.stage === 'transcribe' && event.percent === 55));
|
||||
assert.deepEqual(
|
||||
(await readdir(directory)).filter((file) => file.startsWith('.subminer-')),
|
||||
[],
|
||||
);
|
||||
}));
|
||||
|
||||
test('explicit audio stream and output path are respected without overwriting existing files', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const outputPath = path.join(directory, 'chosen.srt');
|
||||
const result = await generateJapaneseSubtitles({ ...input, audioStreamIndex: 1, outputPath });
|
||||
assert.equal(result, outputPath);
|
||||
assert.match(await readFile(result, 'utf8'), /00:00:01,000 --> 00:00:02,000/);
|
||||
await assert.rejects(generateJapaneseSubtitles({ ...input, outputPath }), /already exists/);
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({ ...input, audioStreamIndex: 99 }),
|
||||
/stream 99 was not found/,
|
||||
);
|
||||
}));
|
||||
|
||||
test('dialogue generation isolates Whisper state between passages and preserves media timing', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const vadModelPath = path.join(directory, 'vad.bin');
|
||||
await writeFile(vadModelPath, 'speech detector model');
|
||||
const vadPath = await executable(
|
||||
directory,
|
||||
'vad',
|
||||
"process.stdout.write('Detected 2 speech segments:\\nSpeech segment 0: start = 1000.00, end = 1100.00\\nSpeech segment 1: start = 10000.00, end = 10100.00\\n');",
|
||||
);
|
||||
const whisperPath = await executable(
|
||||
directory,
|
||||
'dialogue-whisper',
|
||||
`const args = process.argv.slice(2);
|
||||
let files = 0;
|
||||
for (let i = 0; i < args.length; i++) if (args[i] === '-of') {
|
||||
// Reproduce a decoder that degenerates when reused for another audio file.
|
||||
const text = files++ === 0 ? 'はい' : 'お' + 'ぉ'.repeat(40) + 'ぇ'.repeat(178);
|
||||
require('node:fs').writeFileSync(args[i + 1] + '.srt', '1\\n00:00:00,000 --> 00:01:39,000\\n' + text + '\\n');
|
||||
}`,
|
||||
);
|
||||
const output = await generateJapaneseSubtitles({
|
||||
...input,
|
||||
config: { ...input.config, vadModelPath, vadPath, whisperPath },
|
||||
});
|
||||
assert.equal(
|
||||
await readFile(output, 'utf8'),
|
||||
'1\n00:00:12,500 --> 00:00:13,500\nはい\n\n2\n00:01:42,500 --> 00:01:43,500\nはい\n',
|
||||
);
|
||||
}));
|
||||
|
||||
test('dialogue generation uses quiet pauses and stitches overlapping chunks on the media timeline', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const vadModelPath = path.join(directory, 'vad.bin');
|
||||
await writeFile(vadModelPath, 'speech detector model');
|
||||
const vadPath = await executable(
|
||||
directory,
|
||||
'vad',
|
||||
`const assert = require('node:assert/strict');
|
||||
const args = process.argv.slice(2);
|
||||
assert.equal(args[args.indexOf('--vad-min-speech-duration-ms') + 1], '100');
|
||||
assert.equal(args[args.indexOf('-vp') + 1], '350');
|
||||
process.stdout.write('Detected 1 speech segments:\\nSpeech segment 0: start = 1000.00, end = 4500.00\\n');`,
|
||||
);
|
||||
const ffmpegPath = await executable(
|
||||
directory,
|
||||
'pause-ffmpeg',
|
||||
`const args = process.argv.slice(2);
|
||||
if (args.includes('silencedetect=noise=-50dB:d=0.5')) {
|
||||
process.stderr.write('[silencedetect] silence_end: 45 | silence_duration: 45\\n');
|
||||
process.stdout.write('out_time_us=45000000\\n');
|
||||
} else if (args.includes('-af')) {
|
||||
process.stderr.write('[silencedetect] silence_end: 28.1 | silence_duration: 0.2\\n');
|
||||
} else {
|
||||
require('node:fs').writeFileSync(args.at(-1), 'wav');
|
||||
}`,
|
||||
);
|
||||
const whisperPath = await executable(
|
||||
directory,
|
||||
'overlap-whisper',
|
||||
`const args = process.argv.slice(2);
|
||||
for (let i = 0; i < args.length; i++) if (args[i] === '-of') {
|
||||
const time = args[i + 1].endsWith('speech-0')
|
||||
? '00:00:17,800 --> 00:00:18,250'
|
||||
: '00:00:00,100 --> 00:00:00,650';
|
||||
require('node:fs').writeFileSync(args[i + 1] + '.srt', '1\\n' + time + '\\nはい\\n');
|
||||
}`,
|
||||
);
|
||||
const output = await generateJapaneseSubtitles({
|
||||
...input,
|
||||
config: { ...input.config, vadModelPath, vadPath, ffmpegPath, whisperPath },
|
||||
});
|
||||
assert.equal(await readFile(output, 'utf8'), '1\n00:00:30,300 --> 00:00:30,900\nはい\n');
|
||||
}));
|
||||
|
||||
test('silent audio with no detected speech stops generation without transcription', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const vadModelPath = path.join(directory, 'vad.bin');
|
||||
await writeFile(vadModelPath, 'speech detector model');
|
||||
const vadPath = await executable(
|
||||
directory,
|
||||
'vad',
|
||||
"process.stdout.write('Detected 0 speech segments:\\n');",
|
||||
);
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({ ...input, config: { ...input.config, vadModelPath, vadPath } }),
|
||||
/No spoken dialogue detected/,
|
||||
);
|
||||
assert.equal((await readFile(input.callsPath, 'utf8')).trim().split('\n').length, 3);
|
||||
assert.deepEqual(
|
||||
(await readdir(directory)).filter((file) => file.endsWith('.srt')),
|
||||
[],
|
||||
);
|
||||
}));
|
||||
|
||||
test('dialogue generation retains audible audio rejected by VAD', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const vadModelPath = path.join(directory, 'vad.bin');
|
||||
await writeFile(vadModelPath, 'speech detector model');
|
||||
const vadPath = await executable(
|
||||
directory,
|
||||
'vad',
|
||||
"process.stdout.write('Detected 0 speech segments:\\n');",
|
||||
);
|
||||
const ffmpegPath = await executable(
|
||||
directory,
|
||||
'audible-ffmpeg',
|
||||
`
|
||||
const args = process.argv.slice(2);
|
||||
if (args.at(-1) === '-') {
|
||||
process.stdout.write('out_time_us=19000000\\nprogress=end\\n');
|
||||
} else {
|
||||
require('node:fs').writeFileSync(args.at(-1), 'wav');
|
||||
}`,
|
||||
);
|
||||
const output = await generateJapaneseSubtitles({
|
||||
...input,
|
||||
config: { ...input.config, vadModelPath, vadPath, ffmpegPath },
|
||||
});
|
||||
assert.match(await readFile(output, 'utf8'), /00:00:03,500 --> 00:00:04,500\nこんにちは/);
|
||||
}));
|
||||
|
||||
test('empty executable paths find tools on PATH and explicit overrides take precedence', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const previousPath = process.env.PATH;
|
||||
process.env.PATH = directory;
|
||||
try {
|
||||
const config = {
|
||||
...DEFAULT_SUBTITLE_GENERATION_CONFIG,
|
||||
modelPath: input.config.modelPath,
|
||||
};
|
||||
const result = await generateJapaneseSubtitles({ ...input, config });
|
||||
assert.match(await readFile(result, 'utf8'), /こんにちは/);
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({
|
||||
...input,
|
||||
config: { ...config, ffprobePath: path.join(directory, 'missing-override') },
|
||||
}),
|
||||
/missing-override \(subtitleGeneration\.ffprobePath\) is not an executable file/,
|
||||
);
|
||||
} finally {
|
||||
if (previousPath === undefined) delete process.env.PATH;
|
||||
else process.env.PATH = previousPath;
|
||||
}
|
||||
}));
|
||||
|
||||
test('generation rejects remote media, missing models, and missing tools before starting a subprocess', () =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({ ...input, mediaPath: 'https://example.com/movie.mkv' }),
|
||||
/local media file/,
|
||||
);
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({ ...input, config: { ...input.config, modelPath: '' } }),
|
||||
/No Whisper model found/,
|
||||
);
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({
|
||||
...input,
|
||||
config: {
|
||||
...input.config,
|
||||
vadModelPath: path.join(directory, 'vad.bin'),
|
||||
vadPath: path.join(directory, 'missing-detector'),
|
||||
},
|
||||
}),
|
||||
/missing-detector \(subtitleGeneration\.vadPath\) is not an executable file/,
|
||||
);
|
||||
await assert.rejects(readFile(input.callsPath), /ENOENT/);
|
||||
}));
|
||||
|
||||
test(
|
||||
'directory permission preflight rejects a write-only destination',
|
||||
{
|
||||
skip: process.platform === 'win32' || process.getuid?.() === 0,
|
||||
},
|
||||
() =>
|
||||
fixture(async (directory) => {
|
||||
const writeOnly = path.join(directory, 'write-only');
|
||||
await mkdir(writeOnly);
|
||||
try {
|
||||
await chmod(writeOnly, 0o200);
|
||||
await access(writeOnly, constants.W_OK);
|
||||
await assert.rejects(ensureWritableDirectory(writeOnly), /write-only is not writable/);
|
||||
} finally {
|
||||
await chmod(writeOnly, 0o755);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
test(
|
||||
'generation rejects an unwritable destination before extracting audio',
|
||||
{
|
||||
skip: process.platform === 'win32' || process.getuid?.() === 0,
|
||||
},
|
||||
() =>
|
||||
fixture(async (directory) => {
|
||||
const input = await generationFixture(directory);
|
||||
const readOnly = path.join(directory, 'read-only');
|
||||
await mkdir(readOnly, { mode: 0o555 });
|
||||
try {
|
||||
await assert.rejects(
|
||||
generateJapaneseSubtitles({ ...input, outputPath: path.join(readOnly, 'out.srt') }),
|
||||
/read-only is not writable/,
|
||||
);
|
||||
await assert.rejects(readFile(input.callsPath), /ENOENT/);
|
||||
} finally {
|
||||
await chmod(readOnly, 0o755);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
test('process cancellation terminates work and bounds diagnostic output', () =>
|
||||
fixture(async (directory) => {
|
||||
const slow = await executable(
|
||||
directory,
|
||||
'slow',
|
||||
"process.stdout.write('ready\\n');setInterval(()=>{},1000);",
|
||||
);
|
||||
const controller = new AbortController();
|
||||
await assert.rejects(
|
||||
runSubtitleGenerationProcess({
|
||||
command: slow,
|
||||
args: [],
|
||||
signal: controller.signal,
|
||||
onLine: () => controller.abort(),
|
||||
}),
|
||||
/cancelled/,
|
||||
);
|
||||
const failed = await executable(
|
||||
directory,
|
||||
'failed',
|
||||
"process.stderr.write('x'.repeat(100000));process.exitCode=7;",
|
||||
);
|
||||
await assert.rejects(
|
||||
runSubtitleGenerationProcess({ command: failed, args: [] }),
|
||||
(error: unknown) =>
|
||||
error instanceof Error &&
|
||||
error.message.length < 66000 &&
|
||||
error.message.includes('status 7'),
|
||||
);
|
||||
}));
|
||||
|
||||
test('download uses the pinned model revision and removes files that fail integrity', () =>
|
||||
fixture(async (directory) => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = Object.assign(async (request: string | URL | Request) => {
|
||||
assert.equal(
|
||||
request,
|
||||
'https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-small.bin',
|
||||
);
|
||||
return new Response('not a model');
|
||||
}, originalFetch);
|
||||
try {
|
||||
await assert.rejects(
|
||||
downloadSubtitleGenerationModel({
|
||||
config: DEFAULT_SUBTITLE_GENERATION_CONFIG,
|
||||
modelDirectory: directory,
|
||||
}),
|
||||
/integrity verification/,
|
||||
);
|
||||
assert.deepEqual(await readdir(directory), []);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
}));
|
||||
@@ -0,0 +1,315 @@
|
||||
import { access, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import type {
|
||||
SubtitleGenerationConfig,
|
||||
SubtitleGenerationProgress,
|
||||
} from '../../shared/subtitle-generation';
|
||||
import { isMissingFile, resolveSubtitleGenerationModel } from './subtitle-generation-models';
|
||||
import { runSubtitleGenerationProcess } from './subtitle-generation-process';
|
||||
import { publishSubtitleGenerationFile } from './subtitle-generation-files';
|
||||
import { formatTimestamp } from './subtitle-generation-srt';
|
||||
import { transcribeSubtitleDialogue } from './subtitle-generation-dialogue';
|
||||
import {
|
||||
requireSubtitleGenerationTools,
|
||||
resolveSubtitleGenerationTools,
|
||||
} from './subtitle-generation-tools';
|
||||
|
||||
export {
|
||||
downloadSubtitleGenerationModel,
|
||||
resolveSubtitleGenerationModel,
|
||||
} from './subtitle-generation-models';
|
||||
export { resolveSubtitleGenerationTools } from './subtitle-generation-tools';
|
||||
|
||||
function numericTime(value: unknown): number | undefined {
|
||||
if (typeof value !== 'number' && typeof value !== 'string') return undefined;
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) ? number : undefined;
|
||||
}
|
||||
|
||||
function parseAudioProbe(raw: string, selectedIndex: number | undefined) {
|
||||
const value: unknown = JSON.parse(raw);
|
||||
if (
|
||||
typeof value !== 'object' ||
|
||||
value === null ||
|
||||
!('streams' in value) ||
|
||||
!Array.isArray(value.streams)
|
||||
) {
|
||||
throw new Error('ffprobe did not return media streams.');
|
||||
}
|
||||
const streams = value.streams.flatMap((stream: unknown) => {
|
||||
if (
|
||||
typeof stream !== 'object' ||
|
||||
stream === null ||
|
||||
!('codec_type' in stream) ||
|
||||
stream.codec_type !== 'audio' ||
|
||||
!('index' in stream) ||
|
||||
typeof stream.index !== 'number' ||
|
||||
!Number.isInteger(stream.index) ||
|
||||
stream.index < 0
|
||||
)
|
||||
return [];
|
||||
const tags = 'tags' in stream ? stream.tags : undefined;
|
||||
const language =
|
||||
typeof tags === 'object' && tags !== null && 'language' in tags ? tags.language : undefined;
|
||||
return [
|
||||
{
|
||||
index: stream.index,
|
||||
start: 'start_time' in stream ? numericTime(stream.start_time) : undefined,
|
||||
duration: 'duration' in stream ? numericTime(stream.duration) : undefined,
|
||||
japanese: language === 'ja' || language === 'jpn',
|
||||
},
|
||||
];
|
||||
});
|
||||
const selected =
|
||||
selectedIndex === undefined
|
||||
? (streams.find((stream) => stream.japanese) ?? streams[0])
|
||||
: streams.find((stream) => stream.index === selectedIndex);
|
||||
if (!selected)
|
||||
throw new Error(
|
||||
selectedIndex === undefined
|
||||
? 'No audio track found.'
|
||||
: `Audio stream ${selectedIndex} was not found.`,
|
||||
);
|
||||
const format = 'format' in value ? value.format : undefined;
|
||||
const formatStart =
|
||||
typeof format === 'object' && format !== null && 'start_time' in format
|
||||
? (numericTime(format.start_time) ?? 0)
|
||||
: 0;
|
||||
const duration =
|
||||
selected.duration ??
|
||||
(typeof format === 'object' && format !== null && 'duration' in format
|
||||
? numericTime(format.duration)
|
||||
: undefined);
|
||||
// mpv rebases media timestamps to the container start. Extraction rebases the selected audio.
|
||||
return { index: selected.index, offset: (selected.start ?? formatStart) - formatStart, duration };
|
||||
}
|
||||
|
||||
function shiftSubtitleTimestamps(srt: string, offsetSeconds: number): string {
|
||||
let cueCount = 0;
|
||||
const result = srt.replace(
|
||||
/(\d{2,}):(\d{2}):(\d{2}),(\d{3}) --> (\d{2,}):(\d{2}):(\d{2}),(\d{3})/g,
|
||||
(
|
||||
_match,
|
||||
sh: string,
|
||||
sm: string,
|
||||
ss: string,
|
||||
sms: string,
|
||||
eh: string,
|
||||
em: string,
|
||||
es: string,
|
||||
ems: string,
|
||||
) => {
|
||||
cueCount += 1;
|
||||
const start = Number(sh) * 3600000 + Number(sm) * 60000 + Number(ss) * 1000 + Number(sms);
|
||||
const end = Number(eh) * 3600000 + Number(em) * 60000 + Number(es) * 1000 + Number(ems);
|
||||
return `${formatTimestamp(start + offsetSeconds * 1000)} --> ${formatTimestamp(end + offsetSeconds * 1000)}`;
|
||||
},
|
||||
);
|
||||
if (cueCount === 0)
|
||||
throw new Error(
|
||||
'Whisper produced no subtitle cues. The audio may contain no recognized speech.',
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function ensureAvailableOutput(outputPath: string): Promise<void> {
|
||||
try {
|
||||
await stat(outputPath);
|
||||
} catch (error) {
|
||||
if (isMissingFile(error)) return;
|
||||
throw error;
|
||||
}
|
||||
throw new Error(`Subtitle output already exists: ${outputPath}`);
|
||||
}
|
||||
|
||||
// Fail before extraction and transcription when the destination cannot take the file.
|
||||
export async function ensureWritableDirectory(directory: string): Promise<void> {
|
||||
try {
|
||||
await access(directory, constants.W_OK | constants.X_OK);
|
||||
} catch {
|
||||
throw new Error(`Cannot save subtitles: ${directory} is not writable.`);
|
||||
}
|
||||
}
|
||||
|
||||
async function writeSubtitles(input: {
|
||||
mediaPath: string;
|
||||
outputPath?: string;
|
||||
contents: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
const parsed = path.parse(input.mediaPath);
|
||||
const directory = input.outputPath ? path.dirname(path.resolve(input.outputPath)) : parsed.dir;
|
||||
const temporaryDirectory = await mkdtemp(path.join(directory, '.subminer-subtitles-'));
|
||||
try {
|
||||
const staged = path.join(temporaryDirectory, 'subtitles.srt');
|
||||
await writeFile(staged, input.contents, { flag: 'wx' });
|
||||
for (let suffix = 0; ; suffix += 1) {
|
||||
input.signal?.throwIfAborted();
|
||||
const destination = input.outputPath
|
||||
? path.resolve(input.outputPath)
|
||||
: path.join(directory, `${parsed.name}.ja.generated${suffix ? `.${suffix}` : ''}.srt`);
|
||||
try {
|
||||
await publishSubtitleGenerationFile(staged, destination);
|
||||
return destination;
|
||||
} catch (error) {
|
||||
if (
|
||||
!input.outputPath &&
|
||||
error instanceof Error &&
|
||||
'code' in error &&
|
||||
error.code === 'EEXIST'
|
||||
)
|
||||
continue;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await rm(temporaryDirectory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateJapaneseSubtitles(input: {
|
||||
config: SubtitleGenerationConfig;
|
||||
modelDirectory: string;
|
||||
mediaPath: string;
|
||||
audioStreamIndex?: number;
|
||||
outputPath?: string;
|
||||
onProgress?: (progress: SubtitleGenerationProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
input.signal?.throwIfAborted();
|
||||
if (/^[a-z][a-z\d+.-]*:\/\//i.test(input.mediaPath))
|
||||
throw new Error('Subtitle generation requires a local media file.');
|
||||
const mediaPath = path.resolve(input.mediaPath);
|
||||
if (!(await stat(mediaPath)).isFile())
|
||||
throw new Error('Subtitle generation requires a local media file.');
|
||||
if (input.outputPath) await ensureAvailableOutput(path.resolve(input.outputPath));
|
||||
await ensureWritableDirectory(
|
||||
input.outputPath ? path.dirname(path.resolve(input.outputPath)) : path.dirname(mediaPath),
|
||||
);
|
||||
const model = await resolveSubtitleGenerationModel(input.config, input.modelDirectory);
|
||||
if (model.kind === 'missing')
|
||||
throw new Error(
|
||||
'No Whisper model found. Download a model or configure an existing model path.',
|
||||
);
|
||||
if (model.kind === 'invalid') throw new Error(model.message);
|
||||
const tools = requireSubtitleGenerationTools(await resolveSubtitleGenerationTools(input.config));
|
||||
input.onProgress?.({ stage: 'extract', message: 'Inspecting audio tracks...' });
|
||||
const probe = await runSubtitleGenerationProcess({
|
||||
command: tools.ffprobe,
|
||||
args: [
|
||||
'-v',
|
||||
'error',
|
||||
'-show_entries',
|
||||
'stream=index,codec_type,start_time,duration:stream_tags=language:format=start_time,duration',
|
||||
'-of',
|
||||
'json',
|
||||
mediaPath,
|
||||
],
|
||||
signal: input.signal,
|
||||
});
|
||||
const audio = parseAudioProbe(probe, input.audioStreamIndex);
|
||||
const temporaryDirectory = await mkdtemp(path.join(tmpdir(), 'subminer-whisper-'));
|
||||
try {
|
||||
const wavPath = path.join(temporaryDirectory, 'audio.wav');
|
||||
const subtitleBase = path.join(temporaryDirectory, 'subtitles');
|
||||
input.onProgress?.({ stage: 'extract', percent: 0, message: 'Extracting audio...' });
|
||||
await runSubtitleGenerationProcess({
|
||||
command: tools.ffmpeg,
|
||||
args: [
|
||||
'-nostdin',
|
||||
'-hide_banner',
|
||||
'-loglevel',
|
||||
'error',
|
||||
'-i',
|
||||
mediaPath,
|
||||
'-map',
|
||||
`0:${audio.index}`,
|
||||
'-vn',
|
||||
'-af',
|
||||
'asetpts=PTS-STARTPTS',
|
||||
'-ac',
|
||||
'1',
|
||||
'-ar',
|
||||
'16000',
|
||||
'-c:a',
|
||||
'pcm_s16le',
|
||||
'-progress',
|
||||
'pipe:1',
|
||||
'-nostats',
|
||||
wavPath,
|
||||
],
|
||||
signal: input.signal,
|
||||
onLine: (line) => {
|
||||
const match = /^out_time_us=(\d+)$/.exec(line);
|
||||
if (match && audio.duration && audio.duration > 0) {
|
||||
input.onProgress?.({
|
||||
stage: 'extract',
|
||||
percent: Math.min(100, Math.floor(Number(match[1]) / 10000 / audio.duration)),
|
||||
message: 'Extracting audio...',
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
input.onProgress?.({
|
||||
stage: 'transcribe',
|
||||
percent: 0,
|
||||
message: 'Generating Japanese subtitles...',
|
||||
});
|
||||
let srt: string;
|
||||
if (tools.vad !== null) {
|
||||
srt = await transcribeSubtitleDialogue({
|
||||
config: input.config,
|
||||
tools: { ...tools, vad: tools.vad },
|
||||
modelPath: model.path,
|
||||
wavPath,
|
||||
directory: temporaryDirectory,
|
||||
signal: input.signal,
|
||||
onProgress: input.onProgress,
|
||||
});
|
||||
} else {
|
||||
await runSubtitleGenerationProcess({
|
||||
command: tools.whisper,
|
||||
args: [
|
||||
'-m',
|
||||
model.path,
|
||||
'-f',
|
||||
wavPath,
|
||||
'-l',
|
||||
'ja',
|
||||
'-t',
|
||||
String(input.config.threads),
|
||||
'-osrt',
|
||||
'-of',
|
||||
subtitleBase,
|
||||
'-pp',
|
||||
],
|
||||
signal: input.signal,
|
||||
onLine: (line) => {
|
||||
const match = /progress\s*=\s*(\d+(?:\.\d+)?)%/.exec(line);
|
||||
if (match)
|
||||
input.onProgress?.({
|
||||
stage: 'transcribe',
|
||||
percent: Math.min(100, Number(match[1])),
|
||||
message: 'Generating Japanese subtitles...',
|
||||
});
|
||||
},
|
||||
});
|
||||
srt = await readFile(`${subtitleBase}.srt`, 'utf8');
|
||||
}
|
||||
input.signal?.throwIfAborted();
|
||||
input.onProgress?.({ stage: 'write', message: 'Saving Japanese subtitles...' });
|
||||
const contents = shiftSubtitleTimestamps(srt, audio.offset);
|
||||
const outputPath = await writeSubtitles({
|
||||
mediaPath,
|
||||
outputPath: input.outputPath,
|
||||
contents,
|
||||
signal: input.signal,
|
||||
});
|
||||
input.onProgress?.({ stage: 'write', percent: 100, message: 'Japanese subtitles are ready.' });
|
||||
return outputPath;
|
||||
} finally {
|
||||
await rm(temporaryDirectory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ export interface ConfiguredShortcuts {
|
||||
openRuntimeOptions: string | null | undefined;
|
||||
openJimaku: string | null | undefined;
|
||||
openTsukihime: string | null | undefined;
|
||||
openSubtitleGeneration: string | null | undefined;
|
||||
openSessionHelp: string | null | undefined;
|
||||
openControllerSelect: string | null | undefined;
|
||||
openControllerDebug: string | null | undefined;
|
||||
@@ -67,6 +68,7 @@ export function resolveConfiguredShortcuts(
|
||||
openRuntimeOptions: normalizeShortcut(shortcutValue('openRuntimeOptions')),
|
||||
openJimaku: normalizeShortcut(shortcutValue('openJimaku')),
|
||||
openTsukihime: normalizeShortcut(shortcutValue('openTsukihime')),
|
||||
openSubtitleGeneration: normalizeShortcut(shortcutValue('openSubtitleGeneration')),
|
||||
openSessionHelp: normalizeShortcut(shortcutValue('openSessionHelp')),
|
||||
openControllerSelect: normalizeShortcut(shortcutValue('openControllerSelect')),
|
||||
openControllerDebug: normalizeShortcut(shortcutValue('openControllerDebug')),
|
||||
|
||||
+34
@@ -463,6 +463,9 @@ import {
|
||||
} from './main/early-single-instance';
|
||||
import { handleMpvCommandFromIpcRuntime } from './main/ipc-mpv-command';
|
||||
import { registerIpcRuntimeServices } from './main/ipc-runtime';
|
||||
import { createSubtitleGenerationRuntime } from './main/runtime/subtitle-generation-runtime';
|
||||
import { registerSubtitleGenerationIpc } from './main/runtime/subtitle-generation-ipc';
|
||||
import { openSubtitleGenerationModal } from './main/runtime/subtitle-generation-open';
|
||||
import { createAnkiJimakuIpcRuntimeServiceDeps } from './main/dependencies';
|
||||
import { createMainBootServices, type MainBootServicesResult } from './main/boot/services';
|
||||
import { handleCliCommandRuntimeServiceWithContext } from './main/cli-runtime';
|
||||
@@ -2990,6 +2993,14 @@ function openTsukihimeOverlay(): void {
|
||||
);
|
||||
}
|
||||
|
||||
function openSubtitleGenerationOverlay(): void {
|
||||
openOverlayHostedModalWithOsd(
|
||||
openSubtitleGenerationModal,
|
||||
'Subtitle generation overlay unavailable.',
|
||||
'Failed to open subtitle generation overlay.',
|
||||
);
|
||||
}
|
||||
|
||||
function openSessionHelpOverlay(): void {
|
||||
openOverlayHostedModalWithOsd(
|
||||
openSessionHelpModalRuntime,
|
||||
@@ -5510,6 +5521,7 @@ async function dispatchSessionAction(request: SessionActionDispatchRequest): Pro
|
||||
openJimaku: () => openJimakuOverlay(),
|
||||
openTsukihime: () => openTsukihimeOverlay(),
|
||||
openSessionHelp: () => openSessionHelpOverlay(),
|
||||
openSubtitleGeneration: () => openSubtitleGenerationOverlay(),
|
||||
openCharacterDictionaryManager: () => openCharacterDictionaryManagerOverlay(),
|
||||
openControllerSelect: () => openControllerSelectOverlay(),
|
||||
openControllerDebug: () => openControllerDebugOverlay(),
|
||||
@@ -6654,3 +6666,25 @@ function setOverlayVisible(visible: boolean): void {
|
||||
}
|
||||
|
||||
registerIpcRuntimeHandlers();
|
||||
const subtitleGenerationRuntime = createSubtitleGenerationRuntime({
|
||||
getConfig: () => configService.getConfig().subtitleGeneration,
|
||||
getModelDirectory: () =>
|
||||
path.join(path.dirname(configService.getConfigPath()), 'models', 'whisper'),
|
||||
getMpvClient: () => appState.mpvClient,
|
||||
onProgress: (progress) => {
|
||||
for (const window of [overlayManager.getMainWindow(), overlayManager.getModalWindow()]) {
|
||||
if (window && !window.isDestroyed())
|
||||
window.webContents.send(IPC_CHANNELS.event.subtitleGenerationProgress, progress);
|
||||
}
|
||||
},
|
||||
});
|
||||
registerSubtitleGenerationIpc({
|
||||
ipc: ipcMain,
|
||||
isAllowedSender: (sender) =>
|
||||
[overlayManager.getMainWindow(), overlayManager.getModalWindow()].some(
|
||||
(window) => window && !window.isDestroyed() && window.webContents === sender,
|
||||
),
|
||||
runtime: subtitleGenerationRuntime,
|
||||
openModal: () => openSubtitleGenerationModal(createOverlayHostedModalOpenDeps()),
|
||||
});
|
||||
app.on('before-quit', () => subtitleGenerationRuntime.cancel());
|
||||
|
||||
@@ -20,6 +20,7 @@ function createShortcuts(): ConfiguredShortcuts {
|
||||
openRuntimeOptions: null,
|
||||
openJimaku: null,
|
||||
openTsukihime: null,
|
||||
openSubtitleGeneration: null,
|
||||
openSessionHelp: null,
|
||||
openControllerSelect: null,
|
||||
openControllerDebug: null,
|
||||
|
||||
@@ -24,6 +24,7 @@ function createShortcuts(): ConfiguredShortcuts {
|
||||
openRuntimeOptions: null,
|
||||
openJimaku: null,
|
||||
openTsukihime: null,
|
||||
openSubtitleGeneration: null,
|
||||
openSessionHelp: null,
|
||||
openControllerSelect: null,
|
||||
openControllerDebug: null,
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import type { IpcMain, WebContents } from 'electron';
|
||||
import { IPC_CHANNELS } from '../../shared/ipc/contracts';
|
||||
import { isSubtitleGenerationModelId } from '../../shared/subtitle-generation-model-catalog';
|
||||
import type { createSubtitleGenerationRuntime } from './subtitle-generation-runtime';
|
||||
|
||||
export function registerSubtitleGenerationIpc(deps: {
|
||||
ipc: Pick<IpcMain, 'handle'>;
|
||||
isAllowedSender: (sender: WebContents) => boolean;
|
||||
openModal: () => Promise<boolean>;
|
||||
runtime: ReturnType<typeof createSubtitleGenerationRuntime>;
|
||||
}): void {
|
||||
const handlers = [
|
||||
[IPC_CHANNELS.request.requestSubtitleGenerationOpen, () => deps.openModal()],
|
||||
[IPC_CHANNELS.request.getSubtitleGenerationStatus, () => deps.runtime.getStatus()],
|
||||
[
|
||||
IPC_CHANNELS.request.selectSubtitleGenerationModel,
|
||||
(model: unknown) => {
|
||||
if (!isSubtitleGenerationModelId(model))
|
||||
throw new Error('Unknown subtitle generation model.');
|
||||
return deps.runtime.selectModel(model);
|
||||
},
|
||||
],
|
||||
[IPC_CHANNELS.request.startSubtitleGeneration, () => deps.runtime.start()],
|
||||
[IPC_CHANNELS.request.downloadSubtitleGenerationModel, () => deps.runtime.download()],
|
||||
[IPC_CHANNELS.request.downloadSubtitleGenerationVadModel, () => deps.runtime.downloadVad()],
|
||||
[
|
||||
IPC_CHANNELS.request.setSubtitleGenerationVadEnabled,
|
||||
(enabled: unknown) => {
|
||||
if (typeof enabled !== 'boolean')
|
||||
throw new Error('Speech detection selection must be a boolean.');
|
||||
return deps.runtime.setVadEnabled(enabled);
|
||||
},
|
||||
],
|
||||
[IPC_CHANNELS.request.cancelSubtitleGeneration, () => deps.runtime.cancel()],
|
||||
] as const;
|
||||
for (const [channel, handler] of handlers) {
|
||||
deps.ipc.handle(channel, (event, payload: unknown) => {
|
||||
if (!deps.isAllowedSender(event.sender))
|
||||
throw new Error('Subtitle generation is only available from the overlay.');
|
||||
return handler(payload);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { openSubtitleGenerationModal } from './subtitle-generation-open';
|
||||
import { IPC_CHANNELS } from '../../shared/ipc/contracts';
|
||||
|
||||
test('subtitle generation opens in the dedicated modal window with normal close restoration', async () => {
|
||||
const calls: string[] = [];
|
||||
const opened = await openSubtitleGenerationModal({
|
||||
ensureOverlayStartupPrereqs: () => {
|
||||
calls.push('startup');
|
||||
},
|
||||
ensureOverlayWindowsReadyForVisibilityActions: () => {
|
||||
calls.push('windows');
|
||||
},
|
||||
sendToActiveOverlayWindow: (channel, payload, options) => {
|
||||
assert.deepEqual(calls, ['startup', 'windows']);
|
||||
assert.equal(channel, IPC_CHANNELS.event.subtitleGenerationOpen);
|
||||
assert.equal(payload, undefined);
|
||||
assert.deepEqual(options, {
|
||||
restoreOnModalClose: 'subtitle-generation',
|
||||
preferModalWindow: true,
|
||||
});
|
||||
calls.push('open');
|
||||
return true;
|
||||
},
|
||||
waitForModalOpen: async (modal) => {
|
||||
assert.equal(modal, 'subtitle-generation');
|
||||
return true;
|
||||
},
|
||||
logWarn: () => {
|
||||
assert.fail('opening should not require a retry');
|
||||
},
|
||||
});
|
||||
assert.equal(opened, true);
|
||||
assert.deepEqual(calls, ['startup', 'windows', 'open']);
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
import { IPC_CHANNELS } from '../../shared/ipc/contracts';
|
||||
import { openOverlayHostedModal, retryOverlayModalOpen } from './overlay-hosted-modal-open';
|
||||
|
||||
export function openSubtitleGenerationModal(
|
||||
deps: Parameters<typeof openOverlayHostedModal>[0] & Parameters<typeof retryOverlayModalOpen>[0],
|
||||
): Promise<boolean> {
|
||||
return retryOverlayModalOpen(deps, {
|
||||
modal: 'subtitle-generation',
|
||||
timeoutMs: 1500,
|
||||
retryWarning: 'Subtitle generation modal did not acknowledge opening; retrying.',
|
||||
sendOpen: () =>
|
||||
openOverlayHostedModal(deps, {
|
||||
channel: IPC_CHANNELS.event.subtitleGenerationOpen,
|
||||
modal: 'subtitle-generation',
|
||||
preferModalWindow: true,
|
||||
}),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { DEFAULT_SUBTITLE_GENERATION_CONFIG } from '../../shared/subtitle-generation';
|
||||
import {
|
||||
createSubtitleGenerationRuntime,
|
||||
type SubtitleGenerationRuntimeDeps,
|
||||
} from './subtitle-generation-runtime';
|
||||
|
||||
function fixture(overrides: Partial<SubtitleGenerationRuntimeDeps> = {}) {
|
||||
let mediaPath = '/video/episode.mkv';
|
||||
const commands: unknown[][] = [];
|
||||
const client = {
|
||||
connected: true,
|
||||
requestProperty: async (name: string): Promise<unknown> =>
|
||||
name === 'path' ? mediaPath : [{ type: 'audio', selected: true, 'ff-index': 3 }],
|
||||
request: async (command: unknown[]) => {
|
||||
commands.push(command);
|
||||
return { error: 'success' };
|
||||
},
|
||||
};
|
||||
const runtime = createSubtitleGenerationRuntime({
|
||||
getConfig: () => DEFAULT_SUBTITLE_GENERATION_CONFIG,
|
||||
getModelDirectory: () => '/models',
|
||||
getMpvClient: () => client,
|
||||
onProgress: () => {},
|
||||
resolveModel: async () => ({ kind: 'external', path: '/models/local.bin' }),
|
||||
resolveTools: async (config) => ({
|
||||
ffmpeg: { kind: 'found', path: '/usr/bin/ffmpeg' },
|
||||
ffprobe: { kind: 'found', path: '/usr/bin/ffprobe' },
|
||||
whisper: { kind: 'found', path: '/usr/bin/whisper-cli' },
|
||||
vad: config.vadModelPath ? { kind: 'found', path: '/usr/bin/vad' } : null,
|
||||
}),
|
||||
generate: async () => '/video/episode.ja.generated.srt',
|
||||
...overrides,
|
||||
});
|
||||
return {
|
||||
runtime,
|
||||
client,
|
||||
commands,
|
||||
changeMedia: () => {
|
||||
mediaPath = '/video/next.mkv';
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('generation uses the selected audio track and loads the timed SRT with zero delay', async () => {
|
||||
const { runtime, commands } = fixture({
|
||||
generate: async (input) => {
|
||||
assert.equal(input.mediaPath, '/video/episode.mkv');
|
||||
assert.equal(input.audioStreamIndex, 3);
|
||||
return '/video/generated.srt';
|
||||
},
|
||||
});
|
||||
assert.equal((await runtime.start()).ok, true);
|
||||
assert.deepEqual(commands, [
|
||||
['sub-add', '/video/generated.srt', 'select', 'Generated Japanese', 'ja'],
|
||||
['set_property', 'sub-delay', 0],
|
||||
]);
|
||||
});
|
||||
|
||||
test('generation preserves the output without attaching it to a different video', async () => {
|
||||
const subject = fixture({
|
||||
generate: async () => {
|
||||
subject.changeMedia();
|
||||
return '/video/generated.srt';
|
||||
},
|
||||
});
|
||||
const result = await subject.runtime.start();
|
||||
assert.equal(result.ok, true);
|
||||
assert.match(result.message, /Playback changed/);
|
||||
assert.deepEqual(subject.commands, []);
|
||||
});
|
||||
|
||||
test('mpv load failure still reports where the generated subtitles were saved', async () => {
|
||||
const subject = fixture();
|
||||
subject.client.request = async () => ({ error: 'loading failed' });
|
||||
const result = await subject.runtime.start();
|
||||
assert.equal(result.ok, true);
|
||||
assert.match(result.message, /Subtitles saved:.*Could not finish loading/);
|
||||
});
|
||||
|
||||
test('cancellation during the final media check keeps the saved file without loading it', async () => {
|
||||
const subject = fixture();
|
||||
const requestProperty = subject.client.requestProperty;
|
||||
let mediaChecks = 0;
|
||||
subject.client.requestProperty = async (name) => {
|
||||
if (name === 'path' && ++mediaChecks === 3) subject.runtime.cancel();
|
||||
return requestProperty(name);
|
||||
};
|
||||
const result = await subject.runtime.start();
|
||||
assert.equal(result.ok, true);
|
||||
assert.match(result.message, /Cancelled after saving/);
|
||||
assert.deepEqual(subject.commands, []);
|
||||
});
|
||||
|
||||
test('only one job runs, cancellation reaches the worker, and status retains its result', async () => {
|
||||
let signal: AbortSignal | undefined;
|
||||
let entered = () => {};
|
||||
const started = new Promise<void>((resolve) => {
|
||||
entered = resolve;
|
||||
});
|
||||
const { runtime } = fixture({
|
||||
generate: async (input) => {
|
||||
signal = input.signal;
|
||||
input.onProgress?.({ stage: 'transcribe', percent: 25, message: 'Working' });
|
||||
entered();
|
||||
return new Promise((_, reject) =>
|
||||
input.signal?.addEventListener('abort', () => reject(new Error('Aborted')), { once: true }),
|
||||
);
|
||||
},
|
||||
});
|
||||
const first = runtime.start();
|
||||
await started;
|
||||
await assert.rejects(runtime.selectModel('medium'), /current operation/);
|
||||
assert.equal((await runtime.start()).ok, false);
|
||||
assert.equal((await runtime.download()).ok, false);
|
||||
const active = await runtime.getStatus();
|
||||
assert.equal(active.running, true);
|
||||
assert.equal(active.progress?.percent, 25);
|
||||
runtime.cancel();
|
||||
assert.equal(signal?.aborted, true);
|
||||
assert.deepEqual(await first, { ok: false, message: 'Cancelled.' });
|
||||
const completed = await runtime.getStatus();
|
||||
assert.equal(completed.running, false);
|
||||
assert.deepEqual(completed.lastResult, { ok: false, message: 'Cancelled.' });
|
||||
});
|
||||
|
||||
test('external audio cannot silently generate from a different internal track', async () => {
|
||||
const subject = fixture({
|
||||
generate: async () => {
|
||||
assert.fail('must not transcribe');
|
||||
},
|
||||
});
|
||||
subject.client.requestProperty = async (name) =>
|
||||
name === 'path'
|
||||
? '/video/episode.mkv'
|
||||
: [{ type: 'audio', selected: true, external: true, 'ff-index': 0 }];
|
||||
const result = await subject.runtime.start();
|
||||
assert.equal(result.ok, false);
|
||||
assert.match(result.message, /audio track inside/);
|
||||
});
|
||||
|
||||
test('model selection is retained and used for status, download, and generation', async () => {
|
||||
const seen: string[] = [];
|
||||
const { runtime } = fixture({
|
||||
resolveModel: async (config) => ({
|
||||
kind: 'missing',
|
||||
path: `/models/${config.managedModel}.bin`,
|
||||
}),
|
||||
download: async ({ config }) => {
|
||||
seen.push(`download:${config.managedModel}`);
|
||||
return '/models/downloaded.bin';
|
||||
},
|
||||
generate: async ({ config }) => {
|
||||
seen.push(`generate:${config.managedModel}`);
|
||||
return '/video/generated.srt';
|
||||
},
|
||||
});
|
||||
const selected = await runtime.selectModel('medium');
|
||||
assert.equal(selected.managedModel, 'medium');
|
||||
assert.equal(selected.model.path, '/models/medium.bin');
|
||||
assert.equal((await runtime.getStatus()).managedModel, 'medium');
|
||||
assert.equal((await runtime.download()).ok, true);
|
||||
assert.equal((await runtime.start()).ok, true);
|
||||
assert.deepEqual(seen, ['download:medium', 'generate:medium']);
|
||||
assert.equal(DEFAULT_SUBTITLE_GENERATION_CONFIG.managedModel, 'small');
|
||||
});
|
||||
|
||||
test('external model paths prevent managed selection, including unreadable overrides', async () => {
|
||||
const { runtime } = fixture({
|
||||
getConfig: () => ({
|
||||
...DEFAULT_SUBTITLE_GENERATION_CONFIG,
|
||||
modelPath: '/missing/external.bin',
|
||||
}),
|
||||
resolveModel: async () => ({
|
||||
kind: 'invalid',
|
||||
path: '/missing/external.bin',
|
||||
message: 'Missing model',
|
||||
}),
|
||||
});
|
||||
assert.equal((await runtime.getStatus()).externalModelPath, '/missing/external.bin');
|
||||
await assert.rejects(runtime.selectModel('medium'), /Clear Model Path/);
|
||||
});
|
||||
|
||||
test('status reports the speech detector only while dialogue mode is on', async () => {
|
||||
const { runtime } = fixture({
|
||||
resolveVadModel: async () => ({ kind: 'managed', path: '/models/ggml-silero-v6.2.0.bin' }),
|
||||
});
|
||||
assert.equal((await runtime.getStatus()).tools.vad, null);
|
||||
await runtime.setVadEnabled(true);
|
||||
assert.deepEqual((await runtime.getStatus()).tools.vad, { kind: 'found', path: '/usr/bin/vad' });
|
||||
});
|
||||
|
||||
test('speech detection is optional and downloading alone does not enable it', async () => {
|
||||
let installed = false;
|
||||
const paths: string[] = [];
|
||||
const { runtime } = fixture({
|
||||
resolveVadModel: async () => ({
|
||||
kind: installed ? 'managed' : 'missing',
|
||||
path: '/models/ggml-silero-v6.2.0.bin',
|
||||
}),
|
||||
downloadVad: async () => {
|
||||
installed = true;
|
||||
return '/models/ggml-silero-v6.2.0.bin';
|
||||
},
|
||||
generate: async ({ config }) => {
|
||||
paths.push(config.vadModelPath);
|
||||
return '/video/output.srt';
|
||||
},
|
||||
});
|
||||
assert.equal((await runtime.getStatus()).vad.enabled, false);
|
||||
assert.equal((await runtime.start()).ok, true);
|
||||
await runtime.setVadEnabled(true);
|
||||
assert.equal((await runtime.start()).ok, false);
|
||||
await runtime.setVadEnabled(false);
|
||||
assert.equal((await runtime.downloadVad()).ok, true);
|
||||
assert.equal((await runtime.getStatus()).vad.enabled, false);
|
||||
await runtime.setVadEnabled(true);
|
||||
assert.equal((await runtime.start()).ok, true);
|
||||
await runtime.setVadEnabled(false);
|
||||
assert.equal((await runtime.start()).ok, true);
|
||||
assert.deepEqual(paths, ['', '/models/ggml-silero-v6.2.0.bin', '']);
|
||||
});
|
||||
|
||||
test('existing external speech model remains the default and survives session toggles', async () => {
|
||||
const { runtime } = fixture({
|
||||
getConfig: () => ({ ...DEFAULT_SUBTITLE_GENERATION_CONFIG, vadModelPath: '/external/vad.bin' }),
|
||||
resolveVadModel: async (config) => ({ kind: 'external', path: config.vadModelPath }),
|
||||
generate: async ({ config }) => {
|
||||
assert.equal(config.vadModelPath, '/external/vad.bin');
|
||||
return '/video/output.srt';
|
||||
},
|
||||
});
|
||||
assert.equal((await runtime.getStatus()).vad.enabled, true);
|
||||
await runtime.setVadEnabled(false);
|
||||
await runtime.setVadEnabled(true);
|
||||
assert.equal((await runtime.start()).ok, true);
|
||||
});
|
||||
|
||||
test('speech model downloads share the job lock and cancellation', async () => {
|
||||
let enter = () => {};
|
||||
const started = new Promise<void>((resolve) => {
|
||||
enter = resolve;
|
||||
});
|
||||
const { runtime } = fixture({
|
||||
downloadVad: async ({ signal }) => {
|
||||
enter();
|
||||
return new Promise((_, reject) =>
|
||||
signal?.addEventListener('abort', () => reject(new Error('cancelled')), { once: true }),
|
||||
);
|
||||
},
|
||||
});
|
||||
const download = runtime.downloadVad();
|
||||
await started;
|
||||
assert.equal((await runtime.start()).ok, false);
|
||||
await assert.rejects(runtime.setVadEnabled(true), /current operation/);
|
||||
runtime.cancel();
|
||||
assert.deepEqual(await download, { ok: false, message: 'Cancelled.' });
|
||||
});
|
||||
@@ -0,0 +1,264 @@
|
||||
import path from 'node:path';
|
||||
import { SUBTITLE_GENERATION_VAD_MODEL } from '../../shared/subtitle-generation-vad-model';
|
||||
import {
|
||||
downloadSubtitleGenerationVadModel,
|
||||
resolveSubtitleGenerationVadModel,
|
||||
} from '../../core/services/subtitle-generation-vad-model';
|
||||
import type { SubtitleGenerationModelId } from '../../shared/subtitle-generation-model-catalog';
|
||||
import {
|
||||
downloadSubtitleGenerationModel,
|
||||
generateJapaneseSubtitles,
|
||||
resolveSubtitleGenerationModel,
|
||||
resolveSubtitleGenerationTools,
|
||||
} from '../../core/services/subtitle-generation';
|
||||
import type {
|
||||
SubtitleGenerationConfig,
|
||||
SubtitleGenerationProgress,
|
||||
} from '../../shared/subtitle-generation';
|
||||
import type {
|
||||
SubtitleGenerationResult,
|
||||
SubtitleGenerationStatus,
|
||||
} from '../../shared/subtitle-generation-ipc';
|
||||
|
||||
interface GenerationMpvClient {
|
||||
connected: boolean;
|
||||
requestProperty: (name: string) => Promise<unknown>;
|
||||
request: (command: unknown[]) => Promise<{ error?: string }>;
|
||||
}
|
||||
|
||||
export interface SubtitleGenerationRuntimeDeps {
|
||||
getConfig: () => SubtitleGenerationConfig;
|
||||
getModelDirectory: () => string;
|
||||
getMpvClient: () => GenerationMpvClient | null;
|
||||
onProgress: (progress: SubtitleGenerationProgress) => void;
|
||||
generate?: typeof generateJapaneseSubtitles;
|
||||
download?: typeof downloadSubtitleGenerationModel;
|
||||
resolveModel?: typeof resolveSubtitleGenerationModel;
|
||||
resolveTools?: typeof resolveSubtitleGenerationTools;
|
||||
downloadVad?: typeof downloadSubtitleGenerationVadModel;
|
||||
resolveVadModel?: typeof resolveSubtitleGenerationVadModel;
|
||||
}
|
||||
|
||||
async function currentLocalMedia(client: GenerationMpvClient | null): Promise<string | null> {
|
||||
if (!client?.connected) return null;
|
||||
const media = await client.requestProperty('path');
|
||||
if (typeof media !== 'string' || !media || /^[a-z][a-z\d+.-]*:\/\//i.test(media)) return null;
|
||||
if (path.isAbsolute(media)) return path.normalize(media);
|
||||
const directory = await client.requestProperty('working-directory');
|
||||
return typeof directory === 'string' ? path.resolve(directory, media) : null;
|
||||
}
|
||||
|
||||
function selectedAudioIndex(tracks: unknown): number {
|
||||
if (!Array.isArray(tracks)) throw new Error('Unable to inspect the selected audio track.');
|
||||
for (const track of tracks) {
|
||||
if (
|
||||
!track ||
|
||||
typeof track !== 'object' ||
|
||||
!('type' in track) ||
|
||||
track.type !== 'audio' ||
|
||||
!('selected' in track) ||
|
||||
track.selected !== true
|
||||
)
|
||||
continue;
|
||||
if ('external' in track && track.external === true)
|
||||
throw new Error('Select an audio track inside the local video before generating subtitles.');
|
||||
if (
|
||||
'ff-index' in track &&
|
||||
typeof track['ff-index'] === 'number' &&
|
||||
Number.isInteger(track['ff-index']) &&
|
||||
track['ff-index'] >= 0
|
||||
)
|
||||
return track['ff-index'];
|
||||
throw new Error('The selected audio track has no FFmpeg stream index.');
|
||||
}
|
||||
throw new Error('Select an audio track in mpv before generating subtitles.');
|
||||
}
|
||||
|
||||
export function createSubtitleGenerationRuntime(deps: SubtitleGenerationRuntimeDeps) {
|
||||
let controller: AbortController | null = null;
|
||||
let progress: SubtitleGenerationProgress | null = null;
|
||||
let lastResult: SubtitleGenerationResult | null = null;
|
||||
let selectedModel: SubtitleGenerationModelId | null = null;
|
||||
let vadEnabled: boolean | null = null;
|
||||
function getConfig(): SubtitleGenerationConfig {
|
||||
const config = deps.getConfig();
|
||||
return {
|
||||
...config,
|
||||
managedModel: selectedModel ?? config.managedModel,
|
||||
vadModelPath:
|
||||
vadEnabled === null
|
||||
? config.vadModelPath
|
||||
: vadEnabled
|
||||
? config.vadModelPath.trim() ||
|
||||
path.resolve(deps.getModelDirectory(), SUBTITLE_GENERATION_VAD_MODEL.filename)
|
||||
: '',
|
||||
};
|
||||
}
|
||||
const report = (update: SubtitleGenerationProgress) => {
|
||||
progress = update;
|
||||
deps.onProgress(update);
|
||||
};
|
||||
|
||||
async function run(
|
||||
operation: (signal: AbortSignal) => Promise<SubtitleGenerationResult>,
|
||||
): Promise<SubtitleGenerationResult> {
|
||||
if (controller)
|
||||
return { ok: false, message: 'A subtitle generation or model download is already running.' };
|
||||
const active = new AbortController();
|
||||
controller = active;
|
||||
progress = null;
|
||||
lastResult = null;
|
||||
try {
|
||||
lastResult = await operation(active.signal);
|
||||
} catch (error) {
|
||||
lastResult = {
|
||||
ok: false,
|
||||
message: active.signal.aborted
|
||||
? 'Cancelled.'
|
||||
: error instanceof Error
|
||||
? error.message
|
||||
: String(error),
|
||||
};
|
||||
} finally {
|
||||
controller = null;
|
||||
}
|
||||
return lastResult;
|
||||
}
|
||||
|
||||
async function getStatus(): Promise<SubtitleGenerationStatus> {
|
||||
const config = getConfig();
|
||||
const model = await (deps.resolveModel ?? resolveSubtitleGenerationModel)(
|
||||
config,
|
||||
deps.getModelDirectory(),
|
||||
);
|
||||
const mediaPath = await currentLocalMedia(deps.getMpvClient()).catch(() => null);
|
||||
return {
|
||||
model,
|
||||
vad: {
|
||||
enabled: Boolean(config.vadModelPath.trim()),
|
||||
model: await (deps.resolveVadModel ?? resolveSubtitleGenerationVadModel)(
|
||||
deps.getConfig(),
|
||||
deps.getModelDirectory(),
|
||||
),
|
||||
},
|
||||
// Session toggles decide whether the speech detector executable is required.
|
||||
tools: await (deps.resolveTools ?? resolveSubtitleGenerationTools)(config),
|
||||
managedModel: config.managedModel,
|
||||
externalModelPath: config.modelPath.trim() || null,
|
||||
mediaPath,
|
||||
running: controller !== null,
|
||||
progress,
|
||||
lastResult,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
getStatus,
|
||||
async setVadEnabled(enabled: boolean): Promise<SubtitleGenerationStatus> {
|
||||
if (controller)
|
||||
throw new Error('Wait for the current operation before changing speech detection.');
|
||||
vadEnabled = enabled;
|
||||
lastResult = null;
|
||||
progress = null;
|
||||
return getStatus();
|
||||
},
|
||||
downloadVad(): Promise<SubtitleGenerationResult> {
|
||||
return run(async (signal) => {
|
||||
await (deps.downloadVad ?? downloadSubtitleGenerationVadModel)({
|
||||
config: deps.getConfig(),
|
||||
modelDirectory: deps.getModelDirectory(),
|
||||
onProgress: report,
|
||||
signal,
|
||||
});
|
||||
return { ok: true, message: 'Speech detection model is ready.' };
|
||||
});
|
||||
},
|
||||
async selectModel(model: SubtitleGenerationModelId): Promise<SubtitleGenerationStatus> {
|
||||
if (controller) throw new Error('Wait for the current operation before changing models.');
|
||||
if (deps.getConfig().modelPath.trim())
|
||||
throw new Error('Clear Model Path in Settings before choosing a managed model.');
|
||||
selectedModel = model;
|
||||
lastResult = null;
|
||||
progress = null;
|
||||
return getStatus();
|
||||
},
|
||||
cancel(): void {
|
||||
controller?.abort();
|
||||
},
|
||||
download(): Promise<SubtitleGenerationResult> {
|
||||
return run(async (signal) => {
|
||||
await (deps.download ?? downloadSubtitleGenerationModel)({
|
||||
config: getConfig(),
|
||||
modelDirectory: deps.getModelDirectory(),
|
||||
onProgress: report,
|
||||
signal,
|
||||
});
|
||||
return { ok: true, message: 'Model downloaded. Ready to generate Japanese subtitles.' };
|
||||
});
|
||||
},
|
||||
start(): Promise<SubtitleGenerationResult> {
|
||||
return run(async (signal) => {
|
||||
const config = getConfig();
|
||||
if (config.vadModelPath.trim()) {
|
||||
const vad = await (deps.resolveVadModel ?? resolveSubtitleGenerationVadModel)(
|
||||
deps.getConfig(),
|
||||
deps.getModelDirectory(),
|
||||
);
|
||||
if (vad.kind === 'missing')
|
||||
throw new Error(
|
||||
'Download the optional speech detection model or turn off Focus on spoken dialogue.',
|
||||
);
|
||||
if (vad.kind === 'invalid') throw new Error(vad.message);
|
||||
}
|
||||
const client = deps.getMpvClient();
|
||||
const mediaPath = await currentLocalMedia(client);
|
||||
if (!client || !mediaPath)
|
||||
throw new Error('Open a local video or audio file in mpv first.');
|
||||
const audioStreamIndex = selectedAudioIndex(await client.requestProperty('track-list'));
|
||||
if ((await currentLocalMedia(client)) !== mediaPath)
|
||||
throw new Error('The current media changed. Start generation again.');
|
||||
signal.throwIfAborted();
|
||||
const outputPath = await (deps.generate ?? generateJapaneseSubtitles)({
|
||||
config,
|
||||
modelDirectory: deps.getModelDirectory(),
|
||||
mediaPath,
|
||||
audioStreamIndex,
|
||||
onProgress: report,
|
||||
signal,
|
||||
});
|
||||
// Saving succeeds even if playback changes or disconnects during the job.
|
||||
try {
|
||||
const playingMedia = await currentLocalMedia(client);
|
||||
if (!signal.aborted && deps.getMpvClient() === client && playingMedia === mediaPath) {
|
||||
const loaded = await client.request([
|
||||
'sub-add',
|
||||
outputPath,
|
||||
'select',
|
||||
'Generated Japanese',
|
||||
'ja',
|
||||
]);
|
||||
if (loaded.error && loaded.error !== 'success') throw new Error(loaded.error);
|
||||
const delay = await client.request(['set_property', 'sub-delay', 0]);
|
||||
if (delay.error && delay.error !== 'success') throw new Error(delay.error);
|
||||
return {
|
||||
ok: true,
|
||||
outputPath,
|
||||
message: `Japanese subtitles saved and loaded: ${outputPath}`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
outputPath,
|
||||
message: `Subtitles saved: ${outputPath}. ${signal.aborted ? 'Cancelled after saving; the file was not loaded.' : 'Playback changed, so the file was not loaded.'}`,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: true,
|
||||
outputPath,
|
||||
message: `Subtitles saved: ${outputPath}. Could not finish loading into mpv: ${error instanceof Error ? error.message : String(error)}`,
|
||||
};
|
||||
}
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -76,8 +76,10 @@ import type {
|
||||
MediaTimingReviewWaveformRequest,
|
||||
} from './types';
|
||||
import { IPC_CHANNELS } from './shared/ipc/contracts';
|
||||
import type { SubtitleGenerationProgress } from './shared/subtitle-generation';
|
||||
|
||||
const overlayLayer = resolveOverlayLayerFromArgv(process.argv);
|
||||
const onSubtitleGenerationOpen = createQueuedIpcListener(IPC_CHANNELS.event.subtitleGenerationOpen);
|
||||
|
||||
type EmptyListener = () => void;
|
||||
type PayloadedListener<T> = (payload: T) => void;
|
||||
@@ -261,6 +263,28 @@ const onSecondarySubtitleModeEvent = createLatestValueIpcListenerWithPayload<Sec
|
||||
);
|
||||
|
||||
const electronAPI: ElectronAPI = {
|
||||
requestSubtitleGenerationOpen: () =>
|
||||
ipcRenderer.invoke(IPC_CHANNELS.request.requestSubtitleGenerationOpen),
|
||||
onSubtitleGenerationOpen,
|
||||
getSubtitleGenerationStatus: () =>
|
||||
ipcRenderer.invoke(IPC_CHANNELS.request.getSubtitleGenerationStatus),
|
||||
selectSubtitleGenerationModel: (model) =>
|
||||
ipcRenderer.invoke(IPC_CHANNELS.request.selectSubtitleGenerationModel, model),
|
||||
startSubtitleGeneration: () => ipcRenderer.invoke(IPC_CHANNELS.request.startSubtitleGeneration),
|
||||
downloadSubtitleGenerationModel: () =>
|
||||
ipcRenderer.invoke(IPC_CHANNELS.request.downloadSubtitleGenerationModel),
|
||||
downloadSubtitleGenerationVadModel: () =>
|
||||
ipcRenderer.invoke(IPC_CHANNELS.request.downloadSubtitleGenerationVadModel),
|
||||
setSubtitleGenerationVadEnabled: (enabled) =>
|
||||
ipcRenderer.invoke(IPC_CHANNELS.request.setSubtitleGenerationVadEnabled, enabled),
|
||||
cancelSubtitleGeneration: () => ipcRenderer.invoke(IPC_CHANNELS.request.cancelSubtitleGeneration),
|
||||
onSubtitleGenerationProgress: (callback) => {
|
||||
const listener = (_event: IpcRendererEvent, progress: SubtitleGenerationProgress) =>
|
||||
callback(progress);
|
||||
ipcRenderer.on(IPC_CHANNELS.event.subtitleGenerationProgress, listener);
|
||||
return () =>
|
||||
ipcRenderer.removeListener(IPC_CHANNELS.event.subtitleGenerationProgress, listener);
|
||||
},
|
||||
getOverlayLayer: () => overlayLayer,
|
||||
getPathForFile: (file: File) => webUtils.getPathForFile(file),
|
||||
onSubtitle: (callback: (data: SubtitleData) => void) => {
|
||||
|
||||
@@ -5,6 +5,7 @@ type ControllerInteractionModalState = {
|
||||
kikuModalOpen: boolean;
|
||||
runtimeOptionsModalOpen: boolean;
|
||||
subsyncModalOpen: boolean;
|
||||
subtitleGenerationModalOpen?: boolean;
|
||||
youtubePickerModalOpen: boolean;
|
||||
sessionHelpModalOpen: boolean;
|
||||
subtitleSidebarModalOpen: boolean;
|
||||
@@ -18,6 +19,7 @@ export function isControllerInteractionBlocked(state: ControllerInteractionModal
|
||||
state.kikuModalOpen ||
|
||||
state.runtimeOptionsModalOpen ||
|
||||
state.subsyncModalOpen ||
|
||||
Boolean(state.subtitleGenerationModalOpen) ||
|
||||
state.youtubePickerModalOpen ||
|
||||
state.sessionHelpModalOpen
|
||||
);
|
||||
|
||||
@@ -92,6 +92,7 @@ function createEmptyShortcuts(): ConfiguredShortcuts {
|
||||
openRuntimeOptions: null,
|
||||
openJimaku: null,
|
||||
openTsukihime: null,
|
||||
openSubtitleGeneration: null,
|
||||
openSessionHelp: null,
|
||||
openControllerSelect: null,
|
||||
openControllerDebug: null,
|
||||
@@ -1591,6 +1592,28 @@ test('session binding: Ctrl+Alt+S dispatches subsync action locally', async () =
|
||||
}
|
||||
});
|
||||
|
||||
test('session binding: Ctrl+Shift+G dispatches subtitle generation with the sidebar closed', async () => {
|
||||
const { handlers, testGlobals } = createKeyboardHandlerHarness();
|
||||
try {
|
||||
await handlers.setupMpvInputForwarding();
|
||||
handlers.updateSessionBindings([
|
||||
{
|
||||
sourcePath: 'shortcuts.openSubtitleGeneration',
|
||||
originalKey: 'Ctrl+Shift+G',
|
||||
key: { code: 'KeyG', modifiers: ['ctrl', 'shift'] },
|
||||
actionType: 'session-action',
|
||||
actionId: 'openSubtitleGeneration',
|
||||
},
|
||||
]);
|
||||
testGlobals.dispatchKeydown({ key: 'G', code: 'KeyG', ctrlKey: true, shiftKey: true });
|
||||
assert.deepEqual(testGlobals.sessionActions, [
|
||||
{ actionId: 'openSubtitleGeneration', payload: undefined },
|
||||
]);
|
||||
} finally {
|
||||
testGlobals.restore();
|
||||
}
|
||||
});
|
||||
|
||||
test('session binding: Ctrl+Shift+J dispatches jimaku action locally', async () => {
|
||||
const { handlers, testGlobals } = createKeyboardHandlerHarness();
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ export function createKeyboardHandlers(
|
||||
handleRuntimeOptionsKeydown: (e: KeyboardEvent) => boolean;
|
||||
handleCharacterDictionaryKeydown: (e: KeyboardEvent) => boolean;
|
||||
handleSubsyncKeydown: (e: KeyboardEvent) => boolean;
|
||||
handleSubtitleGenerationKeydown?: (e: KeyboardEvent) => boolean;
|
||||
handleKikuKeydown: (e: KeyboardEvent) => boolean;
|
||||
handleJimakuKeydown: (e: KeyboardEvent) => boolean;
|
||||
handleTsukihimeKeydown: (e: KeyboardEvent) => boolean;
|
||||
@@ -1102,6 +1103,11 @@ export function createKeyboardHandlers(
|
||||
);
|
||||
|
||||
document.addEventListener('keydown', (e: KeyboardEvent) => {
|
||||
if (ctx.state.subtitleGenerationModalOpen) {
|
||||
options.handleSubtitleGenerationKeydown?.(e);
|
||||
return;
|
||||
}
|
||||
|
||||
if (ctx.state.mediaTimingReviewModalOpen) {
|
||||
options.handleMediaTimingReviewKeydown(e);
|
||||
return;
|
||||
|
||||
@@ -634,6 +634,134 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
id="subtitleGenerationModal"
|
||||
class="modal hidden"
|
||||
aria-hidden="true"
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby="subtitleGenerationTitle"
|
||||
>
|
||||
<div class="modal-content subtitle-generation-content">
|
||||
<div class="modal-header">
|
||||
<div id="subtitleGenerationTitle" class="modal-title">Generate Japanese subtitles</div>
|
||||
<button id="subtitleGenerationClose" class="modal-close" type="button">Close</button>
|
||||
</div>
|
||||
<div class="modal-body subtitle-generation-body">
|
||||
<p class="subtitle-generation-intro">
|
||||
Turn the current audio track into timed Japanese subtitles. Audio stays on this
|
||||
device.
|
||||
</p>
|
||||
<div class="subtitle-generation-detail">
|
||||
<span class="subtitle-generation-label">Current media</span>
|
||||
<div id="subtitleGenerationMedia">Checking current media...</div>
|
||||
</div>
|
||||
<div class="subtitle-generation-detail">
|
||||
<span class="subtitle-generation-label">Local tools</span>
|
||||
<div id="subtitleGenerationTools">Checking local tools...</div>
|
||||
<p class="subtitle-generation-hint">
|
||||
SubMiner downloads models, not whisper.cpp or FFmpeg. Install them, or set their
|
||||
paths in Settings → Integrations → Japanese Subtitle Generation.
|
||||
</p>
|
||||
</div>
|
||||
<div class="subtitle-generation-detail">
|
||||
<span class="subtitle-generation-label">Speech model</span>
|
||||
<div id="subtitleGenerationModel">Checking local models...</div>
|
||||
<div
|
||||
id="subtitleGenerationModelPicker"
|
||||
class="subtitle-generation-model-picker hidden"
|
||||
>
|
||||
<label for="subtitleGenerationModelSelect">Model</label>
|
||||
<select
|
||||
id="subtitleGenerationModelSelect"
|
||||
aria-describedby="subtitleGenerationModelDescription subtitleGenerationModelRecommendation"
|
||||
></select>
|
||||
<p id="subtitleGenerationModelDescription" class="subtitle-generation-hint"></p>
|
||||
<p id="subtitleGenerationModelRecommendation" class="subtitle-generation-hint">
|
||||
Start with small for a balance of accuracy and CPU time. Larger models need more
|
||||
memory; speed depends on your hardware. Sizes shown are downloads.
|
||||
</p>
|
||||
<p class="subtitle-generation-hint">
|
||||
This choice lasts for this SubMiner session. Set the default model in Settings.
|
||||
</p>
|
||||
</div>
|
||||
<p class="subtitle-generation-hint">
|
||||
Use your own model in Settings → Integrations → Japanese Subtitle Generation → Model
|
||||
Path.
|
||||
</p>
|
||||
<button
|
||||
id="subtitleGenerationDownload"
|
||||
class="kiku-cancel-button hidden"
|
||||
type="button"
|
||||
>
|
||||
Download model
|
||||
</button>
|
||||
</div>
|
||||
<div class="subtitle-generation-detail">
|
||||
<label class="subtitle-generation-vad-toggle" for="subtitleGenerationVadEnabled">
|
||||
<input
|
||||
id="subtitleGenerationVadEnabled"
|
||||
type="checkbox"
|
||||
aria-describedby="subtitleGenerationVadHint"
|
||||
/>
|
||||
Focus on spoken dialogue <span class="subtitle-generation-hint">Optional</span>
|
||||
</label>
|
||||
<div id="subtitleGenerationVadModel" class="subtitle-generation-hint"></div>
|
||||
<p id="subtitleGenerationVadHint" class="subtitle-generation-hint">
|
||||
Keeps uncertain audio to avoid losing dialogue under music. Songs may also be
|
||||
transcribed. Requires whisper.cpp's speech detector.
|
||||
</p>
|
||||
<p class="subtitle-generation-hint">
|
||||
Applies for this session. Set VAD Model Path in Settings to enable it by default.
|
||||
</p>
|
||||
<button
|
||||
id="subtitleGenerationVadDownload"
|
||||
class="kiku-cancel-button hidden"
|
||||
type="button"
|
||||
>
|
||||
Download speech detection model
|
||||
</button>
|
||||
</div>
|
||||
<div id="subtitleGenerationActivity" class="subtitle-generation-activity hidden">
|
||||
<div class="subtitle-generation-progress-heading">
|
||||
<span id="subtitleGenerationStage">Preparing</span>
|
||||
<span id="subtitleGenerationPercent"></span>
|
||||
</div>
|
||||
<progress
|
||||
id="subtitleGenerationProgress"
|
||||
max="100"
|
||||
aria-label="Subtitle generation progress"
|
||||
></progress>
|
||||
</div>
|
||||
<div
|
||||
id="subtitleGenerationStatus"
|
||||
class="runtime-options-status"
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
></div>
|
||||
<div class="subtitle-generation-actions">
|
||||
<button id="subtitleGenerationRefresh" class="kiku-cancel-button" type="button">
|
||||
Check again
|
||||
</button>
|
||||
<button id="subtitleGenerationCancel" class="kiku-cancel-button hidden" type="button">
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
id="subtitleGenerationStart"
|
||||
class="kiku-confirm-button"
|
||||
type="button"
|
||||
disabled
|
||||
>
|
||||
Generate subtitles
|
||||
</button>
|
||||
</div>
|
||||
<p class="subtitle-generation-hint">
|
||||
The generated SRT will be saved locally and loaded into the player. You can close this
|
||||
window while it runs.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="subsyncModal" class="modal hidden" aria-hidden="true">
|
||||
<div class="modal-content subsync-modal-content">
|
||||
<div class="modal-header">
|
||||
@@ -735,6 +863,13 @@
|
||||
<button id="subtitleSidebarClose" class="modal-close" type="button">Close</button>
|
||||
</div>
|
||||
<div class="modal-body subtitle-sidebar-body">
|
||||
<button
|
||||
id="subtitleGenerationOpen"
|
||||
class="kiku-cancel-button subtitle-generation-open"
|
||||
type="button"
|
||||
>
|
||||
Generate Japanese subtitles
|
||||
</button>
|
||||
<div id="subtitleSidebarStatus" class="runtime-options-status"></div>
|
||||
<ul id="subtitleSidebarList" class="subtitle-sidebar-list"></ul>
|
||||
</div>
|
||||
|
||||
@@ -225,6 +225,8 @@ function describeSessionAction(
|
||||
return 'Open jimaku';
|
||||
case 'openTsukihime':
|
||||
return 'Open TsukiHime';
|
||||
case 'openSubtitleGeneration':
|
||||
return 'Generate Japanese subtitles';
|
||||
case 'openYoutubePicker':
|
||||
return 'Open YouTube subtitle picker';
|
||||
case 'openPlaylistBrowser':
|
||||
@@ -266,6 +268,7 @@ function sectionForSessionBinding(binding: CompiledSessionBinding): string {
|
||||
case 'openJimaku':
|
||||
case 'openTsukihime':
|
||||
case 'openCharacterDictionaryManager':
|
||||
case 'openSubtitleGeneration':
|
||||
case 'openControllerSelect':
|
||||
case 'openControllerDebug':
|
||||
case 'openYoutubePicker':
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
describeGenerationModel,
|
||||
describeGenerationProgress,
|
||||
describeGenerationTools,
|
||||
describeGenerationVad,
|
||||
} from './subtitle-generation-view';
|
||||
|
||||
test('missing tools block generation and list every install instruction', () => {
|
||||
const found = { kind: 'found', path: '/usr/bin/tool' } as const;
|
||||
assert.deepEqual(
|
||||
describeGenerationTools({ ffmpeg: found, ffprobe: found, whisper: found, vad: null }),
|
||||
{
|
||||
ready: true,
|
||||
text: 'whisper.cpp and FFmpeg are installed.',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
describeGenerationTools({
|
||||
ffmpeg: { kind: 'missing', message: 'ffmpeg was not found on PATH.' },
|
||||
ffprobe: found,
|
||||
whisper: found,
|
||||
vad: { kind: 'missing', message: 'whisper-vad-speech-segments was not found on PATH.' },
|
||||
}),
|
||||
{
|
||||
ready: false,
|
||||
text: 'ffmpeg was not found on PATH. whisper-vad-speech-segments was not found on PATH.',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('only a missing managed model offers a download', () => {
|
||||
assert.deepEqual(describeGenerationModel({ kind: 'missing', path: '/models/small.bin' }), {
|
||||
ready: false,
|
||||
download: true,
|
||||
text: 'Download a speech model to get started.',
|
||||
});
|
||||
for (const kind of ['managed', 'external'] as const) {
|
||||
const model = describeGenerationModel({ kind, path: '/models/ggml-small.bin' });
|
||||
assert.equal(model.ready, true);
|
||||
assert.equal(model.download, false);
|
||||
}
|
||||
assert.deepEqual(
|
||||
describeGenerationModel({
|
||||
kind: 'invalid',
|
||||
path: '/missing/model.bin',
|
||||
message: 'Configured model does not exist.',
|
||||
}),
|
||||
{
|
||||
ready: false,
|
||||
download: false,
|
||||
text: 'Configured model does not exist.',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('generation progress distinguishes measured work from indeterminate stages', () => {
|
||||
assert.deepEqual(
|
||||
describeGenerationProgress({ stage: 'transcribe', percent: 42.8, message: 'Transcribing' }),
|
||||
{
|
||||
stage: 'Recognizing Japanese speech',
|
||||
percent: 42.8,
|
||||
label: '42%',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(describeGenerationProgress({ stage: 'extract', message: 'Extracting audio' }), {
|
||||
stage: 'Preparing audio',
|
||||
percent: null,
|
||||
label: 'Working...',
|
||||
});
|
||||
assert.equal(
|
||||
describeGenerationProgress({ stage: 'download', percent: 0, message: '' }).label,
|
||||
'0%',
|
||||
);
|
||||
assert.equal(
|
||||
describeGenerationProgress({ stage: 'download', percent: Number.NaN, message: '' }).percent,
|
||||
null,
|
||||
);
|
||||
assert.equal(
|
||||
describeGenerationProgress({ stage: 'write', percent: 120, message: '' }).percent,
|
||||
100,
|
||||
);
|
||||
});
|
||||
|
||||
test('optional speech detection only gates generation when selected', () => {
|
||||
const missing = { kind: 'missing', path: '/vad.bin' } as const;
|
||||
assert.equal(describeGenerationVad({ enabled: false, model: missing }).ready, true);
|
||||
assert.equal(describeGenerationVad({ enabled: false, model: missing }).download, false);
|
||||
assert.equal(describeGenerationVad({ enabled: true, model: missing }).ready, false);
|
||||
assert.equal(describeGenerationVad({ enabled: true, model: missing }).download, true);
|
||||
assert.equal(
|
||||
describeGenerationVad({ enabled: true, model: { kind: 'managed', path: '/vad.bin' } }).ready,
|
||||
true,
|
||||
);
|
||||
const invalid = { kind: 'invalid', path: '/vad.bin', message: 'Cannot read model' } as const;
|
||||
assert.equal(describeGenerationVad({ enabled: true, model: invalid }).download, false);
|
||||
assert.equal(describeGenerationVad({ enabled: false, model: invalid }).ready, true);
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import {
|
||||
missingSubtitleGenerationTools,
|
||||
type SubtitleGenerationModelStatus,
|
||||
type SubtitleGenerationProgress,
|
||||
type SubtitleGenerationTools,
|
||||
} from '../../shared/subtitle-generation';
|
||||
import type { SubtitleGenerationStatus } from '../../shared/subtitle-generation-ipc';
|
||||
|
||||
export function describeGenerationTools(tools: SubtitleGenerationTools) {
|
||||
const missing = missingSubtitleGenerationTools(tools);
|
||||
if (missing.length > 0) return { ready: false, text: missing.join(' ') };
|
||||
return {
|
||||
ready: true,
|
||||
text: tools.vad
|
||||
? 'whisper.cpp, its speech detector, and FFmpeg are installed.'
|
||||
: 'whisper.cpp and FFmpeg are installed.',
|
||||
};
|
||||
}
|
||||
|
||||
export function describeGenerationVad(vad: SubtitleGenerationStatus['vad']) {
|
||||
const model = describeGenerationModel(vad.model);
|
||||
return {
|
||||
ready: !vad.enabled || model.ready,
|
||||
download: vad.enabled && model.download,
|
||||
text: !vad.enabled
|
||||
? 'Optional. Generate from the full audio when unchecked.'
|
||||
: vad.model.kind === 'missing'
|
||||
? 'Download the speech detection model to focus on spoken dialogue.'
|
||||
: vad.model.kind === 'invalid'
|
||||
? vad.model.message
|
||||
: vad.model.kind === 'external'
|
||||
? `Your speech detection model: ${vad.model.path}`
|
||||
: 'Silero speech detection model installed.',
|
||||
};
|
||||
}
|
||||
|
||||
export function describeGenerationModel(model: SubtitleGenerationModelStatus) {
|
||||
switch (model.kind) {
|
||||
case 'external':
|
||||
return { ready: true, download: false, text: `Your model: ${model.path}` };
|
||||
case 'managed':
|
||||
return { ready: true, download: false, text: `SubMiner model: ${model.path}` };
|
||||
case 'missing':
|
||||
return { ready: false, download: true, text: 'Download a speech model to get started.' };
|
||||
case 'invalid':
|
||||
return { ready: false, download: false, text: model.message };
|
||||
default: {
|
||||
const exhaustive: never = model;
|
||||
return exhaustive;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const STAGE_LABELS = {
|
||||
download: 'Downloading speech model',
|
||||
extract: 'Preparing audio',
|
||||
transcribe: 'Recognizing Japanese speech',
|
||||
write: 'Saving subtitles',
|
||||
} satisfies Record<SubtitleGenerationProgress['stage'], string>;
|
||||
|
||||
export function describeGenerationProgress(progress: SubtitleGenerationProgress | null) {
|
||||
const raw = progress?.percent;
|
||||
const percent =
|
||||
raw !== undefined && Number.isFinite(raw) ? Math.max(0, Math.min(100, raw)) : null;
|
||||
return {
|
||||
stage: progress ? STAGE_LABELS[progress.stage] : 'Preparing',
|
||||
percent,
|
||||
label: percent === null ? 'Working...' : `${Math.floor(percent)}%`,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
import type { SubtitleGenerationProgress } from '../../shared/subtitle-generation';
|
||||
import {
|
||||
SUBTITLE_GENERATION_MODELS,
|
||||
RECOMMENDED_SUBTITLE_GENERATION_MODEL,
|
||||
formatSubtitleGenerationModelSize,
|
||||
getSubtitleGenerationModel,
|
||||
isSubtitleGenerationModelId,
|
||||
} from '../../shared/subtitle-generation-model-catalog';
|
||||
import type {
|
||||
SubtitleGenerationResult,
|
||||
SubtitleGenerationStatus,
|
||||
} from '../../shared/subtitle-generation-ipc';
|
||||
import type { ModalStateReader, RendererContext } from '../context';
|
||||
import { syncOverlayMouseIgnoreState } from '../overlay-mouse-ignore';
|
||||
import { createModalFocusGuard } from './modal-focus-guard';
|
||||
import {
|
||||
describeGenerationModel,
|
||||
describeGenerationProgress,
|
||||
describeGenerationTools,
|
||||
describeGenerationVad,
|
||||
} from './subtitle-generation-view';
|
||||
import { SUBTITLE_GENERATION_VAD_MODEL } from '../../shared/subtitle-generation-vad-model';
|
||||
|
||||
function element<T extends HTMLElement>(id: string, constructor: new () => T): T {
|
||||
const node = document.getElementById(id);
|
||||
if (!(node instanceof constructor)) throw new Error(`Missing subtitle generation element: ${id}`);
|
||||
return node;
|
||||
}
|
||||
|
||||
export function createSubtitleGenerationModal(
|
||||
ctx: RendererContext,
|
||||
options: {
|
||||
modalStateReader: Pick<ModalStateReader, 'isAnyModalOpen'>;
|
||||
syncSettingsModalSubtitleSuppression: () => void;
|
||||
},
|
||||
) {
|
||||
const dom = {
|
||||
modal: element('subtitleGenerationModal', HTMLDivElement),
|
||||
close: element('subtitleGenerationClose', HTMLButtonElement),
|
||||
open: element('subtitleGenerationOpen', HTMLButtonElement),
|
||||
media: element('subtitleGenerationMedia', HTMLDivElement),
|
||||
tools: element('subtitleGenerationTools', HTMLDivElement),
|
||||
model: element('subtitleGenerationModel', HTMLDivElement),
|
||||
modelPicker: element('subtitleGenerationModelPicker', HTMLDivElement),
|
||||
modelSelect: element('subtitleGenerationModelSelect', HTMLSelectElement),
|
||||
modelDescription: element('subtitleGenerationModelDescription', HTMLParagraphElement),
|
||||
download: element('subtitleGenerationDownload', HTMLButtonElement),
|
||||
vadEnabled: element('subtitleGenerationVadEnabled', HTMLInputElement),
|
||||
vadModel: element('subtitleGenerationVadModel', HTMLDivElement),
|
||||
vadDownload: element('subtitleGenerationVadDownload', HTMLButtonElement),
|
||||
activity: element('subtitleGenerationActivity', HTMLDivElement),
|
||||
stage: element('subtitleGenerationStage', HTMLSpanElement),
|
||||
percent: element('subtitleGenerationPercent', HTMLSpanElement),
|
||||
progress: element('subtitleGenerationProgress', HTMLProgressElement),
|
||||
status: element('subtitleGenerationStatus', HTMLDivElement),
|
||||
refresh: element('subtitleGenerationRefresh', HTMLButtonElement),
|
||||
cancel: element('subtitleGenerationCancel', HTMLButtonElement),
|
||||
start: element('subtitleGenerationStart', HTMLButtonElement),
|
||||
};
|
||||
let snapshot: SubtitleGenerationStatus | null = null;
|
||||
let progress: SubtitleGenerationProgress | null = null;
|
||||
let result: SubtitleGenerationResult | null = null;
|
||||
let pending = false;
|
||||
let cancelling = false;
|
||||
let checking = false;
|
||||
let error: string | null = null;
|
||||
let priorFocus: Element | null = null;
|
||||
let poll: ReturnType<typeof setTimeout> | null = null;
|
||||
let unsubscribe: (() => void) | null = null;
|
||||
for (const model of SUBTITLE_GENERATION_MODELS) {
|
||||
const option = document.createElement('option');
|
||||
option.value = model.id;
|
||||
const recommended = model.id === RECOMMENDED_SUBTITLE_GENERATION_MODEL ? ' (recommended)' : '';
|
||||
option.textContent = `${model.id}${recommended} · ${formatSubtitleGenerationModelSize(model.size)}`;
|
||||
dom.modelSelect.append(option);
|
||||
}
|
||||
|
||||
const focus = createModalFocusGuard({
|
||||
isOpen: () => ctx.state.subtitleGenerationModalOpen,
|
||||
getModalRoot: () => dom.modal,
|
||||
getPreferredFocusTargets: () => [dom.modelSelect, dom.download, dom.start],
|
||||
getFallbackFocusTarget: () => dom.close,
|
||||
isModalLayer: ctx.platform.isModalLayer,
|
||||
});
|
||||
|
||||
function render(): void {
|
||||
const busy = pending || Boolean(snapshot?.running);
|
||||
const model = snapshot ? describeGenerationModel(snapshot.model) : null;
|
||||
const vad = snapshot ? describeGenerationVad(snapshot.vad) : null;
|
||||
const tools = snapshot ? describeGenerationTools(snapshot.tools) : null;
|
||||
const readyMessage = !snapshot?.mediaPath
|
||||
? 'Open local media to generate subtitles.'
|
||||
: !tools?.ready
|
||||
? 'Install the missing tools or set their paths in Settings, then click Check again.'
|
||||
: !model?.ready
|
||||
? 'Set up a speech model to continue.'
|
||||
: !vad?.ready
|
||||
? 'Download the speech detection model or uncheck Focus on spoken dialogue.'
|
||||
: 'Ready when you are.';
|
||||
dom.media.textContent = snapshot?.mediaPath ?? 'Open a local media file in the player first.';
|
||||
dom.tools.textContent = tools?.text ?? 'Checking local tools...';
|
||||
dom.model.textContent = model?.text ?? 'Checking local models...';
|
||||
dom.modelPicker.classList.toggle('hidden', !snapshot || Boolean(snapshot.externalModelPath));
|
||||
dom.modelSelect.disabled = busy || checking || !snapshot || Boolean(snapshot.externalModelPath);
|
||||
if (snapshot) {
|
||||
dom.modelSelect.value = snapshot.managedModel;
|
||||
dom.modelDescription.textContent = getSubtitleGenerationModel(
|
||||
snapshot.managedModel,
|
||||
).description;
|
||||
}
|
||||
dom.download.classList.toggle('hidden', !model?.download);
|
||||
dom.download.textContent = snapshot
|
||||
? `Download ${snapshot.managedModel} model`
|
||||
: 'Download model';
|
||||
dom.download.disabled = busy || checking;
|
||||
if (!checking) dom.vadEnabled.checked = snapshot?.vad.enabled ?? false;
|
||||
dom.vadEnabled.disabled = busy || checking || !snapshot;
|
||||
dom.vadModel.textContent = vad?.text ?? 'Checking speech detection...';
|
||||
dom.vadDownload.classList.toggle('hidden', !vad?.download);
|
||||
dom.vadDownload.textContent = `Download speech detection model · ${formatSubtitleGenerationModelSize(SUBTITLE_GENERATION_VAD_MODEL.size)}`;
|
||||
dom.vadDownload.disabled = busy || checking;
|
||||
dom.start.disabled =
|
||||
busy || checking || !tools?.ready || !model?.ready || !vad?.ready || !snapshot?.mediaPath;
|
||||
dom.refresh.disabled = busy || checking;
|
||||
dom.cancel.classList.toggle('hidden', !busy);
|
||||
dom.cancel.disabled = cancelling;
|
||||
dom.cancel.textContent = cancelling ? 'Cancelling...' : 'Cancel';
|
||||
dom.activity.classList.toggle('hidden', !busy);
|
||||
const activity = describeGenerationProgress(progress);
|
||||
dom.stage.textContent = activity.stage;
|
||||
dom.percent.textContent = activity.label;
|
||||
if (activity.percent === null) dom.progress.removeAttribute('value');
|
||||
else dom.progress.value = activity.percent;
|
||||
dom.status.classList.toggle('error', Boolean(error || (!busy && result && !result.ok)));
|
||||
dom.status.textContent =
|
||||
error ??
|
||||
(busy
|
||||
? cancelling
|
||||
? 'Stopping the current operation...'
|
||||
: (progress?.message ?? 'Starting...')
|
||||
: (result?.message ?? (checking ? 'Checking local setup...' : readyMessage)));
|
||||
}
|
||||
|
||||
function stopPolling(): void {
|
||||
if (poll !== null) clearTimeout(poll);
|
||||
poll = null;
|
||||
}
|
||||
|
||||
async function refresh(): Promise<void> {
|
||||
if (checking) return;
|
||||
checking = true;
|
||||
error = null;
|
||||
render();
|
||||
try {
|
||||
snapshot = await window.electronAPI.getSubtitleGenerationStatus();
|
||||
if (!pending) {
|
||||
progress = snapshot.progress;
|
||||
result = snapshot.lastResult ?? result;
|
||||
}
|
||||
} catch (cause) {
|
||||
error = cause instanceof Error ? cause.message : 'Could not check subtitle generation setup.';
|
||||
} finally {
|
||||
checking = false;
|
||||
render();
|
||||
stopPolling();
|
||||
if (ctx.state.subtitleGenerationModalOpen && snapshot?.running && !pending) {
|
||||
poll = setTimeout(() => void refresh(), 1500);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function run(action: 'download' | 'download-vad' | 'generate'): Promise<void> {
|
||||
if (pending || snapshot?.running || checking || !snapshot) return;
|
||||
const model = describeGenerationModel(snapshot.model);
|
||||
const vad = describeGenerationVad(snapshot.vad);
|
||||
const tools = describeGenerationTools(snapshot.tools);
|
||||
if (
|
||||
action === 'download'
|
||||
? !model.download
|
||||
: action === 'download-vad'
|
||||
? !vad.download
|
||||
: !tools.ready || !model.ready || !vad.ready || !snapshot.mediaPath
|
||||
)
|
||||
return;
|
||||
pending = true;
|
||||
result = null;
|
||||
error = null;
|
||||
progress = { stage: action === 'generate' ? 'extract' : 'download', message: 'Starting...' };
|
||||
render();
|
||||
try {
|
||||
result = await (action === 'download'
|
||||
? window.electronAPI.downloadSubtitleGenerationModel()
|
||||
: action === 'download-vad'
|
||||
? window.electronAPI.downloadSubtitleGenerationVadModel()
|
||||
: window.electronAPI.startSubtitleGeneration());
|
||||
} catch (cause) {
|
||||
result = {
|
||||
ok: false,
|
||||
message: cause instanceof Error ? cause.message : 'Subtitle generation failed.',
|
||||
};
|
||||
} finally {
|
||||
pending = false;
|
||||
cancelling = false;
|
||||
// Recheck model availability after downloads and recover controls after errors.
|
||||
await refresh();
|
||||
}
|
||||
}
|
||||
|
||||
async function selectModel(): Promise<void> {
|
||||
const model = dom.modelSelect.value;
|
||||
if (pending || snapshot?.running || checking || !isSubtitleGenerationModelId(model)) return;
|
||||
checking = true;
|
||||
error = null;
|
||||
render();
|
||||
try {
|
||||
snapshot = await window.electronAPI.selectSubtitleGenerationModel(model);
|
||||
result = snapshot.lastResult;
|
||||
progress = snapshot.progress;
|
||||
} catch (cause) {
|
||||
error = cause instanceof Error ? cause.message : 'Could not select the model.';
|
||||
} finally {
|
||||
checking = false;
|
||||
render();
|
||||
}
|
||||
}
|
||||
|
||||
async function selectVad(): Promise<void> {
|
||||
if (pending || snapshot?.running || checking || !snapshot) return;
|
||||
const enabled = dom.vadEnabled.checked;
|
||||
checking = true;
|
||||
error = null;
|
||||
render();
|
||||
try {
|
||||
snapshot = await window.electronAPI.setSubtitleGenerationVadEnabled(enabled);
|
||||
result = snapshot.lastResult;
|
||||
progress = snapshot.progress;
|
||||
} catch (cause) {
|
||||
error = cause instanceof Error ? cause.message : 'Could not change speech detection.';
|
||||
} finally {
|
||||
checking = false;
|
||||
render();
|
||||
}
|
||||
}
|
||||
|
||||
async function cancel(): Promise<void> {
|
||||
if (cancelling) return;
|
||||
cancelling = true;
|
||||
render();
|
||||
try {
|
||||
await window.electronAPI.cancelSubtitleGeneration();
|
||||
await refresh();
|
||||
} catch (cause) {
|
||||
error = cause instanceof Error ? cause.message : 'Could not cancel the operation.';
|
||||
} finally {
|
||||
cancelling = false;
|
||||
render();
|
||||
}
|
||||
}
|
||||
|
||||
function open(): void {
|
||||
if (ctx.state.subtitleGenerationModalOpen) return;
|
||||
priorFocus = document.activeElement;
|
||||
ctx.state.subtitleGenerationModalOpen = true;
|
||||
options.syncSettingsModalSubtitleSuppression();
|
||||
dom.modal.classList.remove('hidden');
|
||||
dom.modal.setAttribute('aria-hidden', 'false');
|
||||
syncOverlayMouseIgnoreState(ctx);
|
||||
focus.attach();
|
||||
focus.focusFallbackTarget();
|
||||
window.electronAPI.notifyOverlayModalOpened('subtitle-generation');
|
||||
void refresh();
|
||||
}
|
||||
|
||||
function close(): void {
|
||||
if (!ctx.state.subtitleGenerationModalOpen) return;
|
||||
ctx.state.subtitleGenerationModalOpen = false;
|
||||
options.syncSettingsModalSubtitleSuppression();
|
||||
dom.modal.classList.add('hidden');
|
||||
dom.modal.setAttribute('aria-hidden', 'true');
|
||||
focus.detach();
|
||||
stopPolling();
|
||||
window.electronAPI.notifyOverlayModalClosed('subtitle-generation');
|
||||
if (priorFocus instanceof HTMLElement && priorFocus.isConnected)
|
||||
priorFocus.focus({ preventScroll: true });
|
||||
if (!options.modalStateReader.isAnyModalOpen()) syncOverlayMouseIgnoreState(ctx);
|
||||
}
|
||||
|
||||
function handleKeydown(event: KeyboardEvent): boolean {
|
||||
if (!ctx.state.subtitleGenerationModalOpen) return false;
|
||||
if (event.key === 'Escape') {
|
||||
event.preventDefault();
|
||||
close();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function wireDomEvents(): void {
|
||||
dom.close.addEventListener('click', close);
|
||||
dom.open.addEventListener('click', () => {
|
||||
void window.electronAPI
|
||||
.requestSubtitleGenerationOpen()
|
||||
.then((opened) => {
|
||||
if (!opened)
|
||||
ctx.dom.subtitleSidebarStatus.textContent = 'Could not open subtitle generation.';
|
||||
})
|
||||
.catch((cause: unknown) => {
|
||||
ctx.dom.subtitleSidebarStatus.textContent =
|
||||
cause instanceof Error ? cause.message : 'Could not open subtitle generation.';
|
||||
});
|
||||
});
|
||||
dom.download.addEventListener('click', () => void run('download'));
|
||||
dom.vadDownload.addEventListener('click', () => void run('download-vad'));
|
||||
dom.vadEnabled.addEventListener('change', () => void selectVad());
|
||||
dom.modelSelect.addEventListener('change', () => void selectModel());
|
||||
dom.start.addEventListener('click', () => void run('generate'));
|
||||
dom.cancel.addEventListener('click', () => void cancel());
|
||||
dom.refresh.addEventListener('click', () => void refresh());
|
||||
unsubscribe = window.electronAPI.onSubtitleGenerationProgress((update) => {
|
||||
progress = update;
|
||||
render();
|
||||
});
|
||||
}
|
||||
|
||||
function dispose(): void {
|
||||
stopPolling();
|
||||
focus.detach();
|
||||
unsubscribe?.();
|
||||
unsubscribe = null;
|
||||
}
|
||||
|
||||
return { open, close, handleKeydown, wireDomEvents, dispose };
|
||||
}
|
||||
@@ -113,6 +113,20 @@ test('findActiveSubtitleCueIndex prefers current subtitle timing over near-futur
|
||||
assert.equal(findActiveSubtitleCueIndex(cues, { text: 'previous', startTime: 231 }, 233, 0), 0);
|
||||
});
|
||||
|
||||
test('findActiveSubtitleCueIndex follows playback through empty subtitle gaps', () => {
|
||||
const cues = [
|
||||
{ startTime: 0, endTime: 2, text: 'first' },
|
||||
{ startTime: 100, endTime: 102, text: 'later' },
|
||||
{ startTime: 105, endTime: 107, text: 'next' },
|
||||
];
|
||||
|
||||
assert.equal(findActiveSubtitleCueIndex(cues, { text: 'later', startTime: 100 }, 101, 1), 1);
|
||||
assert.equal(findActiveSubtitleCueIndex(cues, { text: '', startTime: 0 }, 103, 1), 2);
|
||||
assert.equal(findActiveSubtitleCueIndex(cues, { text: 'next', startTime: 105 }, 105, 2), 2);
|
||||
assert.equal(findActiveSubtitleCueIndex(cues, { text: '', startTime: 0 }, 108, 2), -1);
|
||||
assert.equal(findActiveSubtitleCueIndex(cues, { text: 'first', startTime: 0 }, 0, 2), 0);
|
||||
});
|
||||
|
||||
test('subtitle sidebar mining context resolves selected row cue timing', () => {
|
||||
const globals = globalThis as typeof globalThis & {
|
||||
Element?: unknown;
|
||||
|
||||
@@ -120,8 +120,12 @@ export function findActiveSubtitleCueIndex(
|
||||
return -1;
|
||||
}
|
||||
|
||||
// The mpv client maps cleared sub-start to zero. Empty text has no active cue timing.
|
||||
const hasCurrentTiming =
|
||||
typeof current?.startTime === 'number' && Number.isFinite(current.startTime);
|
||||
current !== null &&
|
||||
normalizeCueText(current.text).length > 0 &&
|
||||
typeof current.startTime === 'number' &&
|
||||
Number.isFinite(current.startTime);
|
||||
|
||||
if (hasCurrentTiming) {
|
||||
const timingMatch = cues.findIndex(
|
||||
|
||||
@@ -11,6 +11,7 @@ function isBlockingOverlayModalOpen(state: RendererState): boolean {
|
||||
state.kikuModalOpen ||
|
||||
state.runtimeOptionsModalOpen ||
|
||||
state.subsyncModalOpen ||
|
||||
state.subtitleGenerationModalOpen ||
|
||||
state.sessionHelpModalOpen,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -45,6 +45,7 @@ import { isControllerInteractionBlocked } from './controller-interaction-blockin
|
||||
import { createCharacterDictionaryModal } from './modals/character-dictionary.js';
|
||||
import { createRuntimeOptionsModal } from './modals/runtime-options.js';
|
||||
import { createSubsyncModal } from './modals/subsync.js';
|
||||
import { createSubtitleGenerationModal } from './modals/subtitle-generation.js';
|
||||
import { createYoutubeTrackPickerModal } from './modals/youtube-track-picker.js';
|
||||
import { createMediaTimingReviewModal } from './modals/media-timing-review.js';
|
||||
import { createPositioningController } from './positioning.js';
|
||||
@@ -80,6 +81,12 @@ const ctx = {
|
||||
};
|
||||
|
||||
const modalDescriptors = [
|
||||
{
|
||||
id: 'subtitle-generation',
|
||||
isOpen: () => ctx.state.subtitleGenerationModalOpen,
|
||||
close: () => subtitleGenerationModal.close(),
|
||||
suppressesSubtitles: true,
|
||||
},
|
||||
{
|
||||
id: 'controller-select',
|
||||
isOpen: () => ctx.state.controllerSelectModalOpen,
|
||||
@@ -213,6 +220,10 @@ const subsyncModal = createSubsyncModal(ctx, {
|
||||
modalStateReader: { isAnyModalOpen },
|
||||
syncSettingsModalSubtitleSuppression,
|
||||
});
|
||||
const subtitleGenerationModal = createSubtitleGenerationModal(ctx, {
|
||||
modalStateReader: { isAnyModalOpen },
|
||||
syncSettingsModalSubtitleSuppression,
|
||||
});
|
||||
const controllerSelectModal = createControllerSelectModal(ctx, {
|
||||
modalStateReader: { isAnyModalOpen },
|
||||
syncSettingsModalSubtitleSuppression,
|
||||
@@ -283,6 +294,7 @@ const keyboardHandlers = createKeyboardHandlers(ctx, {
|
||||
handleRuntimeOptionsKeydown: runtimeOptionsModal.handleRuntimeOptionsKeydown,
|
||||
handleCharacterDictionaryKeydown: characterDictionaryModal.handleCharacterDictionaryKeydown,
|
||||
handleSubsyncKeydown: subsyncModal.handleSubsyncKeydown,
|
||||
handleSubtitleGenerationKeydown: subtitleGenerationModal.handleKeydown,
|
||||
handleKikuKeydown: kikuModal.handleKikuKeydown,
|
||||
handleJimakuKeydown: jimakuModal.handleJimakuKeydown,
|
||||
handleTsukihimeKeydown: tsukihimeModal.handleTsukihimeKeydown,
|
||||
@@ -535,6 +547,9 @@ const recovery = createRendererRecoveryController({
|
||||
registerRendererGlobalErrorHandlers(window, recovery);
|
||||
|
||||
function registerModalOpenHandlers(): void {
|
||||
window.electronAPI.onSubtitleGenerationOpen(() => {
|
||||
runGuarded('subtitle-generation:open', () => subtitleGenerationModal.open());
|
||||
});
|
||||
window.electronAPI.onOpenRuntimeOptions(() => {
|
||||
runGuarded('runtime-options:open', () => {
|
||||
runtimeOptionsModal.openRuntimeOptionsModal();
|
||||
@@ -835,6 +850,7 @@ async function init(): Promise<void> {
|
||||
kikuModal.wireDomEvents();
|
||||
runtimeOptionsModal.wireDomEvents();
|
||||
subsyncModal.wireDomEvents();
|
||||
subtitleGenerationModal.wireDomEvents();
|
||||
controllerSelectModal.wireDomEvents();
|
||||
controllerDebugModal.wireDomEvents();
|
||||
sessionHelpModal.wireDomEvents();
|
||||
@@ -842,6 +858,7 @@ async function init(): Promise<void> {
|
||||
subtitleSidebarModal.wireDomEvents();
|
||||
characterDictionaryModal.wireDomEvents();
|
||||
window.addEventListener('beforeunload', () => {
|
||||
subtitleGenerationModal.dispose();
|
||||
subtitleSidebarModal.disposeDomEvents();
|
||||
});
|
||||
|
||||
|
||||
@@ -87,6 +87,7 @@ export type RendererState = {
|
||||
characterDictionaryStatus: string;
|
||||
|
||||
subsyncModalOpen: boolean;
|
||||
subtitleGenerationModalOpen: boolean;
|
||||
subsyncSubtitleTracks: SubsyncSubtitleTrack[];
|
||||
subsyncSubmitting: boolean;
|
||||
|
||||
@@ -219,6 +220,7 @@ export function createRendererState(): RendererState {
|
||||
characterDictionaryStatus: '',
|
||||
|
||||
subsyncModalOpen: false,
|
||||
subtitleGenerationModalOpen: false,
|
||||
subsyncSubtitleTracks: [],
|
||||
subsyncSubmitting: false,
|
||||
|
||||
|
||||
@@ -2976,6 +2976,142 @@ iframe[id^='yomitan-popup'],
|
||||
width: min(560px, 92%);
|
||||
}
|
||||
|
||||
.subtitle-generation-content {
|
||||
width: min(580px, 92%);
|
||||
max-height: 92%;
|
||||
border-top: 3px solid var(--ctp-green);
|
||||
}
|
||||
|
||||
.subtitle-generation-body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 18px;
|
||||
overflow-y: auto;
|
||||
scrollbar-width: thin;
|
||||
scrollbar-color: var(--ctp-surface1) transparent;
|
||||
}
|
||||
|
||||
.subtitle-generation-body > * {
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.subtitle-generation-intro {
|
||||
margin: 0;
|
||||
color: var(--ctp-subtext1);
|
||||
font-size: 14px;
|
||||
line-height: 1.65;
|
||||
}
|
||||
|
||||
.subtitle-generation-detail {
|
||||
padding: 12px 14px;
|
||||
border-left: 2px solid var(--ctp-surface1);
|
||||
background: rgba(24, 25, 38, 0.35);
|
||||
overflow-wrap: anywhere;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.subtitle-generation-label {
|
||||
display: block;
|
||||
margin-bottom: 6px;
|
||||
color: var(--ctp-green);
|
||||
font-size: 11px;
|
||||
letter-spacing: 0.06em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.subtitle-generation-hint {
|
||||
margin: 8px 0 0;
|
||||
color: var(--ctp-subtext0);
|
||||
font-size: 12px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.subtitle-generation-model-picker {
|
||||
margin-top: 14px;
|
||||
}
|
||||
|
||||
.subtitle-generation-model-picker label {
|
||||
display: block;
|
||||
margin-bottom: 6px;
|
||||
color: var(--ctp-subtext1);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.subtitle-generation-model-picker select {
|
||||
width: 100%;
|
||||
padding: 8px 10px;
|
||||
border: 1px solid var(--ctp-surface1);
|
||||
border-radius: 6px;
|
||||
background: var(--ctp-mantle);
|
||||
color: var(--ctp-text);
|
||||
font: inherit;
|
||||
}
|
||||
|
||||
.subtitle-generation-model-picker select:focus-visible {
|
||||
outline: 2px solid var(--ctp-green);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
.subtitle-generation-model-picker select:disabled {
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
.subtitle-generation-detail button {
|
||||
margin-top: 12px;
|
||||
}
|
||||
|
||||
.subtitle-generation-vad-toggle {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.subtitle-generation-vad-toggle input {
|
||||
accent-color: var(--ctp-green);
|
||||
}
|
||||
|
||||
.subtitle-generation-vad-toggle .subtitle-generation-hint {
|
||||
margin: 0 0 0 auto;
|
||||
}
|
||||
|
||||
.subtitle-generation-progress-heading {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 9px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.subtitle-generation-activity progress {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: 8px;
|
||||
accent-color: var(--ctp-green);
|
||||
}
|
||||
|
||||
.subtitle-generation-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: flex-end;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.subtitle-generation-actions button:disabled,
|
||||
.subtitle-generation-detail button:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.subtitle-generation-open {
|
||||
align-self: flex-start;
|
||||
margin: 0 16px 8px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.subtitle-sidebar-body:has(.subtitle-sidebar-item) .subtitle-generation-open {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.subsync-form {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { RuntimeOptionId, RuntimeOptionValue } from '../../types/runtime-op
|
||||
export const OVERLAY_HOSTED_MODALS = [
|
||||
'runtime-options',
|
||||
'subsync',
|
||||
'subtitle-generation',
|
||||
'jimaku',
|
||||
'tsukihime',
|
||||
'youtube-track-picker',
|
||||
@@ -50,6 +51,14 @@ export const IPC_CHANNELS = {
|
||||
dispatchSessionAction: 'session-action:dispatch',
|
||||
},
|
||||
request: {
|
||||
requestSubtitleGenerationOpen: 'subtitle-generation:open',
|
||||
getSubtitleGenerationStatus: 'subtitle-generation:status',
|
||||
startSubtitleGeneration: 'subtitle-generation:start',
|
||||
selectSubtitleGenerationModel: 'subtitle-generation:select-model',
|
||||
downloadSubtitleGenerationModel: 'subtitle-generation:download',
|
||||
downloadSubtitleGenerationVadModel: 'subtitle-generation:download-vad',
|
||||
setSubtitleGenerationVadEnabled: 'subtitle-generation:set-vad-enabled',
|
||||
cancelSubtitleGeneration: 'subtitle-generation:cancel',
|
||||
getVisibleOverlayVisibility: 'get-visible-overlay-visibility',
|
||||
getCurrentSubtitle: 'get-current-subtitle',
|
||||
getCurrentSubtitleRaw: 'get-current-subtitle-raw',
|
||||
@@ -133,6 +142,8 @@ export const IPC_CHANNELS = {
|
||||
mediaTimingReviewResolve: 'media-timing-review:resolve',
|
||||
},
|
||||
event: {
|
||||
subtitleGenerationOpen: 'subtitle-generation:opened',
|
||||
subtitleGenerationProgress: 'subtitle-generation:progress',
|
||||
subtitleSet: 'subtitle:set',
|
||||
overlayPointerRecoveryRequest: 'overlay:pointer-recovery-request',
|
||||
subtitleVisibility: 'mpv:subVisibility',
|
||||
|
||||
@@ -44,6 +44,7 @@ const SESSION_ACTION_IDS: SessionActionId[] = [
|
||||
'openControllerDebug',
|
||||
'openJimaku',
|
||||
'openTsukihime',
|
||||
'openSubtitleGeneration',
|
||||
'openYoutubePicker',
|
||||
'openPlaylistBrowser',
|
||||
'replayCurrentSubtitle',
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import type {
|
||||
SubtitleGenerationConfig,
|
||||
SubtitleGenerationModelStatus,
|
||||
SubtitleGenerationProgress,
|
||||
SubtitleGenerationTools,
|
||||
} from './subtitle-generation';
|
||||
|
||||
export type SubtitleGenerationResult =
|
||||
| { ok: true; message: string; outputPath?: string }
|
||||
| { ok: false; message: string };
|
||||
|
||||
export interface SubtitleGenerationStatus {
|
||||
model: SubtitleGenerationModelStatus;
|
||||
vad: { enabled: boolean; model: SubtitleGenerationModelStatus };
|
||||
tools: SubtitleGenerationTools;
|
||||
managedModel: SubtitleGenerationConfig['managedModel'];
|
||||
externalModelPath: string | null;
|
||||
mediaPath: string | null;
|
||||
running: boolean;
|
||||
progress: SubtitleGenerationProgress | null;
|
||||
lastResult: SubtitleGenerationResult | null;
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
// Official multilingual models from whisper.cpp/models/download-ggml-model.sh.
|
||||
// Byte sizes and SHA256: https://huggingface.co/api/models/ggerganov/whisper.cpp/tree/main.
|
||||
const MODEL_CATALOG = {
|
||||
tiny: {
|
||||
id: 'tiny',
|
||||
size: 77691713,
|
||||
sha256: 'be07e048e1e599ad46341c8d2a135645097a538221678b7acdd1b1919c6e1b21',
|
||||
description: 'Fastest and lightest; more transcription errors.',
|
||||
},
|
||||
'tiny-q5_1': {
|
||||
id: 'tiny-q5_1',
|
||||
size: 32152673,
|
||||
sha256: '818710568da3ca15689e31a743197b520007872ff9576237bda97bd1b469c3d7',
|
||||
description:
|
||||
'tiny with 5-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
'tiny-q8_0': {
|
||||
id: 'tiny-q8_0',
|
||||
size: 43537433,
|
||||
sha256: 'c2085835d3f50733e2ff6e4b41ae8a2b8d8110461e18821b09a15c40c42d1cca',
|
||||
description:
|
||||
'tiny with 8-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
base: {
|
||||
id: 'base',
|
||||
size: 147951465,
|
||||
sha256: '60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe',
|
||||
description: 'Fast and lightweight; less accurate than small.',
|
||||
},
|
||||
'base-q5_1': {
|
||||
id: 'base-q5_1',
|
||||
size: 59707625,
|
||||
sha256: '422f1ae452ade6f30a004d7e5c6a43195e4433bc370bf23fac9cc591f01a8898',
|
||||
description:
|
||||
'base with 5-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
'base-q8_0': {
|
||||
id: 'base-q8_0',
|
||||
size: 81768585,
|
||||
sha256: 'c577b9a86e7e048a0b7eada054f4dd79a56bbfa911fbdacf900ac5b567cbb7d9',
|
||||
description:
|
||||
'base with 8-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
small: {
|
||||
id: 'small',
|
||||
size: 487601967,
|
||||
sha256: '1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b',
|
||||
description: 'Recommended starting point for accuracy and processing time.',
|
||||
},
|
||||
'small-q5_1': {
|
||||
id: 'small-q5_1',
|
||||
size: 190085487,
|
||||
sha256: 'ae85e4a935d7a567bd102fe55afc16bb595bdb618e11b2fc7591bc08120411bb',
|
||||
description:
|
||||
'small with 5-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
'small-q8_0': {
|
||||
id: 'small-q8_0',
|
||||
size: 264464607,
|
||||
sha256: '49c8fb02b65e6049d5fa6c04f81f53b867b5ec9540406812c643f177317f779f',
|
||||
description:
|
||||
'small with 8-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
medium: {
|
||||
id: 'medium',
|
||||
size: 1533763059,
|
||||
sha256: '6c14d5adee5f86394037b4e4e8b59f1673b6cee10e3cf0b11bbdbee79c156208',
|
||||
description: 'Prioritizes accuracy over small; takes longer to process.',
|
||||
},
|
||||
'medium-q5_0': {
|
||||
id: 'medium-q5_0',
|
||||
size: 539212467,
|
||||
sha256: '19fea4b380c3a618ec4723c3eef2eb785ffba0d0538cf43f8f235e7b3b34220f',
|
||||
description:
|
||||
'medium with 5-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
'medium-q8_0': {
|
||||
id: 'medium-q8_0',
|
||||
size: 823369779,
|
||||
sha256: '42a1ffcbe4167d224232443396968db4d02d4e8e87e213d3ee2e03095dea6502',
|
||||
description:
|
||||
'medium with 8-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
'large-v1': {
|
||||
id: 'large-v1',
|
||||
size: 3094623691,
|
||||
sha256: '7d99f41a10525d0206bddadd86760181fa920438b6b33237e3118ff6c83bb53d',
|
||||
description: 'Older large model; high memory use and longer processing.',
|
||||
},
|
||||
'large-v2': {
|
||||
id: 'large-v2',
|
||||
size: 3094623691,
|
||||
sha256: '9a423fe4d40c82774b6af34115b8b935f34152246eb19e80e376071d3f999487',
|
||||
description: 'Older large model; high memory use and longer processing.',
|
||||
},
|
||||
'large-v2-q5_0': {
|
||||
id: 'large-v2-q5_0',
|
||||
size: 1080732091,
|
||||
sha256: '3a214837221e4530dbc1fe8d734f302af393eb30bd0ed046042ebf4baf70f6f2',
|
||||
description:
|
||||
'large-v2 with 5-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
'large-v2-q8_0': {
|
||||
id: 'large-v2-q8_0',
|
||||
size: 1656129691,
|
||||
sha256: 'fef54e6d898246a65c8285bfa83bd1807e27fadf54d5d4e81754c47634737e8c',
|
||||
description:
|
||||
'large-v2 with 8-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
'large-v3': {
|
||||
id: 'large-v3',
|
||||
size: 3095033483,
|
||||
sha256: '64d182b440b98d5203c4f9bd541544d84c605196c4f7b845dfa11fb23594d1e2',
|
||||
description: 'Prioritizes accuracy; high memory use and longer processing.',
|
||||
},
|
||||
'large-v3-q5_0': {
|
||||
id: 'large-v3-q5_0',
|
||||
size: 1081140203,
|
||||
sha256: 'd75795ecff3f83b5faa89d1900604ad8c780abd5739fae406de19f23ecd98ad1',
|
||||
description:
|
||||
'large-v3 with 5-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
'large-v3-turbo': {
|
||||
id: 'large-v3-turbo',
|
||||
size: 1624555275,
|
||||
sha256: '1fc70f774d38eb169993ac391eea357ef47c88757ef72ee5943879b7e8e2bc69',
|
||||
description: 'Faster large-v3 variant with an accuracy tradeoff; uses more memory than small.',
|
||||
},
|
||||
'large-v3-turbo-q5_0': {
|
||||
id: 'large-v3-turbo-q5_0',
|
||||
size: 574041195,
|
||||
sha256: '394221709cd5ad1f40c46e6031ca61bce88931e6e088c188294c6d5a55ffa7e2',
|
||||
description:
|
||||
'large-v3-turbo with 5-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
'large-v3-turbo-q8_0': {
|
||||
id: 'large-v3-turbo-q8_0',
|
||||
size: 874188075,
|
||||
sha256: '317eb69c11673c9de1e1f0d459b253999804ec71ac4c23c17ecf5fbe24e259a1',
|
||||
description:
|
||||
'large-v3-turbo with 8-bit quantization: lower memory use, with a possible accuracy tradeoff.',
|
||||
},
|
||||
} as const;
|
||||
|
||||
export type SubtitleGenerationModelId = keyof typeof MODEL_CATALOG;
|
||||
|
||||
export const SUBTITLE_GENERATION_MODELS = Object.values(MODEL_CATALOG);
|
||||
|
||||
export const RECOMMENDED_SUBTITLE_GENERATION_MODEL = 'small' satisfies SubtitleGenerationModelId;
|
||||
|
||||
export function isSubtitleGenerationModelId(value: unknown): value is SubtitleGenerationModelId {
|
||||
return typeof value === 'string' && Object.hasOwn(MODEL_CATALOG, value);
|
||||
}
|
||||
|
||||
export function getSubtitleGenerationModel(id: SubtitleGenerationModelId) {
|
||||
return MODEL_CATALOG[id];
|
||||
}
|
||||
|
||||
export function formatSubtitleGenerationModelSize(size: number): string {
|
||||
if (size < 1024 ** 2) return `${Math.ceil(size / 1024)} KiB`;
|
||||
return size >= 1024 ** 3
|
||||
? `${(size / 1024 ** 3).toFixed(1)} GiB`
|
||||
: `${Math.round(size / 1024 ** 2)} MiB`;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export const SUBTITLE_GENERATION_VAD_MODEL = {
|
||||
filename: 'ggml-silero-v6.2.0.bin',
|
||||
size: 885098,
|
||||
sha256: '2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987',
|
||||
url: 'https://huggingface.co/ggml-org/whisper-vad/resolve/9ffd54a1e1ee413ddf265af9913beaf518d1639b/ggml-silero-v6.2.0.bin',
|
||||
} as const;
|
||||
@@ -0,0 +1,104 @@
|
||||
import {
|
||||
isSubtitleGenerationModelId,
|
||||
RECOMMENDED_SUBTITLE_GENERATION_MODEL,
|
||||
type SubtitleGenerationModelId,
|
||||
} from './subtitle-generation-model-catalog';
|
||||
|
||||
export interface SubtitleGenerationConfig {
|
||||
whisperPath: string;
|
||||
modelPath: string;
|
||||
managedModel: SubtitleGenerationModelId;
|
||||
threads: number;
|
||||
ffmpegPath: string;
|
||||
ffprobePath: string;
|
||||
vadModelPath: string;
|
||||
vadPath: string;
|
||||
}
|
||||
|
||||
export const DEFAULT_SUBTITLE_GENERATION_CONFIG: SubtitleGenerationConfig = {
|
||||
whisperPath: '',
|
||||
modelPath: '',
|
||||
managedModel: RECOMMENDED_SUBTITLE_GENERATION_MODEL,
|
||||
threads: 4,
|
||||
ffmpegPath: '',
|
||||
ffprobePath: '',
|
||||
vadModelPath: '',
|
||||
vadPath: '',
|
||||
};
|
||||
|
||||
export interface SubtitleGenerationProgress {
|
||||
stage: 'download' | 'extract' | 'transcribe' | 'write';
|
||||
percent?: number;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export type SubtitleGenerationModelStatus =
|
||||
| { kind: 'external' | 'managed'; path: string }
|
||||
| { kind: 'missing'; path: string }
|
||||
| { kind: 'invalid'; path: string; message: string };
|
||||
|
||||
export type SubtitleGenerationToolStatus =
|
||||
| { kind: 'found'; path: string }
|
||||
| { kind: 'missing'; message: string };
|
||||
|
||||
/** Executables generation depends on. `vad` is null unless dialogue mode is on. */
|
||||
export interface SubtitleGenerationTools {
|
||||
ffmpeg: SubtitleGenerationToolStatus;
|
||||
ffprobe: SubtitleGenerationToolStatus;
|
||||
whisper: SubtitleGenerationToolStatus;
|
||||
vad: SubtitleGenerationToolStatus | null;
|
||||
}
|
||||
|
||||
export function missingSubtitleGenerationTools(tools: SubtitleGenerationTools): string[] {
|
||||
return [tools.ffmpeg, tools.ffprobe, tools.whisper, tools.vad].flatMap((tool) =>
|
||||
tool?.kind === 'missing' ? [tool.message] : [],
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveSubtitleGenerationConfig(
|
||||
value: unknown,
|
||||
onWarning?: (key: string, value: unknown, message: string) => void,
|
||||
): SubtitleGenerationConfig {
|
||||
const result = { ...DEFAULT_SUBTITLE_GENERATION_CONFIG };
|
||||
if (value === undefined) return result;
|
||||
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
||||
onWarning?.('subtitleGeneration', value, 'Expected an object.');
|
||||
return result;
|
||||
}
|
||||
for (const [key, rawValue] of Object.entries(value)) {
|
||||
if (
|
||||
key !== 'whisperPath' &&
|
||||
key !== 'modelPath' &&
|
||||
key !== 'ffmpegPath' &&
|
||||
key !== 'ffprobePath' &&
|
||||
key !== 'vadModelPath' &&
|
||||
key !== 'vadPath'
|
||||
)
|
||||
continue;
|
||||
const candidate: unknown = rawValue;
|
||||
if (typeof candidate === 'string') {
|
||||
result[key] = candidate.trim();
|
||||
} else onWarning?.(key, candidate, 'Expected a string.');
|
||||
}
|
||||
if ('managedModel' in value) {
|
||||
if (isSubtitleGenerationModelId(value.managedModel)) {
|
||||
result.managedModel = value.managedModel;
|
||||
} else
|
||||
onWarning?.(
|
||||
'managedModel',
|
||||
value.managedModel,
|
||||
'Expected a supported multilingual Whisper model.',
|
||||
);
|
||||
}
|
||||
if ('threads' in value) {
|
||||
if (
|
||||
typeof value.threads === 'number' &&
|
||||
Number.isInteger(value.threads) &&
|
||||
value.threads >= 1 &&
|
||||
value.threads <= 256
|
||||
) {
|
||||
result.threads = value.threads;
|
||||
} else onWarning?.('threads', value.threads, 'Expected an integer between 1 and 256.');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { AnkiConnectConfig, WordCardKind } from './anki';
|
||||
import type { SubtitleGenerationConfig } from '../shared/subtitle-generation';
|
||||
import type {
|
||||
AiConfig,
|
||||
AiFeatureConfig,
|
||||
@@ -124,6 +125,7 @@ export interface ShortcutsConfig {
|
||||
openRuntimeOptions?: string | null;
|
||||
openJimaku?: string | null;
|
||||
openTsukihime?: string | null;
|
||||
openSubtitleGeneration?: string | null;
|
||||
openSessionHelp?: string | null;
|
||||
openControllerSelect?: string | null;
|
||||
openControllerDebug?: string | null;
|
||||
@@ -149,6 +151,7 @@ export interface Config {
|
||||
shortcuts?: RawShortcutsConfig;
|
||||
secondarySub?: SecondarySubConfig;
|
||||
subsync?: SubsyncConfig;
|
||||
subtitleGeneration?: Partial<SubtitleGenerationConfig>;
|
||||
startupWarmups?: StartupWarmupsConfig;
|
||||
subtitleStyle?: SubtitleStyleConfig;
|
||||
subtitleSidebar?: SubtitleSidebarConfig;
|
||||
@@ -298,6 +301,7 @@ export interface ResolvedConfig {
|
||||
shortcuts: Required<ShortcutsConfig>;
|
||||
secondarySub: Required<SecondarySubConfig>;
|
||||
subsync: Required<SubsyncConfig>;
|
||||
subtitleGeneration: SubtitleGenerationConfig;
|
||||
startupWarmups: {
|
||||
lowPowerMode: boolean;
|
||||
mecab: boolean;
|
||||
|
||||
@@ -12,6 +12,12 @@ import type {
|
||||
MediaTimingReviewWaveformResult,
|
||||
} from './anki';
|
||||
import type { ChangelogSnapshot } from './changelog';
|
||||
import type { SubtitleGenerationProgress } from '../shared/subtitle-generation';
|
||||
import type { SubtitleGenerationModelId } from '../shared/subtitle-generation-model-catalog';
|
||||
import type {
|
||||
SubtitleGenerationResult,
|
||||
SubtitleGenerationStatus,
|
||||
} from '../shared/subtitle-generation-ipc';
|
||||
import type { ResolvedConfig, ShortcutsConfig } from './config';
|
||||
import type {
|
||||
CompiledSessionBinding,
|
||||
@@ -431,6 +437,20 @@ export interface SessionNumericSelectionStartPayload {
|
||||
}
|
||||
|
||||
export interface ElectronAPI {
|
||||
requestSubtitleGenerationOpen: () => Promise<boolean>;
|
||||
onSubtitleGenerationOpen: (callback: () => void) => void;
|
||||
getSubtitleGenerationStatus: () => Promise<SubtitleGenerationStatus>;
|
||||
selectSubtitleGenerationModel: (
|
||||
model: SubtitleGenerationModelId,
|
||||
) => Promise<SubtitleGenerationStatus>;
|
||||
startSubtitleGeneration: () => Promise<SubtitleGenerationResult>;
|
||||
downloadSubtitleGenerationModel: () => Promise<SubtitleGenerationResult>;
|
||||
downloadSubtitleGenerationVadModel: () => Promise<SubtitleGenerationResult>;
|
||||
setSubtitleGenerationVadEnabled: (enabled: boolean) => Promise<SubtitleGenerationStatus>;
|
||||
cancelSubtitleGeneration: () => Promise<void>;
|
||||
onSubtitleGenerationProgress: (
|
||||
callback: (progress: SubtitleGenerationProgress) => void,
|
||||
) => () => void;
|
||||
getOverlayLayer: () => 'visible' | 'modal' | null;
|
||||
getPathForFile: (file: File) => string;
|
||||
onSubtitle: (callback: (data: SubtitleData) => void) => void;
|
||||
@@ -579,6 +599,7 @@ export interface ElectronAPI {
|
||||
modal:
|
||||
| 'runtime-options'
|
||||
| 'subsync'
|
||||
| 'subtitle-generation'
|
||||
| 'jimaku'
|
||||
| 'tsukihime'
|
||||
| 'youtube-track-picker'
|
||||
@@ -596,6 +617,7 @@ export interface ElectronAPI {
|
||||
modal:
|
||||
| 'runtime-options'
|
||||
| 'subsync'
|
||||
| 'subtitle-generation'
|
||||
| 'jimaku'
|
||||
| 'tsukihime'
|
||||
| 'youtube-track-picker'
|
||||
|
||||
@@ -23,6 +23,7 @@ export type SessionActionId =
|
||||
| 'openControllerDebug'
|
||||
| 'openJimaku'
|
||||
| 'openTsukihime'
|
||||
| 'openSubtitleGeneration'
|
||||
| 'openYoutubePicker'
|
||||
| 'openPlaylistBrowser'
|
||||
| 'replayCurrentSubtitle'
|
||||
|
||||
Reference in New Issue
Block a user