diff --git a/README.md b/README.md index 4fd7b218..8528cfd0 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,10 @@ Browse sibling episode files and the active mpv queue in one overlay modal. Open Jimaku Search and download Japanese subtitles + + Local Subtitle Generation + Generate Japanese subtitles from local audio in a standalone modal (Ctrl+Shift+G), the sidebar button, or launcher, with progress and optional managed model downloads. Requires whisper.cpp and FFmpeg. Optional Silero speech detection prioritizes dialogue in separately timed passages. Setup guide + TsukiHime Search and download subtitles extracted from anime releases, with Japanese and secondary-language tabs (Ctrl+Shift+T) — no API key, requires xz on your PATH diff --git a/changes/japanese-subtitle-generation.md b/changes/japanese-subtitle-generation.md new file mode 100644 index 00000000..8f9da780 --- /dev/null +++ b/changes/japanese-subtitle-generation.md @@ -0,0 +1,6 @@ +type: added +area: subtitles + +- Generate local Japanese SRT subtitles with whisper.cpp from a standalone modal opened with Ctrl+Shift+G, the subtitle sidebar button, or `subminer generate-subs`, with shared progress reporting, cancellation, safe output files, and automatic loading into the matching mpv video. +- Configure an existing multilingual model in Settings or choose an official multilingual model, including quantized variants, in the modal or launcher. The modal shows download sizes, speed and accuracy guidance, and a recommended starting model before explicitly downloading a verified SubMiner-managed model. Executable paths are optional overrides; empty fields find whisper-cli, ffmpeg, and ffprobe on PATH. +- Optionally select Prioritize dialogue in the modal and download the separate Silero speech detection model with progress and cancellation. The choice lasts for the session; a configured VAD model path sets the default. With the detector executable installed, transcribe separate speech passages at their original positions, reset text context between passages, and keep subtitle cues within those passages instead of spanning music breaks. diff --git a/changes/subtitle-sidebar-gap-follow.md b/changes/subtitle-sidebar-gap-follow.md new file mode 100644 index 00000000..03d18f75 --- /dev/null +++ b/changes/subtitle-sidebar-gap-follow.md @@ -0,0 +1,4 @@ +type: fixed +area: subtitles + +- Keep the subtitle sidebar near playback during gaps when the subtitle file has a cue starting at zero. diff --git a/config.example.jsonc b/config.example.jsonc index dcb47a5c..bdfc9978 100644 --- a/config.example.jsonc +++ b/config.example.jsonc @@ -6,6 +6,23 @@ */ { + // ========================================== + // Japanese Subtitle Generation + // Generate timed Japanese subtitles from local audio using whisper.cpp. + // Configure an existing GGML model path or explicitly download a SubMiner-managed model. + // Hot-reload: settings apply to the next generation or model download. + // ========================================== + "subtitleGeneration": { + "whisperPath": "", // Optional path override for whisper.cpp. Leave empty to find whisper-cli on PATH. + "modelPath": "", // Path to an existing multilingual whisper.cpp GGML model. Leave empty to use a SubMiner-managed model. A configured path always takes precedence. + "managedModel": "small", // Multilingual whisper.cpp model to use when modelPath is empty. Download it explicitly from the generation modal or launcher. Values: tiny | tiny-q5_1 | tiny-q8_0 | base | base-q5_1 | base-q8_0 | small | small-q5_1 | small-q8_0 | medium | medium-q5_0 | medium-q8_0 | large-v1 | large-v2 | large-v2-q5_0 | large-v2-q8_0 | large-v3 | large-v3-q5_0 | large-v3-turbo | large-v3-turbo-q5_0 | large-v3-turbo-q8_0 + "threads": 4, // Positive integer CPU thread count for whisper.cpp Japanese transcription. + "ffmpegPath": "", // Optional FFmpeg path override for audio extraction. Leave empty to find ffmpeg on PATH. + "ffprobePath": "", // Optional FFprobe path override for audio tracks and timing. Leave empty to find ffprobe on PATH. + "vadModelPath": "", // Path to a whisper.cpp Silero VAD model. Enables dialogue-focused generation from separate speech passages. Leave empty to transcribe the full audio, including songs. + "vadPath": "" // Optional speech detector executable override. With vadModelPath configured, leave empty to find whisper-vad-speech-segments on PATH. + }, // Generate timed Japanese subtitles from local audio using whisper.cpp. + // ========================================== // Visible Overlay Auto-Start // Show the visible subtitle overlay automatically after managed mpv playback starts SubMiner. @@ -206,6 +223,7 @@ "openRuntimeOptions": "CommandOrControl+Shift+O", // Accelerator that opens the runtime options modal. "openJimaku": "Ctrl+Shift+J", // Accelerator that opens the Jimaku subtitle search modal. "openTsukihime": "Ctrl+Shift+T", // Accelerator that opens the TsukiHime subtitle search modal (configured secondary/Japanese primary tabs). + "openSubtitleGeneration": "Ctrl+Shift+G", // Accelerator that opens the standalone Japanese subtitle generation modal. "openSessionHelp": "CommandOrControl+Slash", // Accelerator that opens the session help / keybinding cheatsheet. "openControllerSelect": "Alt+C", // Accelerator that opens the controller selection and learn-mode modal. "openControllerDebug": "Alt+Shift+C", // Accelerator that opens the controller debug modal with live axis/button readouts. diff --git a/docs-site/.vitepress/config.ts b/docs-site/.vitepress/config.ts index 122da831..acace595 100644 --- a/docs-site/.vitepress/config.ts +++ b/docs-site/.vitepress/config.ts @@ -369,6 +369,7 @@ const sidebar: DefaultTheme.SidebarItem[] = [ { text: 'Jellyfin', link: '/jellyfin-integration' }, { text: 'YouTube', link: '/youtube-integration' }, { text: 'Jimaku', link: '/jimaku-integration' }, + { text: 'Subtitle Generation', link: '/subtitle-generation' }, { text: 'TsukiHime', link: '/tsukihime-integration' }, { text: 'AniList', link: '/anilist-integration' }, { text: 'AniSkip', link: '/aniskip-integration' }, diff --git a/docs-site/configuration.md b/docs-site/configuration.md index d19c4b47..684c10f8 100644 --- a/docs-site/configuration.md +++ b/docs-site/configuration.md @@ -1054,7 +1054,7 @@ This example is intentionally compact. The option table below documents availabl | `metadata.pattern` | string | Format pattern for metadata: `%f`=filename, `%F`=filename+ext, `%t`=time, `%T`=time with milliseconds, `
`=newline | | `isLapis` | object | Lapis/shared sentence-card config: `{ enabled, sentenceCardModel }`. Sentence/audio field names are fixed to `Sentence` and `SentenceAudio`. | | `isKiku` | object | Kiku-only config: `{ enabled, fieldGrouping, deleteDuplicateInAuto }` (shared sentence/audio/model settings are inherited from `isLapis`) | -| `isSenren` | object | Senren-only config: `{ enabled, fieldGrouping, deleteDuplicateInAuto }`. Merges duplicates using Senren's scene-switching markup. Mutually exclusive with `isKiku.enabled`. | +| `isSenren` | object | Senren-only config: `{ enabled, fieldGrouping, deleteDuplicateInAuto }`. Merges duplicates using Senren's scene-switching markup. Mutually exclusive with `isKiku.enabled`. | `ankiConnect.ai` only controls feature-local enablement plus optional `model` / `systemPrompt` overrides. API key resolution, base URL, and timeout live under the shared top-level [`ai`](#shared-ai-provider) config. @@ -1199,6 +1199,14 @@ The keyboard shortcut lives under `shortcuts.openTsukihime` (default `Ctrl+Shift See [TsukiHime Integration](/tsukihime-integration) for the modal workflow, language tabs, and troubleshooting. +### Japanese Subtitle Generation + +Open the standalone modal with `Ctrl+Shift+G`, configurable through `shortcuts.openSubtitleGeneration`, or use the subtitle sidebar button. See [shortcuts](/shortcuts) for the shared mpv and overlay keybindings. + +`subtitleGeneration` configures local Japanese transcription for both the launcher and overlay. In **Settings → Integrations → Japanese Subtitle Generation**, set `modelPath` to an existing multilingual whisper.cpp GGML model, or leave it empty and choose a `managedModel` as the default. The generation modal lets you select another model for the current session, with download sizes and accuracy versus speed guidance. Downloads are explicit. Leave `whisperPath`, `ffmpegPath`, and `ffprobePath` empty to find the executables on `PATH`, or set them to override the executable paths. `threads` controls the CPU thread count. Settings apply to the next operation. See [subtitle generation](/subtitle-generation) for setup and behavior, and the [generated configuration example](/config.example.jsonc) for defaults. + +The generation modal offers an optional **Prioritize dialogue** checkbox and a separate Silero model download. Set `subtitleGeneration.vadModelPath` to a Silero GGML VAD model to make dialogue mode the default. `vadPath` overrides the speech detector executable. See [dialogue generation setup](/subtitle-generation#prioritizing-spoken-dialogue) for session behavior, the additional tool, and limitations. + ### Subtitle Sync Sync a subtitle track from the overlay picker using `alass` or `ffsubsync`. The picker lets you choose which track gets retimed (the active primary track by default) and, for alass, which reference it is aligned against (the secondary subtitle track by default). Both are **optional external tools** that must be installed separately and available on your `PATH` (or configured via the path options below). diff --git a/docs-site/installation.md b/docs-site/installation.md index 2ab66030..f67cf525 100644 --- a/docs-site/installation.md +++ b/docs-site/installation.md @@ -14,6 +14,10 @@ Only **mpv** is strictly required to run SubMiner. Everything else enhances the Several entries below exist only for the `subminer` command-line launcher, which is Linux and macOS only. On Windows you launch playback with the **SubMiner mpv** shortcut instead, so you can ignore those rows. +[Local Japanese subtitle generation](/subtitle-generation) additionally requires whisper.cpp's `whisper-cli`, FFmpeg, and `ffprobe`. Configure their executable paths in Settings if needed. SubMiner can download a speech model explicitly, or use your existing multilingual GGML model. + +Optional [dialogue-focused generation](/subtitle-generation#prioritizing-spoken-dialogue) also uses whisper.cpp's speech segment detector and a separate Silero GGML VAD model. + | Dependency | Status | Platforms | What it does | | -------------------- | ----------- | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | | mpv | Required | All | The video player SubMiner overlays on. Must support `--input-ipc-server`. | diff --git a/docs-site/launcher-script.md b/docs-site/launcher-script.md index 0310eec6..c74acaee 100644 --- a/docs-site/launcher-script.md +++ b/docs-site/launcher-script.md @@ -159,6 +159,7 @@ subminer stats -b # start background stats daemon | `subminer stats rebuild` / `backfill` | Rebuild or backfill rollup data | | `subminer doctor` | Dependency + config + socket diagnostics (`--refresh-known-words` refreshes the known-word cache) | | `subminer settings` | Open the SubMiner settings window | +| `subminer generate-subs [video]` | Generate [Japanese subtitles](/usage#generate-japanese-subtitles-locally) locally | | `subminer logs -e` | Export a sanitized local-date log ZIP and print its path | | `subminer config path` | Print active config file path | | `subminer config show` | Print active config contents | diff --git a/docs-site/public/config.example.jsonc b/docs-site/public/config.example.jsonc index dcb47a5c..bdfc9978 100644 --- a/docs-site/public/config.example.jsonc +++ b/docs-site/public/config.example.jsonc @@ -6,6 +6,23 @@ */ { + // ========================================== + // Japanese Subtitle Generation + // Generate timed Japanese subtitles from local audio using whisper.cpp. + // Configure an existing GGML model path or explicitly download a SubMiner-managed model. + // Hot-reload: settings apply to the next generation or model download. + // ========================================== + "subtitleGeneration": { + "whisperPath": "", // Optional path override for whisper.cpp. Leave empty to find whisper-cli on PATH. + "modelPath": "", // Path to an existing multilingual whisper.cpp GGML model. Leave empty to use a SubMiner-managed model. A configured path always takes precedence. + "managedModel": "small", // Multilingual whisper.cpp model to use when modelPath is empty. Download it explicitly from the generation modal or launcher. Values: tiny | tiny-q5_1 | tiny-q8_0 | base | base-q5_1 | base-q8_0 | small | small-q5_1 | small-q8_0 | medium | medium-q5_0 | medium-q8_0 | large-v1 | large-v2 | large-v2-q5_0 | large-v2-q8_0 | large-v3 | large-v3-q5_0 | large-v3-turbo | large-v3-turbo-q5_0 | large-v3-turbo-q8_0 + "threads": 4, // Positive integer CPU thread count for whisper.cpp Japanese transcription. + "ffmpegPath": "", // Optional FFmpeg path override for audio extraction. Leave empty to find ffmpeg on PATH. + "ffprobePath": "", // Optional FFprobe path override for audio tracks and timing. Leave empty to find ffprobe on PATH. + "vadModelPath": "", // Path to a whisper.cpp Silero VAD model. Enables dialogue-focused generation from separate speech passages. Leave empty to transcribe the full audio, including songs. + "vadPath": "" // Optional speech detector executable override. With vadModelPath configured, leave empty to find whisper-vad-speech-segments on PATH. + }, // Generate timed Japanese subtitles from local audio using whisper.cpp. + // ========================================== // Visible Overlay Auto-Start // Show the visible subtitle overlay automatically after managed mpv playback starts SubMiner. @@ -206,6 +223,7 @@ "openRuntimeOptions": "CommandOrControl+Shift+O", // Accelerator that opens the runtime options modal. "openJimaku": "Ctrl+Shift+J", // Accelerator that opens the Jimaku subtitle search modal. "openTsukihime": "Ctrl+Shift+T", // Accelerator that opens the TsukiHime subtitle search modal (configured secondary/Japanese primary tabs). + "openSubtitleGeneration": "Ctrl+Shift+G", // Accelerator that opens the standalone Japanese subtitle generation modal. "openSessionHelp": "CommandOrControl+Slash", // Accelerator that opens the session help / keybinding cheatsheet. "openControllerSelect": "Alt+C", // Accelerator that opens the controller selection and learn-mode modal. "openControllerDebug": "Alt+Shift+C", // Accelerator that opens the controller debug modal with live axis/button readouts. diff --git a/docs-site/shortcuts.md b/docs-site/shortcuts.md index f2e0d079..1b739359 100644 --- a/docs-site/shortcuts.md +++ b/docs-site/shortcuts.md @@ -82,6 +82,7 @@ Mouse-hover playback behavior is configured separately from shortcuts: `subtitle | `Ctrl/Cmd+Shift+O` | Open runtime options palette | `shortcuts.openRuntimeOptions` | | `Ctrl/Cmd+/` | Open session help modal | `shortcuts.openSessionHelp` | | `Ctrl+Shift+J` | Open Jimaku subtitle search modal | `shortcuts.openJimaku` | +| `Ctrl+Shift+G` | Open Japanese subtitle generation modal | `shortcuts.openSubtitleGeneration` | | `Ctrl+Shift+T` | Open TsukiHime subtitle search modal (EN/JA tabs) | `shortcuts.openTsukihime` | | `Ctrl/Cmd+N` | Toggle overlay notification history panel | `shortcuts.toggleNotificationHistory` | | `Ctrl+Alt+C` | Open the manual YouTube subtitle picker | `keybindings` | diff --git a/docs-site/subtitle-generation.md b/docs-site/subtitle-generation.md new file mode 100644 index 00000000..6a985166 --- /dev/null +++ b/docs-site/subtitle-generation.md @@ -0,0 +1,66 @@ +# Japanese subtitle generation + +Generate Japanese SRT subtitles from a local video's audio using [whisper.cpp](https://github.com/ggml-org/whisper.cpp). The launcher and overlay use the same local generation service. Audio stays on your computer. Model downloads require an internet connection; generation with an installed model does not. + +## Setup + +Install whisper.cpp's `whisper-cli` executable and FFmpeg, including `ffprobe`. SubMiner downloads models, not these executables. Leave `whisperPath`, `ffmpegPath`, and `ffprobePath` empty to find the executables on `PATH`. To use a specific installation, set a path override under **Settings → Integrations → Japanese Subtitle Generation**. + +Choose one model source: + +- Set `subtitleGeneration.modelPath` to an existing **multilingual whisper.cpp GGML `.bin` model**. Python Whisper checkpoints and English-only models are not suitable for Japanese transcription. +- Leave that path empty and choose a model directly in the generation modal. Each option shows its download size; the selected model has speed and accuracy guidance. The modal offers **Download model** when that model is missing. Your choice lasts for the current SubMiner session, including closing and reopening the modal. Set `subtitleGeneration.managedModel` in Settings to change the default for future sessions. + +Managed models are stored in `models/whisper/` beside your SubMiner configuration file. Downloads show progress, verify the expected file size and SHA256, and publish the model only after verification. Cancelling or failing a download removes its temporary files. A configured external path always takes precedence; an unreadable path displays an error instead of silently downloading another model. + +See the [generated configuration example](/config.example.jsonc) for current defaults. Changes apply to the next operation. + +## Prioritizing spoken dialogue + +To focus on dialogue, check the optional **Prioritize dialogue** box in the generation modal. If the speech detection model is missing, click **Download Silero** to install it. This separate download uses the same progress, cancellation, and integrity checks as Whisper downloads. Checking the box never downloads automatically, and leaving it unchecked lets you generate without the Silero model. + +You also need whisper.cpp's [speech segment detector](https://github.com/ggml-org/whisper.cpp/tree/master/examples/vad-speech-segments). SubMiner downloads the model, not this executable. The detector is found as `whisper-vad-speech-segments` on `PATH`. Builds from the upstream source may name it `vad-speech-segments`; set `vadPath` in **Settings → Integrations → Japanese Subtitle Generation** when needed. + +The checkbox choice lasts for the current SubMiner session, including closing and reopening the modal. To make dialogue mode your default, set `vadModelPath` in Settings to a [Silero GGML VAD model](https://huggingface.co/ggml-org/whisper-vad/tree/main). The modal downloads `ggml-silero-v6.2.0.bin` into the same `models/whisper/` directory as managed Whisper models. An existing configured VAD path takes precedence and checks the box initially. Unchecking it temporarily disables dialogue mode without changing that path. Downloading the model alone does not enable dialogue mode. + +With speech detection configured, SubMiner transcribes short speech passages separately and restores each passage's position on the original audio timeline. It resets transcription context between passages and limits subtitle cues to the passage that produced them. A line cannot stretch across an omitted music break, and repeated dialogue in separate passages remains separate. Progress reports completed batches of dialogue passages. + +This mode prioritizes spoken dialogue over songs and background sounds. It can miss quiet speech or speech mixed with loud music, and recognition errors are still possible. Uncheck **Prioritize dialogue** to return to full-audio transcription for the session, or clear `vadModelPath` to change the default. A selected detector or model that fails stops generation with an error. Existing subtitles are preserved. + +## Choosing a model + +Start with **small** for a balance of Japanese recognition quality and CPU time. This is a general starting recommendation, not a benchmark for your hardware. Tiny and base need less memory and usually finish sooner, with more recognition errors. Medium and large models favor accuracy but need more resources. Large-v3-turbo is optimized for speed compared with large-v3, with some accuracy tradeoff; actual performance depends on your CPU, GPU, whisper.cpp build, and audio. + +The picker includes whisper.cpp's official multilingual tiny, base, small, medium, large-v1, large-v2, large-v3, and large-v3-turbo downloads, including their available quantized variants. Quantized models use less disk space and memory, with possible accuracy loss. English-only `.en` models are excluded. See the [upstream model list](https://github.com/ggml-org/whisper.cpp/blob/master/models/download-ggml-model.sh) and [Whisper's model guidance](https://github.com/openai/whisper#available-models-and-languages). + +A configured external Model Path takes precedence and hides the managed model picker. Clear it in Settings to choose a managed model. Changing the picker never downloads automatically, and it cannot change the model during an active download or generation. + +## From the overlay + +1. Open a local video in mpv and select its Japanese audio track. +2. Press **Ctrl+Shift+G** to open the standalone generation modal. You can also click **Generate Japanese subtitles** in the subtitle sidebar. Neither an open sidebar nor an existing subtitle track is required. +3. Choose a model and download it if prompted, or configure your existing model path in Settings and click **Check again**. +4. Optionally check **Prioritize dialogue** and download Silero if prompted. +5. Click **Generate subtitles**. + +The modal shows audio preparation, transcription, and saving progress. Percentages appear when the underlying tool reports them. **Cancel** stops the current operation. Closing the modal lets the job continue; reopening it shows the current progress or result. + +**Escape** or **Close** closes the modal using the same focus and overlay restoration as other SubMiner modals. Change or disable its shortcut with `shortcuts.openSubtitleGeneration` in Settings. Ctrl+G remains assigned to field grouping. + +SubMiner saves `