Compare commits

...
7 Commits
64 changed files with 4395 additions and 334 deletions

No files matched your search

+24
View File
@@ -1,5 +1,29 @@
# Changelog
## v0.19.3 (2026-08-13)
### Added
- Changelog Modal: Adds an in-app changelog you can open from the tray ("View Changelog") or the "What's New" button on the update notification, so the notification stays reachable while you read. It shows the newest published release notes (falling back to the bundled changelog if that fetch fails), folds older versions while keeping the current one expanded, and supports keyboard navigation (`J`/`K`/arrows, `Enter`, `R`, `Esc`).
### Changed
- Subtitle Tokenization Performance: Reworks subtitle dictionary lookups to cut per-line work roughly in half, cache repeated lookups across lines, and stop tokenization from competing with on-screen subtitle prefetching. Also fixes several accuracy issues along the way: dropped readings on trailing kana, character names being skipped after a dictionary sync, annotations not refreshing after mining a card, and halfwidth katakana character names losing their reading or being swallowed by other words.
### Fixed
- Character Dictionary Large Imports: Large character dictionaries (e.g. One Piece) no longer fail to install from a fixed timeout budget; the import now scales its time budget to dictionary size and reports detailed progress (page/character counts, image download progress, elapsed time) instead of one static message.
- Stats Delete Responsiveness: Deleting sessions, episodes, or library entries no longer freezes the stats page or an active video player; deletes are now batched into a single transaction.
- Styled Subtitle Cue Parsing: Heavily typeset subtitles (karaoke, signs) no longer flood the subtitle sidebar with garbage; vector drawing commands are no longer shown as text, and duplicate/animation-burst cues now collapse into one.
- X11 mpv Renderer: Fixes an mpv crash on the first fullscreen toggle for X11/XWayland users with `gpu-next` shaders (e.g. ArtCNN), which was caused by X11 mode forcing the legacy OpenGL renderer.
- X11 Overlay Display Scaling: Fixes the overlay appearing oversized and offset from mpv on X11/XWayland under fractional or mixed-monitor display scaling.
<details>
<summary>Internal changes</summary>
### Internal
- Subtitle text is now decoded from ASS exactly once at ingest, so the renderer, timing tracker, and tokenizer all share one decoded value instead of each re-deriving it.
- Added per-stage debug timings (`scanMs`, `mecabMs`, `frequencyMs`, `annotateMs`) to the subtitle tokenization pipeline log.
</details>
## v0.19.2 (2026-08-04)
### Changed
Symlink
+1
View File
@@ -0,0 +1 @@
AGENTS.md
+2 -2
View File
@@ -41,7 +41,7 @@
"fast-uri": "3.1.5",
"form-data": "4.0.6",
"ip-address": "10.2.0",
"js-yaml": "4.3.0",
"js-yaml": "4.3.1",
"lodash": "4.18.0",
"minimatch": "10.2.5",
"picomatch": "4.0.4",
@@ -498,7 +498,7 @@
"jiti": ["jiti@2.6.1", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ=="],
"js-yaml": ["js-yaml@4.3.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q=="],
"js-yaml": ["js-yaml@4.3.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ=="],
"json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="],
-7
View File
@@ -1,7 +0,0 @@
type: added
area: overlay
- Added an in-app changelog modal, opened from the tray ("View Changelog") or the "What's New" button on the update-available notification, which now stays on screen so "Update" is still reachable after reading the notes. It renders inside the player bounds when a video is playing and in its own window otherwise, the same as the help modal.
- The changelog is fetched from the newest published release, so release notes for versions newer than the installed build are visible; a failed download falls back to the changelog bundled with the install and says so in the modal.
- Versions are foldable: the current `0.x` line is expanded and older lines are folded, matching the docs-site changelog. A badge marks the installed version and newer versions are tagged "New".
- Keyboard: `J`/`K` or arrows move between versions, `Enter` folds/unfolds, `R` refetches, `Esc` closes.
@@ -1,20 +0,0 @@
type: changed
area: subtitles
- Subtitle tokenization no longer runs a duplicate full `parseText` pass per line: the termsFind scanner walk is now the only tokenizer and emits its own hoverable filler runs for unmatched text (parseText is kept only as an error fallback). This roughly halves the dictionary work per line.
- The Yomitan scanning helpers are now installed once per parser window (`__subminerYomitanScan`) instead of re-shipping and re-parsing a ~500-line script for every subtitle line; each line only evaluates a tiny call.
- termsFind lookups are cached across subtitle lines in a window-persistent LRU keyed by substring, so repeated particles and verb forms stop costing backend round trips. The cache invalidates on dictionary/settings changes and window reloads.
- The scanner walk now skips lookups at punctuation and whitespace positions (latin letters and digits still look up, e.g. Tシャツ). The shrinking-window retry ladder keeps following the consumed lengths the backend reports, and only blind guesses (windows the backend consumed whole, which tell it nothing) are capped at four per position. A line that hits that cap escalates to a single `parseText` for the whole line, so a hard line still resolves to dictionary tokens instead of an unparsed run, without letting the ladder run to one lookup per window length.
- Tokenizer runtime dependencies are built once instead of per line, fixing a JLPT lookup cache that never hit (it was keyed on a per-call closure identity and leaked a Map per line) and a `which mecab` availability check that re-ran synchronously on every line when MeCab is absent.
- Subtitle changes no longer restart the prefetch run per line (which discarded in-flight tokenization work); prefetch now only pauses for the live line and restarts on real seeks, cache invalidation, or option changes. Prefetch also stays paused for the whole time the subtitle processing controller is working on the line, including the provisional raw emit that precedes tokenization, so it never competes with the on-screen line for the parser window. The pause is released when the controller reports it has settled, which also covers the lines that finish without an emit (a suppressed duplicate or a failed tokenization) and used to leave prefetching paused indefinitely.
- Added per-stage debug timings (`scanMs`, `mecabMs`, `frequencyMs`, `annotateMs`) to the subtitle tokenization pipeline log.
- Fixed a reading that stopped covering its surface when an unmatched kana run extended the preceding token (for example a trailing る on 待ち合わせ), which silently disabled the known-word reading fallback for those tokens.
- Subtitle prefetching no longer stays paused for the rest of a cue when the same subtitle text is reported twice and there is nothing to tokenize. This covers the startup and overlay priming paths as well as ordinary subtitle changes.
- Character name and image lookups are now refreshed centrally whenever a character dictionary sync changes its content, so a newly added name can no longer be skipped by a stale candidate list.
- A subtitle that was on screen when its annotations were invalidated (by mining a card, for example) is now re-annotated instead of staying plain for the rest of the line.
- Character name annotations no longer cost a dictionary lookup at every position in a line. The scanner now knows which name forms the current title's character dictionary actually contains and only checks where one can start, which removes the whole overhead of having the character dictionary enabled (measured: 21 lookups per line down to 10, the same as with it disabled). Titles with no cached character data keep the previous exhaustive scan, so a missing snapshot costs speed rather than a missing name.
- The cross-line termsFind cache is now bounded by the number of retained dictionary entries as well as by key count, so a run of lookups that each carry hundreds of entries with full glossaries cannot grow the parser window's memory without limit. The budget is re-checked when a lookup resolves, so a single oversized response is dropped rather than parked in the cache and reused.
- The unnamed-mob disambiguator filter (Girl A / Girl B) now only drops a single letter or digit split off a name, instead of every one-character term: a name that is genuinely one character keeps its terms whatever the script (𠮷, あ, 별 김, ア・ベ). The character dictionary and the scanner's name pre-pass also share one Han code-point table now, so a name the dictionary accepts is a name the scanner will look for.
- A character name written in halfwidth katakana takes part in the greedy name pre-pass again, so a longer generic word can no longer swallow the start of it, and it now carries a reading (it used to come out blank, which disables known-word matching and frequency lookups for the token). Voiced halfwidth kana compose properly, so ガク reads ガク rather than ガク, and kana normalization folds halfwidth throughout so those tokens compare equal to the same word written fullwidth. Because the fold makes halfwidth text indexable, the character-name prefilter now judges halfwidth spellings like any other, and a position only bypasses it when an unfoldable voiced mark sits inside the lookup window. That covers a name that starts on a kanji and turns halfwidth later (山ガク), and one stretched out with emphatic characters in between (山ーーーーーーガク).
- Dictionary-entry classification (source dictionaries, character-dictionary media ids) is memoized per entry object for as long as the entry is cached, instead of being recomputed for every headword comparison and every retry window.
- Autoplay priming no longer broadcasts the plain subtitle twice: it tells the processing controller the line has already been painted, so the controller goes straight to the annotated payload.
+24
View File
@@ -1,5 +1,29 @@
# Changelog
## v0.19.3 (2026-08-13)
**Added**
- Changelog Modal: Adds an in-app changelog you can open from the tray ("View Changelog") or the "What's New" button on the update notification, so the notification stays reachable while you read. It shows the newest published release notes (falling back to the bundled changelog if that fetch fails), folds older versions while keeping the current one expanded, and supports keyboard navigation (`J`/`K`/arrows, `Enter`, `R`, `Esc`).
**Changed**
- Subtitle Tokenization Performance: Reworks subtitle dictionary lookups to cut per-line work roughly in half, cache repeated lookups across lines, and stop tokenization from competing with on-screen subtitle prefetching. Also fixes several accuracy issues along the way: dropped readings on trailing kana, character names being skipped after a dictionary sync, annotations not refreshing after mining a card, and halfwidth katakana character names losing their reading or being swallowed by other words.
**Fixed**
- Character Dictionary Large Imports: Large character dictionaries (e.g. One Piece) no longer fail to install from a fixed timeout budget; the import now scales its time budget to dictionary size and reports detailed progress (page/character counts, image download progress, elapsed time) instead of one static message.
- Stats Delete Responsiveness: Deleting sessions, episodes, or library entries no longer freezes the stats page or an active video player; deletes are now batched into a single transaction.
- Styled Subtitle Cue Parsing: Heavily typeset subtitles (karaoke, signs) no longer flood the subtitle sidebar with garbage; vector drawing commands are no longer shown as text, and duplicate/animation-burst cues now collapse into one.
- X11 mpv Renderer: Fixes an mpv crash on the first fullscreen toggle for X11/XWayland users with `gpu-next` shaders (e.g. ArtCNN), which was caused by X11 mode forcing the legacy OpenGL renderer.
- X11 Overlay Display Scaling: Fixes the overlay appearing oversized and offset from mpv on X11/XWayland under fractional or mixed-monitor display scaling.
<details>
<summary>Internal changes</summary>
**Internal**
- Subtitle text is now decoded from ASS exactly once at ingest, so the renderer, timing tracker, and tokenizer all share one decoded value instead of each re-deriving it.
- Added per-stage debug timings (`scanMs`, `mecabMs`, `frequencyMs`, `annotateMs`) to the subtitle tokenization pipeline log.
</details>
## v0.19.2 (2026-08-04)
**Changed**
+4 -3
View File
@@ -405,8 +405,9 @@ On any Wayland session that is not Hyprland or Sway (KDE Plasma, GNOME, and othe
SubMiner handles this automatically:
- It launches its own window under XWayland (it sets `--ozone-platform-hint=x11`).
- Every mpv it launches (via the `subminer` launcher, Jellyfin, or YouTube) is pinned to XWayland too - Wayland environment hints are stripped and an X11 GPU context (`--gpu-context=x11egl,x11`) is applied.
- It launches its own window under XWayland (it sets `--ozone-platform=x11`).
- Every mpv it launches (via the `subminer` launcher, Jellyfin, or YouTube) is pinned to XWayland too - Wayland environment hints are stripped and an X11 GPU context (`--gpu-context=x11vk,x11egl,x11`) is applied. Only the window context is overridden; your `vo`/`gpu-api` and user shaders are left alone.
- Fractional and mixed-monitor display scaling is handled per screen when SubMiner maps XWayland mpv coordinates to the overlay.
- While mpv is windowed, the overlay is a managed X11 window owned by the tracked mpv window (`WM_TRANSIENT_FOR`), so it stays above mpv while other foreground X11/Xwayland apps can still cover both windows.
- While tracked mpv is fullscreen, SubMiner swaps the visible overlay to a focusable-false X11 override-redirect window. That path can stay above the active fullscreen mpv window without requiring a KDE/KWin-specific rule, and SubMiner hides/releases it when mpv is no longer the active X11/Xwayland window.
- The visible overlay is shown inactive on Linux, so normal hover should not steal keyboard focus from mpv.
@@ -420,7 +421,7 @@ Requirements: `xdotool`, `xprop`, and `xwininfo` must be installed. SubMiner use
This almost always means mpv came up as a **native Wayland** window that the XWayland overlay cannot cover. It happens when mpv is launched **manually** (your own command), because SubMiner can only force XWayland on the mpv processes it launches itself. Fix it one of these ways:
- Launch playback through SubMiner (the `subminer` launcher or the tray), which forces XWayland for you, or
- Force XWayland in your own mpv invocation, e.g. `mpv --gpu-context=x11egl …`, or launch with `WAYLAND_DISPLAY= mpv …`, or set `gpu-context=x11egl` in your `mpv.conf`.
- Force XWayland in your own mpv invocation, e.g. `mpv --gpu-context=x11vk,x11egl,x11 …`, or launch with `WAYLAND_DISPLAY= mpv …`, or set `gpu-context=x11vk` (Vulkan) / `gpu-context=x11egl` (OpenGL) in your `mpv.conf`.
To confirm mpv is on XWayland, `xdotool search --class mpv` should return a window id (a native Wayland mpv returns nothing).
@@ -64,18 +64,23 @@ External subtitle files only (SRT, VTT, ASS). Embedded subtitle tracks are out o
A cue parser extracts both timing and text content from subtitle files for prefetching.
**Parsed cue structure:**
```typescript
interface SubtitleCue {
startTime: number; // seconds
endTime: number; // seconds
text: string; // raw subtitle text
startTime: number; // seconds
endTime: number; // seconds
text: string; // plain text, decoded from the source format
}
```
**Supported formats:**
- SRT/VTT: Regex-based parsing of timing lines + text content between timing blocks.
- ASS: Parse `[Events]` section, extract `Dialogue:` lines, split on the first 9 commas only (ASS v4+ has 10 fields; the last field is Text which can itself contain commas). Strip ASS override tags (`{\...}`) from the text before storing.
ASS text fields contain inline override tags like `{\b1}`, `{\an8}`, `{\fad(200,300)}`. The cue parser strips these during extraction so the tokenizer receives clean text.
- ASS: Parse `[Events]` section, extract `Dialogue:` lines, read the field order from the `Format:` row, and take everything after the Text field index as the text (Text can itself contain commas).
**ASS decoding.** The parser is where ASS text is decoded, once, via `assToPlainText()` in `src/core/services/ass-text.ts`. That decoder mirrors mpv's `ass_to_plaintext` so a cue read from a file reads identically to the same line arriving live on `sub-text`: `{...}` override blocks are markup, `\pN … \p0` vector drawing runs are dropped rather than shown as text, `\N`/`\n`/`\h` are the only escapes (`\{`, `\}` and `\\` are not), and an unclosed `{` is rendered verbatim. Every layer downstream — renderer, timing tracker, tokenizer, tokenization cache keys — receives plain text and uses `normalizePlainSubtitleText()` for whitespace only, so nothing decodes the same string twice and one authored line always maps to one cache key.
**Duplicate collapsing.** Typeset scripts emit one `Dialogue:` event per animation frame, plus layered copies of the same line. The parser collapses identical text over an identical span unconditionally, and collapses contiguous same-text runs of at least three events when the run looks like an animation. For ASS that means shared style and actor plus authoring evidence: a temporal tag (`\t`, `\move`, `\k`/`\kf`/`\ko`/`\K`, or anything wrapped in `\t(...)`), an animated `Effect` column (`Karaoke`, `Banner`, `Scroll`), or override values that change across the run. Static tags shared by every event (`\pos`, an identical `\clip`) are not evidence. SRT/VTT carry no such metadata, so there collapsing needs at least five contiguous events all under 0.1s — the frame timing left behind by ASS-to-SRT conversion. The parser keeps this authoring metadata (style, actor, layer, `Effect`, parsed override commands, source order) private; `parseSubtitleCues()` returns only `SubtitleCue`.
#### Prefetch Service Lifecycle
@@ -153,6 +158,7 @@ tokens (already have frequencyRank values from parser-level applyFrequencyRanks)
### Dependency Analysis
All annotations either depend on MeCab POS data or benefit from running after it:
- **Known word marking:** Needs base tokens (surface/headword). No POS dependency, but no reason to run separately.
- **Frequency filtering:** Uses `pos1Exclusions` and `pos2Exclusions` to clear frequency ranks on excluded tokens (particles, noise). Depends on MeCab POS data.
- **JLPT marking:** Uses `shouldIgnoreJlptForMecabPos1` to filter. Depends on MeCab POS data.
@@ -169,18 +175,14 @@ function annotateTokens(tokens, deps, options): MergedToken[] {
// Single pass: known word + frequency filtering + JLPT computed together
const annotated = tokens.map((token) => {
const isKnown = nPlusOneEnabled
? token.isKnown || computeIsKnown(token, deps)
: false;
const isKnown = nPlusOneEnabled ? token.isKnown || computeIsKnown(token, deps) : false;
// Filter frequency rank using POS exclusions (rank values already set at parser level)
const frequencyRank = frequencyEnabled
? filterFrequencyRank(token, pos1Exclusions, pos2Exclusions)
: undefined;
const jlptLevel = jlptEnabled
? computeJlptLevel(token, deps.getJlptLevel)
: undefined;
const jlptLevel = jlptEnabled ? computeJlptLevel(token, deps.getJlptLevel) : undefined;
return { ...token, isKnown, frequencyRank, jlptLevel };
});
@@ -221,6 +223,7 @@ Replace `document.createElement('span')` calls in the renderer with `templateSpa
### Current Behavior
In `renderWithTokens` (`subtitle-render.ts`), each render cycle:
1. Clears DOM with `innerHTML = ''`
2. Creates a `DocumentFragment`
3. Calls `document.createElement('span')` for each token (~10-15 per subtitle)
@@ -256,27 +259,30 @@ Full recycling (collecting old nodes, clearing attributes, reusing them) require
## Combined Impact Summary
| Scenario | Before | After | Improvement |
|----------|--------|-------|-------------|
| Normal playback (prefetch-warmed) | ~200-320ms | ~30-50ms | ~80-85% |
| Cache hit (repeated subtitle) | ~72ms | ~55-65ms | ~10-20% |
| Cache miss (immediate seek) | ~200-320ms | ~150-260ms | ~20-25% |
| Scenario | Before | After | Improvement |
| --------------------------------- | ---------- | ---------- | ----------- |
| Normal playback (prefetch-warmed) | ~200-320ms | ~30-50ms | ~80-85% |
| Cache hit (repeated subtitle) | ~72ms | ~55-65ms | ~10-20% |
| Cache miss (immediate seek) | ~200-320ms | ~150-260ms | ~20-25% |
---
## Files Summary
### New Files
- `src/core/services/subtitle-prefetch.ts`
- `src/core/services/subtitle-cue-parser.ts`
### Modified Files
- `src/core/services/subtitle-processing-controller.ts` (expose `preCacheTokenization`)
- `src/core/services/tokenizer/annotation-stage.ts` (batched single-pass)
- `src/renderer/subtitle-render.ts` (template cloneNode)
- `src/main.ts` (wire up prefetch service)
### Test Files
- New tests for subtitle cue parser (SRT, VTT, ASS formats)
- New tests for subtitle prefetch service (priority window, seek, pause/resume)
- Updated tests for annotation stage (same behavior, new implementation)
+1
View File
@@ -25,6 +25,7 @@ Read when: you need to find the owner module for a behavior or test surface
- Anki workflow: `src/anki-integration/`, `src/core/services/anki-jimaku*.ts`
- Immersion tracking: `src/core/services/immersion-tracker/`
Includes stats storage/query schema such as `imm_videos`, `imm_media_art`, and `imm_youtube_videos` for per-video and YouTube-specific library metadata.
`delete-maintenance-scheduler.ts` coalesces and serializes stats deletes; expensive deletion and summary rebuilds run in `delete-maintenance-worker-thread.ts` while the tracker queues playback writes. Each batch uses one transaction, lexical update, rollup refresh, and lifetime rebuild.
- AniList tracking + character dictionary: `src/core/services/anilist/`, `src/main/runtime/composers/anilist-*`, `src/main/character-dictionary-runtime.ts`, `src/main/character-dictionary-runtime/`
- Jellyfin integration: `src/core/services/jellyfin*.ts`, `src/main/runtime/composers/jellyfin-*`
- Window trackers: `src/window-trackers/`
+5 -7
View File
@@ -222,7 +222,7 @@ test('buildMpvEnv preserves native Wayland env for supported Hyprland and Sway a
});
});
test('buildMpvBackendArgs forces an explicit X11 renderer stack when backend resolves to x11', () => {
test('buildMpvBackendArgs pins the X11 window context when backend resolves to x11', () => {
withPlatform('linux', () => {
assert.deepEqual(
buildMpvBackendArgs(makeArgs({ backend: 'x11' }), {
@@ -230,12 +230,12 @@ test('buildMpvBackendArgs forces an explicit X11 renderer stack when backend res
WAYLAND_DISPLAY: 'wayland-0',
XDG_SESSION_TYPE: 'wayland',
}),
['--vo=gpu', '--gpu-api=opengl', '--gpu-context=x11egl,x11'],
['--gpu-context=x11vk,x11egl,x11'],
);
});
});
test('buildMpvBackendArgs forces the same X11 renderer stack for unsupported Wayland auto fallback', () => {
test('buildMpvBackendArgs pins the same X11 window context for unsupported Wayland auto fallback', () => {
withPlatform('linux', () => {
assert.deepEqual(
buildMpvBackendArgs(makeArgs({ backend: 'auto' }), {
@@ -245,7 +245,7 @@ test('buildMpvBackendArgs forces the same X11 renderer stack for unsupported Way
XDG_CURRENT_DESKTOP: 'KDE',
XDG_SESSION_DESKTOP: 'plasma',
}),
['--vo=gpu', '--gpu-api=opengl', '--gpu-context=x11egl,x11'],
['--gpu-context=x11vk,x11egl,x11'],
);
});
});
@@ -292,9 +292,7 @@ test('buildConfiguredMpvDefaultArgs appends maximized launch mode to configured
'--secondary-sub-visibility=no',
'--alang=ja,jp,jpn,japanese,en,eng,english,enus,en-us',
'--slang=ja,jp,jpn,japanese,en,eng,english,enus,en-us',
'--vo=gpu',
'--gpu-api=opengl',
'--gpu-context=x11egl,x11',
'--gpu-context=x11vk,x11egl,x11',
'--window-maximized=yes',
],
);
+2 -2
View File
@@ -2,7 +2,7 @@
"name": "subminer",
"productName": "SubMiner",
"desktopName": "SubMiner.desktop",
"version": "0.19.2",
"version": "0.19.3",
"description": "All-in-one sentence mining overlay with AnkiConnect and dictionary integration",
"packageManager": "bun@1.3.5",
"main": "dist/main-entry.js",
@@ -89,7 +89,7 @@
"fast-uri": "3.1.5",
"form-data": "4.0.6",
"ip-address": "10.2.0",
"js-yaml": "4.3.0",
"js-yaml": "4.3.1",
"lodash": "4.18.0",
"minimatch": "10.2.5",
"picomatch": "4.0.4",
+48
View File
@@ -0,0 +1,48 @@
## Highlights
### Added
- **In-App Changelog**
- View release notes without leaving the app, from the tray menu ("View Changelog") or the "What's New" button on update notifications.
- Shows notes for the latest published release even when it's newer than your installed build, and falls back to the notes bundled with your install if the download fails.
- Older versions fold automatically, your installed version is badged, and newer ones are tagged "New"; navigate with `J`/`K` or the arrow keys, `Enter` to expand/collapse, `R` to refresh, and `Esc` to close.
### Changed
- **Faster Subtitle Tokenization**
- Subtitle lines are parsed and looked up roughly twice as efficiently, with results cached across lines so repeated words and grammar no longer re-query the dictionary.
- Enabling a character dictionary no longer slows subtitle scanning as much, since name lookups now only check positions where a known name can actually start.
- Fixed related accuracy issues along the way: readings that could go missing on certain word endings, subtitle text that stayed unannotated after mining a card, character names that could drop out of disambiguation rules, and halfwidth-katakana character names that weren't recognized or read correctly.
### Fixed
- **Large Character Dictionary Generation**
- Big character dictionaries (long-running series like One Piece) no longer fail to install with a timeout error; the import time budget now scales with dictionary size instead of using a fixed 7-second limit.
- The "Generating character dictionary" notification now shows real progress (character/page counts, image download progress with an ETA, name-processing progress) and an elapsed-time clock, so a long-running import no longer looks frozen.
- **Stats Deletion Responsiveness**
- Deleting sessions, episodes, or library entries on the stats page no longer freezes the page or an active video player; deletes are now batched into a single transaction.
- **Subtitle Sidebar Clutter from Styled Subtitles**
- Heavily typeset subtitles (karaoke openings/endings, stylized signs) no longer flood the subtitle sidebar with garbled vector-drawing text or duplicate "shadow" copies of the same line.
- Subtitle text is now decoded consistently in one place, matching what mpv actually renders on screen, so it can no longer diverge or get cached inconsistently.
- **X11/XWayland Playback and Overlay Fixes**
- Fixed a crash on the first fullscreen toggle when using an mpv `gpu-next` shader (e.g. ArtCNN) in X11/XWayland mode; SubMiner no longer forces mpv onto its older OpenGL renderer.
- Fixed the overlay appearing oversized and offset from the video under fractional or mixed-monitor display scaling in X11/XWayland mode.
## What's Changed
- perf(tokenizer): single-pass Yomitan scan with cross-line caching and prefetch fixes by @ksyasuda in #185
- fix(subtitles): collapse duplicate ASS events and decode text once by @ksyasuda in #186
- feat(overlay): add in-app changelog modal by @ksyasuda in #187
- fix(playback): stop forcing legacy OpenGL renderer on X11 mpv backend by @ksyasuda in #188
- fix(dictionary): stop large character dictionaries from timing out by @ksyasuda in #189
- fix(overlay): handle X11 display scaling across monitors by @ksyasuda in #193
- fix(stats): batch deletes off the main thread by @ksyasuda in #194
## Installation
See the README and docs/installation guide for full setup steps.
## Assets
- Linux: `SubMiner.AppImage`
- macOS: `SubMiner-*.dmg` and `SubMiner-*.zip`
- Windows: `SubMiner-*.exe` and `SubMiner-*-win.zip`
- Optional extras: `subminer-assets.tar.gz` and the `subminer` launcher
Note: the `subminer` wrapper script uses Bun (`#!/usr/bin/env bun`), so `bun` must be installed and on `PATH`.
+195
View File
@@ -0,0 +1,195 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
assOverrideSignature,
assToPlainText,
collectAssOverrideCommands,
extractAssOverrideBlocks,
hasAssTemporalOverride,
isAnimatedAssEffectKind,
isAssTemporalCommand,
normalizePlainSubtitleText,
parseAssEffectField,
} from './ass-text';
test('assToPlainText drops vector drawing runs', () => {
assert.equal(
assToPlainText(
'{\\an5\\pos(730,1042)\\p1\\blur1}m 20 0 b 10 0 0 10 0 20 b 0 31 10 40 20 40 {\\p0}',
),
'',
);
});
test('assToPlainText keeps text around drawing runs on the same event', () => {
assert.equal(
assToPlainText('{\\p1}m 0 0 l 10 10{\\p0}本文{\\p1}m 5 5 l 6 6{\\p0}続き'),
'本文続き',
);
});
test('assToPlainText leaves \\pos alone when no drawing mode is active', () => {
assert.equal(assToPlainText('{\\pos(960,1068)\\bord3}位置指定'), '位置指定');
});
test('assToPlainText does not read \\pos as a drawing tag', () => {
assert.equal(assToPlainText('{\\p1\\pos(1,2)}m 0 0 l 5 5'), '');
});
test('assToPlainText resolves line-break and space escapes', () => {
assert.equal(assToPlainText('一行目\\N二行目'), '一行目\n二行目');
assert.equal(assToPlainText('一行目\\n二行目'), '一行目\n二行目');
assert.equal(assToPlainText('一行目\\N二行目', ' '), '一行目 二行目');
assert.equal(assToPlainText('間\\h隔'), '間 隔');
});
test('assToPlainText matches mpv on brace and backslash sequences', () => {
// mpv has no `\{` / `\}` / `\\` escapes: the backslashes are literal text and the
// braces still open and close an override block.
assert.equal(assToPlainText('\\{注\\}'), '\\');
assert.equal(assToPlainText('\\\\N'), '\\\n');
});
test('assToPlainText renders an unclosed override block verbatim', () => {
// mpv shows the stray brace; guessing where the block ended can eat a whole line.
assert.equal(assToPlainText('本文{\\pos(1,2)'), '本文{\\pos(1,2)');
});
test('assToPlainText is idempotent', () => {
const samples = [
'{\\an5\\p1}m 0 0 l 5 5{\\p0}本文',
'\\{注\\}',
'\\\\N',
'本文{\\pos(1,2)',
'一行目\\N二行目\\h終わり',
];
for (const sample of samples) {
const once = assToPlainText(sample);
assert.equal(assToPlainText(once), once, sample);
}
});
test('assToPlainText normalizes CRLF before converting', () => {
assert.equal(assToPlainText('一行目\r\n二行目'), '一行目\n二行目');
});
test('normalizePlainSubtitleText settles whitespace without decoding ASS', () => {
// A brace reaching this layer is literal text mpv chose to show, not markup.
assert.equal(normalizePlainSubtitleText('本文{\\pos(1,2)'), '本文{\\pos(1,2)');
assert.equal(normalizePlainSubtitleText('一行目\\N二行目'), '一行目\n二行目');
assert.equal(
normalizePlainSubtitleText('一行目\\N二行目', { collapseLineBreaks: true }),
'一行目 二行目',
);
assert.equal(normalizePlainSubtitleText(' 余白 ', { trim: false }), ' 余白 ');
});
test('normalizePlainSubtitleText is idempotent', () => {
for (const sample of ['一行目\\N二行目', '間\\h隔', '本文{\\pos(1,2)', ' 余白 ']) {
const once = normalizePlainSubtitleText(sample);
assert.equal(normalizePlainSubtitleText(once), once, sample);
}
});
test('extractAssOverrideBlocks returns block contents', () => {
assert.deepEqual(extractAssOverrideBlocks('{\\an8}上{\\fad(200,200)}下'), [
'\\an8',
'\\fad(200,200)',
]);
assert.deepEqual(extractAssOverrideBlocks('括弧なし'), []);
});
test('collectAssOverrideCommands captures names and arguments from blocks only', () => {
const commands = collectAssOverrideCommands('{\\pos(1,2)\\1c&HFFFFFF&\\kf30}歌詞');
assert.deepEqual(commands, [
{ name: 'pos', args: '1,2', animated: false },
{ name: '1c', args: '&HFFFFFF&', animated: false },
{ name: 'kf', args: '30', animated: false },
]);
// A `\pos(...)` sitting in visible text is not typesetting markup.
assert.deepEqual(collectAssOverrideCommands('\\pos(730,1042) と書いてある'), []);
});
test('collectAssOverrideCommands marks tags animated by a wrapping \\t', () => {
const commands = collectAssOverrideCommands('{\\clip(0,0,10,10)\\t(0,500,\\frz30)}文字');
assert.deepEqual(
commands.map((command) => [command.name, command.animated]),
[
['clip', false],
['t', false],
['frz', true],
],
);
assert.equal(hasAssTemporalOverride(commands), true);
});
test('collectAssOverrideCommands stops descending into deeply nested \\t tags', () => {
// Nested far past the recursion cap. Uncapped, this recurses once per level, and a
// pathological line (real files reach one or two levels) overflows the stack.
const nesting = 32;
const block = `{${'\\t(0,500,'.repeat(nesting)}\\frz30${')'.repeat(nesting)}}文字`;
const commands = collectAssOverrideCommands(block);
// The outer `\t` plus one per allowed recursion level, and nothing from below the cap.
assert.equal(commands.length, 9);
assert.deepEqual(new Set(commands.map((command) => command.name)), new Set(['t']));
assert.equal(hasAssTemporalOverride(commands), true);
});
test('hasAssTemporalOverride ignores static placement and shape tags', () => {
assert.equal(
hasAssTemporalOverride(collectAssOverrideCommands('{\\pos(1,2)\\clip(m 1 1)\\blur2}文字')),
false,
);
assert.equal(hasAssTemporalOverride(collectAssOverrideCommands('{\\move(1,2,3,4)}文字')), true);
});
test('isAssTemporalCommand covers only intrinsically animated tags', () => {
for (const command of ['t', 'move', 'k', 'kf', 'ko', 'K']) {
assert.equal(isAssTemporalCommand(command), true, command);
}
for (const command of ['clip', 'iclip', 'frz', 'fscx', 'blur', 'be', 'pos', 'fad']) {
assert.equal(isAssTemporalCommand(command), false, command);
}
});
test('assOverrideSignature distinguishes events by their override values', () => {
const first = assOverrideSignature(collectAssOverrideCommands('{\\clip(m 1 1)}歌詞'));
const second = assOverrideSignature(collectAssOverrideCommands('{\\clip(m 2 2)}歌詞'));
const repeat = assOverrideSignature(collectAssOverrideCommands('{\\clip(m 1 1)}別の行'));
assert.notEqual(first, second);
assert.equal(first, repeat);
});
test('parseAssEffectField classifies the event-level Effect column', () => {
assert.equal(parseAssEffectField(''), 'none');
assert.equal(parseAssEffectField(' '), 'none');
assert.equal(parseAssEffectField('Banner;20;1;0'), 'banner');
assert.equal(parseAssEffectField('Scroll up;0;0;30;10'), 'scroll');
assert.equal(parseAssEffectField('Scroll down;0;0;30;10'), 'scroll');
assert.equal(parseAssEffectField('Karaoke'), 'karaoke');
assert.equal(parseAssEffectField('fx-template'), 'other');
});
test('parseAssEffectField matches stock effect names exactly', () => {
// Custom effect names that merely start with a stock name are not stock effects.
assert.equal(parseAssEffectField('scrolling-credit'), 'other');
assert.equal(parseAssEffectField('bannerfx;1'), 'other');
assert.equal(parseAssEffectField('karaoke-template'), 'other');
assert.equal(parseAssEffectField('Scroll'), 'other');
});
test('isAnimatedAssEffectKind covers the stock animated effects only', () => {
assert.equal(isAnimatedAssEffectKind('karaoke'), true);
assert.equal(isAnimatedAssEffectKind('banner'), true);
assert.equal(isAnimatedAssEffectKind('scroll'), true);
// Typesetting groups put static template names in this column too.
assert.equal(isAnimatedAssEffectKind('other'), false);
assert.equal(isAnimatedAssEffectKind('none'), false);
});
+280
View File
@@ -0,0 +1,280 @@
/*
* ASS/SSA text handling, split into two deliberately distinct contracts:
*
* assToPlainText() raw ASS event text -> plain text. Ingestion only.
* normalizePlainSubtitleText() already-decoded text -> display/lookup form.
*
* Subtitle text is decoded from ASS exactly once, at the point it enters the app: the
* file cue parser does it for sidecar/embedded scripts, and mpv does it for live text
* (`sub-text` is already run through mpv's own `ass_to_plaintext`). Everything
* downstream -- renderer, timing tracker, tokenizer, tokenization cache keys -- gets
* plain text and only normalizes whitespace, so no layer decodes the same string twice.
*
* assToPlainText mirrors mpv's `ass_to_plaintext` rather than inventing its own rules,
* so a cue parsed from a file reads the same as the same line arriving live:
* - `{...}` override blocks are markup
* - `\pN ... \p0` runs are vector paths, not dialogue
* - `\N`, `\n` and `\h` are the only escapes; `\{`, `\}` and `\\` are NOT escapes,
* so `\{注\}` decodes to a lone backslash exactly as mpv renders it
* - an unclosed `{` is rendered verbatim instead of swallowing the rest of the line
* Because the decoder never emits an escape or a closed brace, running it twice is a
* no-op -- but downstream code should still use normalizePlainSubtitleText.
*/
/** What `\N` and `\n` become. */
export type AssLineBreak = '\n' | ' ';
// `\p<n>` with n > 0 switches libass into vector-drawing mode: everything until the
// next `\p0` is a path (`m 20 0 b 10 0 ...`), not dialogue. The negative lookahead keeps
// `\pos(...)` from being read as a drawing tag.
const ASS_DRAWING_SCALE_PATTERN = /\\p(?![a-zA-Z])(\d*)/g;
function readDrawingScale(block: string): number | null {
ASS_DRAWING_SCALE_PATTERN.lastIndex = 0;
let scale: number | null = null;
let match: RegExpExecArray | null;
// Drawing mode is whatever the last `\p` tag in this block set it to.
while ((match = ASS_DRAWING_SCALE_PATTERN.exec(block)) !== null) {
scale = match[1] ? Number(match[1]) : 0;
}
return scale;
}
/** Resolve `\N`, `\n` and `\h`. The only text-level escapes libass recognises. */
function resolveWhitespaceEscapes(text: string, lineBreak: AssLineBreak): string {
return text.replace(/\\([Nnh])/g, (_match, escaped: string) =>
escaped === 'h' ? ' ' : lineBreak,
);
}
/** Strip `{...}` override blocks and the drawing runs they enable. */
function stripAssMarkup(raw: string): string {
let out = '';
let cursor = 0;
let drawing = false;
while (cursor < raw.length) {
if (raw[cursor] !== '{') {
if (!drawing) {
out += raw[cursor];
}
cursor += 1;
continue;
}
const close = raw.indexOf('}', cursor + 1);
if (close === -1) {
// mpv shows an unclosed `{` and everything after it. Guessing where the block was
// meant to end can eat a whole line of dialogue.
if (!drawing) {
out += raw.slice(cursor);
}
break;
}
const scale = readDrawingScale(raw.slice(cursor, close + 1));
if (scale !== null) {
drawing = scale > 0;
}
cursor = close + 1;
}
return out;
}
/**
* Decode a raw ASS/SSA event text field. Call this once, where the text enters the app;
* downstream layers take the result as plain text.
*/
export function assToPlainText(text: string, lineBreak: AssLineBreak = '\n'): string {
if (!text) return '';
return resolveWhitespaceEscapes(stripAssMarkup(text.replace(/\r\n/g, '\n')), lineBreak);
}
export interface NormalizePlainSubtitleTextOptions {
/** Fold every line break into a single space. */
collapseLineBreaks?: boolean;
trim?: boolean;
}
/**
* Whitespace normalization for text that has already been decoded -- by mpv for live
* subtitles, by the cue parser for files. Override blocks and drawing runs are none of
* this function's business; a `{` that reaches here is literal text mpv chose to show.
*
* `\N`/`\n`/`\h` are still folded, because subtitle sources outside the ASS path (asbplayer
* and other websocket clients) forward them raw and the display layer has to cope.
*/
export function normalizePlainSubtitleText(
text: string,
options: NormalizePlainSubtitleTextOptions = {},
): string {
if (!text) return '';
const { collapseLineBreaks = false, trim = true } = options;
let normalized = resolveWhitespaceEscapes(
text.replace(/\r\n/g, '\n'),
collapseLineBreaks ? ' ' : '\n',
);
if (collapseLineBreaks) {
normalized = normalized.replace(/\n/g, ' ').replace(/\s+/g, ' ');
}
return trim ? normalized.trim() : normalized;
}
/** The contents of each `{...}` block, without the braces. */
export function extractAssOverrideBlocks(text: string): string[] {
const blocks: string[] = [];
let cursor = 0;
while (cursor < text.length) {
const open = text.indexOf('{', cursor);
if (open === -1) {
break;
}
const close = text.indexOf('}', open + 1);
if (close === -1) {
break;
}
blocks.push(text.slice(open + 1, close));
cursor = close + 1;
}
return blocks;
}
export interface AssOverrideCommand {
/** Tag name without the backslash, e.g. `pos`, `kf`, `1c`. */
name: string;
/** Everything the tag was given, e.g. `960,1068` for `\pos(960,1068)`. */
args: string;
/** Nested inside a `\t(...)` argument, so its value is animated over the event. */
animated: boolean;
}
const ASS_OVERRIDE_NAME_PATTERN = /[1-4]?[a-zA-Z]+/y;
function readCommandArgs(block: string, start: number): { args: string; next: number } {
if (block[start] === '(') {
let depth = 0;
for (let i = start; i < block.length; i += 1) {
if (block[i] === '(') depth += 1;
else if (block[i] === ')') {
depth -= 1;
if (depth === 0) {
return { args: block.slice(start + 1, i), next: i + 1 };
}
}
}
return { args: block.slice(start + 1), next: block.length };
}
const nextTag = block.indexOf('\\', start);
const end = nextTag === -1 ? block.length : nextTag;
return { args: block.slice(start, end), next: end };
}
// `\t(...)` can wrap another `\t(...)`, and nothing in the format stops an author (or a
// malformed file) from nesting them thousands deep. Real typesetting never goes past one
// or two levels, so stop recursing well before the call stack is at risk.
const MAX_ANIMATION_NESTING_DEPTH = 8;
function parseOverrideBlock(
block: string,
animated: boolean,
into: AssOverrideCommand[],
depth = 0,
): void {
let cursor = 0;
while (cursor < block.length) {
if (block[cursor] !== '\\') {
cursor += 1;
continue;
}
ASS_OVERRIDE_NAME_PATTERN.lastIndex = cursor + 1;
const nameMatch = ASS_OVERRIDE_NAME_PATTERN.exec(block);
if (!nameMatch) {
cursor += 1;
continue;
}
const name = nameMatch[0];
const { args, next } = readCommandArgs(block, cursor + 1 + name.length);
into.push({ name, args: args.trim(), animated });
// `\t(0,500,\frz30)` animates whatever it wraps, so record the inner tags too.
if (name === 't' && args.includes('\\') && depth < MAX_ANIMATION_NESTING_DEPTH) {
parseOverrideBlock(args, true, into, depth + 1);
}
cursor = next;
}
}
/**
* Override commands with their arguments, in source order. Only `{...}` blocks are
* inspected, so a `\pos(...)` sitting in visible text is never mistaken for markup.
*/
export function collectAssOverrideCommands(text: string): AssOverrideCommand[] {
const commands: AssOverrideCommand[] = [];
for (const block of extractAssOverrideBlocks(text)) {
parseOverrideBlock(block, false, commands);
}
return commands;
}
// Tags that are animated by definition: `\t` interpolates, `\move` travels, and the
// karaoke tags advance a highlight across the event's own duration. Everything else --
// `\pos`, `\clip`, `\frz`, `\blur`, `\fad` -- is a static value for the event, so its
// presence says nothing about whether neighbouring events form one animation.
const ASS_TEMPORAL_COMMANDS = new Set(['t', 'move', 'k', 'kf', 'ko', 'K']);
export function isAssTemporalCommand(name: string): boolean {
return ASS_TEMPORAL_COMMANDS.has(name);
}
/** True when the event animates on its own, or animates a static tag through `\t(...)`. */
export function hasAssTemporalOverride(commands: readonly AssOverrideCommand[]): boolean {
return commands.some((command) => command.animated || isAssTemporalCommand(command.name));
}
/**
* Canonical form of an event's override values, for comparing consecutive events. Two
* events with the same signature were typeset identically, so neither is a frame of an
* animation the other belongs to.
*/
export function assOverrideSignature(commands: readonly AssOverrideCommand[]): string {
return commands.map((command) => `${command.name}(${command.args})`).join('|');
}
export type AssEffectKind = 'none' | 'banner' | 'scroll' | 'karaoke' | 'other';
// The stock effects, matched exactly. Typesetting groups put their own template names in
// this column -- `scrolling-credit` is a static sign, not libass's `Scroll up` -- so a
// prefix match would hand out animation evidence to arbitrary custom effects.
const STOCK_ASS_EFFECTS = new Map<string, AssEffectKind>([
['banner', 'banner'],
['scroll up', 'scroll'],
['scroll down', 'scroll'],
['karaoke', 'karaoke'],
]);
/**
* The event-level `Effect` column. The stock values (`Banner;...`, `Scroll up;...`,
* `Scroll down;...`, `Karaoke`) all animate; anything else is a custom name and lands in
* `other`.
*/
export function parseAssEffectField(raw: string): AssEffectKind {
const value = raw.trim().toLowerCase();
if (!value) return 'none';
const name = value.split(';', 1)[0]!.trim();
return STOCK_ASS_EFFECTS.get(name) ?? 'other';
}
const ANIMATED_ASS_EFFECT_KINDS = new Set<AssEffectKind>(['banner', 'scroll', 'karaoke']);
export function isAnimatedAssEffectKind(kind: AssEffectKind): boolean {
return ANIMATED_ASS_EFFECT_KINDS.has(kind);
}
@@ -1414,6 +1414,353 @@ test('deleteSession ignores the currently active session and keeps new writes fl
}
});
test('deleteSession yields the main event loop while delete maintenance is pending', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
const deleteGate: { release?: () => void } = {};
let deleteRunnerCalled = false;
let bufferedWritesAtDeleteStart = -1;
try {
const Ctor = await loadTrackerCtor();
const createdTracker = new Ctor(
{ dbPath },
{
runDeleteMaintenanceTask: async () => {
deleteRunnerCalled = true;
bufferedWritesAtDeleteStart = (tracker as unknown as { queue: unknown[] }).queue.length;
await new Promise<void>((resolve) => {
deleteGate.release = resolve;
});
},
},
);
tracker = createdTracker;
createdTracker.handleMediaChange('/tmp/delete-yield-first.mkv', 'Delete Yield First');
createdTracker.handleMediaChange('/tmp/delete-yield-active.mkv', 'Delete Yield Active');
const privateApi = createdTracker as unknown as {
db: DatabaseSync;
queue: unknown[];
flushNow: () => void;
};
const sessionId = (
privateApi.db
.prepare(
`SELECT session_id AS sessionId
FROM imm_sessions
WHERE ended_at_ms IS NOT NULL
ORDER BY session_id
LIMIT 1`,
)
.get() as { sessionId: number } | null
)?.sessionId;
assert.ok(sessionId);
const deletePromise = createdTracker.deleteSession(sessionId);
let timerAdvanced = false;
setTimeout(() => {
timerAdvanced = true;
}, 0);
await waitForCondition(() => deleteRunnerCalled);
assert.equal(deleteRunnerCalled, true, 'delete should be dispatched to the maintenance runner');
assert.equal(
bufferedWritesAtDeleteStart,
0,
'writes buffered before delete should flush first',
);
await waitForCondition(() => timerAdvanced);
createdTracker.recordSubtitleLine('queued during delete', 0, 1);
privateApi.flushNow();
assert.ok(privateApi.queue.length > 0, 'tracking writes should wait for delete maintenance');
assert.ok(deleteGate.release);
deleteGate.release();
await deletePromise;
} finally {
deleteGate.release?.();
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('delete maintenance flushes the entire write queue before locking writes', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
const deleteGate: { release?: () => void } = {};
let queuedWritesAtDeleteStart = -1;
let writeLockedAtDeleteStart = false;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath },
{
runDeleteMaintenanceTask: async () => {
const privateApi = tracker as unknown as {
queue: unknown[];
writeLock: { locked: boolean };
};
queuedWritesAtDeleteStart = privateApi.queue.length;
writeLockedAtDeleteStart = privateApi.writeLock.locked;
await new Promise<void>((resolve) => {
deleteGate.release = resolve;
});
},
},
);
const privateApi = tracker as unknown as {
batchSize: number;
flushNow: () => void;
queue: unknown[];
};
privateApi.batchSize = 1;
privateApi.queue.push({}, {}, {});
privateApi.flushNow = () => {
privateApi.queue.shift();
};
const deletePromise = tracker.deleteSession(101);
await waitForCondition(() => deleteGate.release !== undefined);
assert.equal(queuedWritesAtDeleteStart, 0);
assert.equal(writeLockedAtDeleteStart, true);
deleteGate.release?.();
await deletePromise;
} finally {
deleteGate.release?.();
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('delete maintenance tasks stay serialized under concurrent requests', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
const releases: Array<() => void> = [];
let activeTasks = 0;
let maxActiveTasks = 0;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath },
{
runDeleteMaintenanceTask: async () => {
activeTasks += 1;
maxActiveTasks = Math.max(maxActiveTasks, activeTasks);
await new Promise<void>((resolve) => {
releases.push(resolve);
});
activeTasks -= 1;
},
},
);
const firstDelete = tracker.deleteSession(101);
await waitForCondition(() => releases.length === 1);
assert.equal(maxActiveTasks, 1);
const secondDelete = tracker.deleteSession(102);
releases[0]?.();
await waitForCondition(() => releases.length === 2);
assert.equal(maxActiveTasks, 1);
releases[1]?.();
await Promise.all([firstDelete, secondDelete]);
} finally {
for (const release of releases) release();
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('concurrent delete requests share one maintenance worker batch', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
const tasks: unknown[] = [];
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath },
{
runDeleteMaintenanceTask: async (_path, task) => {
tasks.push(task);
},
},
);
const firstDelete = tracker.deleteSession(201);
const secondDelete = tracker.deleteSessions([202, 203]);
const thirdDelete = tracker.deleteVideo(204);
await Promise.all([firstDelete, secondDelete, thirdDelete]);
assert.equal(tasks.length, 1, 'concurrent deletes should use one maintenance pass');
assert.deepEqual(tasks[0], {
kind: 'batch',
tasks: [
{ kind: 'session', sessionId: 201 },
{ kind: 'sessions', sessionIds: [202, 203] },
{ kind: 'video', videoId: 204 },
],
});
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('destroy rejects delete requests waiting behind active maintenance', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let releaseFirstTask: () => void = () => {};
try {
const Ctor = await loadTrackerCtor();
let markFirstTaskStarted: () => void = () => {};
const firstTaskStarted = new Promise<void>((resolve) => {
markFirstTaskStarted = resolve;
});
tracker = new Ctor(
{ dbPath },
{
runDeleteMaintenanceTask: async () => {
markFirstTaskStarted();
await new Promise<void>((resolve) => {
releaseFirstTask = resolve;
});
},
},
);
const firstDelete = tracker.deleteSession(301);
await firstTaskStarted;
const queuedDelete = tracker.deleteSession(302);
tracker.destroy();
const queuedOutcome = await Promise.race([
queuedDelete.then(
() => 'resolved',
(error: unknown) =>
error instanceof Error && /shutting down/.test(error.message)
? 'rejected'
: 'wrong-error',
),
new Promise<'pending'>((resolve) => setTimeout(() => resolve('pending'), 25)),
]);
assert.equal(queuedOutcome, 'rejected');
releaseFirstTask();
await firstDelete;
} finally {
releaseFirstTask();
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('delete requested after destroy rejects without running maintenance', async () => {
const dbPath = makeDbPath();
let maintenanceCalls = 0;
const Ctor = await loadTrackerCtor();
const tracker = new Ctor(
{ dbPath },
{
runDeleteMaintenanceTask: async () => {
maintenanceCalls += 1;
},
},
);
tracker.destroy();
await assert.rejects(tracker.deleteSession(303), /shutting down/);
assert.equal(maintenanceCalls, 0);
cleanupDbPath(dbPath);
});
test('deleteSessions skips maintenance when no sessions are deletable', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
const tasks: unknown[] = [];
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath },
{
runDeleteMaintenanceTask: async (_path, task) => {
tasks.push(task);
},
},
);
await tracker.deleteSessions([]);
assert.deepEqual(tasks, []);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('queued video delete is skipped when that video becomes active before dispatch', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
const tasks: Array<{ kind: string }> = [];
let releaseFirstTask: () => void = () => {};
try {
const Ctor = await loadTrackerCtor();
const createdTracker = new Ctor(
{ dbPath },
{
runDeleteMaintenanceTask: async (_path, task) => {
tasks.push(task);
if (tasks.length === 1) {
await new Promise<void>((resolve) => {
releaseFirstTask = resolve;
});
}
},
},
);
tracker = createdTracker;
createdTracker.handleMediaChange('/tmp/delete-race-target.mkv', 'Delete Race Target');
createdTracker.handleMediaChange('/tmp/delete-race-other.mkv', 'Delete Race Other');
const privateApi = createdTracker as unknown as { db: DatabaseSync };
const targetVideoId = (
privateApi.db
.prepare(`SELECT video_id AS videoId FROM imm_videos WHERE video_key LIKE '%target.mkv'`)
.get() as { videoId: number } | null
)?.videoId;
assert.ok(targetVideoId);
const firstDelete = createdTracker.deleteSession(999_001);
await waitForCondition(() => tasks.length === 1);
const queuedVideoDelete = createdTracker.deleteVideo(targetVideoId);
createdTracker.handleMediaChange('/tmp/delete-race-target.mkv', 'Delete Race Target');
releaseFirstTask();
await Promise.all([firstDelete, queuedVideoDelete]);
assert.deepEqual(
tasks.map((task) => task.kind),
['session'],
);
} finally {
releaseFirstTask();
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('deleteVideo ignores the currently active video and keeps new writes flushable', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
+90 -40
View File
@@ -83,14 +83,15 @@ import {
} from './immersion-tracker/query-library';
import {
cleanupVocabularyStats,
deleteAnime as deleteAnimeQuery,
deleteSession as deleteSessionQuery,
deleteSessions as deleteSessionsQuery,
deleteVideo as deleteVideoQuery,
getVideoDurationMs,
markVideoWatched,
upsertCoverArt,
} from './immersion-tracker/query-maintenance';
import {
DeleteMaintenanceWorkerRuntime,
type RunDeleteMaintenanceTask,
} from './immersion-tracker/delete-maintenance-worker-runtime';
import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
import { repairJellyfinStreamVideoLinks } from './immersion-tracker/jellyfin-link-repair';
import {
repairLegacySeasonlessAnimeRows,
@@ -182,6 +183,7 @@ const YOUTUBE_SCREENSHOT_MAX_SECONDS = 120;
const YOUTUBE_OEMBED_ENDPOINT = 'https://www.youtube.com/oembed';
const YOUTUBE_ID_PATTERN = /^[A-Za-z0-9_-]{6,}$/;
const YOUTUBE_METADATA_REFRESH_MS = 24 * 60 * 60 * 1000;
const DELETE_MAINTENANCE_BATCH_WINDOW_MS = 10;
function isValidYouTubeVideoId(value: string | null): boolean {
return Boolean(value && YOUTUBE_ID_PATTERN.test(value));
@@ -385,6 +387,8 @@ export class ImmersionTrackerService {
private readonly vacuumIntervalMs: number;
private readonly dbPath: string;
private readonly writeLock = { locked: false };
private readonly destroyDeleteMaintenanceRunner: () => void;
private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
private flushTimer: ReturnType<typeof setTimeout> | null = null;
private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
private flushScheduled = false;
@@ -406,9 +410,38 @@ export class ImmersionTrackerService {
| ((row: LegacyVocabularyPosRow) => Promise<LegacyVocabularyPosResolution | null>)
| undefined;
constructor(options: ImmersionTrackerOptions) {
constructor(
options: ImmersionTrackerOptions,
dependencies: {
runDeleteMaintenanceTask?: RunDeleteMaintenanceTask;
destroyDeleteMaintenanceRunner?: () => void;
} = {},
) {
this.dbPath = options.dbPath;
this.resolveLegacyVocabularyPos = options.resolveLegacyVocabularyPos;
let runDeleteMaintenanceTask: RunDeleteMaintenanceTask;
if (dependencies.runDeleteMaintenanceTask) {
runDeleteMaintenanceTask = dependencies.runDeleteMaintenanceTask;
this.destroyDeleteMaintenanceRunner =
dependencies.destroyDeleteMaintenanceRunner ?? (() => {});
} else {
const deleteMaintenanceRuntime = new DeleteMaintenanceWorkerRuntime();
runDeleteMaintenanceTask = (dbPath, task) => deleteMaintenanceRuntime.run(dbPath, task);
this.destroyDeleteMaintenanceRunner = () => deleteMaintenanceRuntime.destroy();
}
this.deleteMaintenanceScheduler = new DeleteMaintenanceScheduler({
batchWindowMs: DELETE_MAINTENANCE_BATCH_WINDOW_MS,
runTask: (task) => runDeleteMaintenanceTask(this.dbPath, task),
onBusy: () => {
this.flushTelemetry(true);
while (this.queue.length > 0) this.flushNow();
this.writeLock.locked = true;
},
onIdle: () => {
this.writeLock.locked = false;
if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
},
});
const parentDir = path.dirname(this.dbPath);
if (!fs.existsSync(parentDir)) {
fs.mkdirSync(parentDir, { recursive: true });
@@ -512,6 +545,8 @@ export class ImmersionTrackerService {
}
this.finalizeActiveSession();
this.isDestroyed = true;
this.deleteMaintenanceScheduler.destroy();
this.destroyDeleteMaintenanceRunner();
this.db.close();
}
@@ -709,51 +744,66 @@ export class ImmersionTrackerService {
this.logger.warn(`Ignoring delete request for active immersion session ${sessionId}`);
return;
}
deleteSessionQuery(this.db, sessionId);
await this.enqueueDeleteMaintenanceTask(() => ({ kind: 'session', sessionId }));
}
async deleteSessions(sessionIds: number[]): Promise<void> {
const activeSessionId = this.sessionState?.sessionId;
const deletableSessionIds =
activeSessionId === undefined
? sessionIds
: sessionIds.filter((sessionId) => sessionId !== activeSessionId);
if (deletableSessionIds.length !== sessionIds.length) {
this.logger.warn(
`Ignoring bulk delete request for active immersion session ${activeSessionId}`,
);
}
deleteSessionsQuery(this.db, deletableSessionIds);
await this.enqueueDeleteMaintenanceTask(() => {
const activeSessionId = this.sessionState?.sessionId;
const deletableSessionIds =
activeSessionId === undefined
? sessionIds
: sessionIds.filter((sessionId) => sessionId !== activeSessionId);
if (deletableSessionIds.length !== sessionIds.length) {
this.logger.warn(
`Ignoring bulk delete request for active immersion session ${activeSessionId}`,
);
}
if (deletableSessionIds.length === 0) return null;
return { kind: 'sessions', sessionIds: deletableSessionIds };
});
}
async deleteVideo(videoId: number): Promise<void> {
if (this.sessionState?.videoId === videoId) {
this.logger.warn(`Ignoring delete request for active immersion video ${videoId}`);
return;
}
deleteVideoQuery(this.db, videoId);
await this.enqueueDeleteMaintenanceTask(() => {
if (this.sessionState?.videoId === videoId) {
this.logger.warn(`Ignoring delete request for active immersion video ${videoId}`);
return null;
}
return { kind: 'video', videoId };
});
}
async deleteAnime(animeId: number): Promise<void> {
// The active video's anime link is assigned asynchronously after the title
// is parsed, so a guard reading imm_videos too early sees a null and lets
// the delete through — then the late update recreates the anime row.
const pendingVideoId = this.sessionState?.videoId;
if (pendingVideoId !== undefined) {
await this.pendingAnimeMetadataUpdates.get(pendingVideoId);
}
const activeVideoId = this.sessionState?.videoId;
if (activeVideoId !== undefined) {
const activeAnime = this.db
.prepare('SELECT anime_id FROM imm_videos WHERE video_id = ?')
.get(activeVideoId) as { anime_id: number | null } | null;
if (activeAnime?.anime_id === animeId) {
this.logger.warn(`Ignoring delete request for active immersion anime ${animeId}`);
return;
await this.enqueueDeleteMaintenanceTask(async () => {
// Resolve this at dispatch time because another queued delete can leave
// enough time for playback to switch to an episode of this anime.
const pendingVideoId = this.sessionState?.videoId;
if (pendingVideoId !== undefined) {
await this.pendingAnimeMetadataUpdates.get(pendingVideoId);
}
const activeVideoId = this.sessionState?.videoId;
if (activeVideoId !== undefined) {
const activeAnime = this.db
.prepare('SELECT anime_id FROM imm_videos WHERE video_id = ?')
.get(activeVideoId) as { anime_id: number | null } | null;
if (activeAnime?.anime_id === animeId) {
this.logger.warn(`Ignoring delete request for active immersion anime ${animeId}`);
return null;
}
}
return { kind: 'anime', animeId };
});
}
private enqueueDeleteMaintenanceTask(
resolveTask: Parameters<DeleteMaintenanceScheduler['enqueue']>[0],
): Promise<void> {
if (this.isDestroyed) {
return Promise.reject(new Error('Immersion tracker is shutting down'));
}
deleteAnimeQuery(this.db, animeId);
return this.deleteMaintenanceScheduler.enqueue(resolveTask);
}
async reassignAnimeAnilist(
@@ -1811,7 +1861,7 @@ export class ImmersionTrackerService {
}
private runMaintenance(): void {
if (this.isDestroyed) return;
if (this.isDestroyed || this.writeLock.locked) return;
try {
this.flushTelemetry(true);
this.flushNow();
@@ -50,6 +50,7 @@ import {
updateAnimeAnilistInfo,
upsertCoverArt,
} from '../query-maintenance.js';
import { deleteMaintenanceBatch } from '../query-delete-maintenance.js';
import { getLocalEpochDay } from '../query-shared.js';
import { EVENT_CARD_MINED, EVENT_SUBTITLE_LINE, SOURCE_TYPE_LOCAL } from '../types.js';
@@ -985,3 +986,197 @@ test('split maintenance helpers delete multiple sessions and whole videos with d
cleanupDbPath(dbPath);
}
});
test('delete maintenance batch preserves retained data across overlapping session, video, and anime targets', () => {
const { db, dbPath, stmts } = createDb();
try {
const retainedAnimeId = getOrCreateAnimeRecord(db, {
parsedTitle: 'Retained Anime',
canonicalTitle: 'Retained Anime',
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
const deletedAnimeId = getOrCreateAnimeRecord(db, {
parsedTitle: 'Deleted Anime',
canonicalTitle: 'Deleted Anime',
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
const retainedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/batch-retain.mkv', {
canonicalTitle: 'Batch Retain',
sourcePath: '/tmp/batch-retain.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const deletedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/batch-video.mkv', {
canonicalTitle: 'Batch Video',
sourcePath: '/tmp/batch-video.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const animeVideoId = getOrCreateVideoRecord(db, 'local:/tmp/batch-anime.mkv', {
canonicalTitle: 'Batch Anime',
sourcePath: '/tmp/batch-anime.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
for (const [videoId, animeId, episode] of [
[retainedVideoId, retainedAnimeId, 1],
[deletedVideoId, retainedAnimeId, 2],
[animeVideoId, deletedAnimeId, 1],
] as const) {
linkVideoToAnimeRecord(db, videoId, {
animeId,
parsedBasename: `batch-${episode}.mkv`,
parsedTitle: animeId === retainedAnimeId ? 'Retained Anime' : 'Deleted Anime',
parsedSeason: 1,
parsedEpisode: episode,
parserSource: 'test',
parserConfidence: 1,
parseMetadataJson: null,
});
}
const startedAtMs = 1_700_000_000_000;
const deletedSessionId = startSessionRecord(db, retainedVideoId, startedAtMs).sessionId;
const retainedSessionId = startSessionRecord(
db,
retainedVideoId,
startedAtMs + 1_000,
).sessionId;
const videoSessionId = startSessionRecord(db, deletedVideoId, startedAtMs + 2_000).sessionId;
const animeSessionId = startSessionRecord(db, animeVideoId, startedAtMs + 3_000).sessionId;
for (const [sessionId, sessionStartedAtMs] of [
[deletedSessionId, startedAtMs],
[retainedSessionId, startedAtMs + 1_000],
[videoSessionId, startedAtMs + 2_000],
[animeSessionId, startedAtMs + 3_000],
] as const) {
finalizeSessionMetrics(db, sessionId, sessionStartedAtMs);
}
for (const [index, sessionId, videoId, animeId] of [
[1, deletedSessionId, retainedVideoId, retainedAnimeId],
[2, retainedSessionId, retainedVideoId, retainedAnimeId],
[3, videoSessionId, deletedVideoId, retainedAnimeId],
[4, animeSessionId, animeVideoId, deletedAnimeId],
] as const) {
insertWordOccurrence(db, stmts, {
sessionId,
videoId,
animeId,
lineIndex: index,
text: '猫日',
word: { headword: '猫', word: '猫', reading: 'ねこ' },
});
insertKanjiOccurrence(db, stmts, {
sessionId,
videoId,
animeId,
lineIndex: index + 10,
text: '猫日',
kanji: '日',
});
}
const rollupDay = getLocalEpochDay(db, startedAtMs);
const rollupMonth = (
db
.prepare(
`SELECT CAST(strftime('%Y%m', CAST(? AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) AS rollupMonth`,
)
.get(startedAtMs) as { rollupMonth: number }
).rollupMonth;
for (const videoId of [retainedVideoId, deletedVideoId, animeVideoId]) {
db.prepare(
`INSERT INTO imm_daily_rollups (
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (?, ?, 99, 99, 99, 99, 99, ?, ?)`,
).run(rollupDay, videoId, startedAtMs, startedAtMs);
db.prepare(
`INSERT INTO imm_monthly_rollups (
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (?, ?, 99, 99, 99, 99, 99, ?, ?)`,
).run(rollupMonth, videoId, startedAtMs, startedAtMs);
}
deleteMaintenanceBatch(db, [
{ kind: 'session', sessionId: deletedSessionId },
{ kind: 'session', sessionId: videoSessionId },
{ kind: 'video', videoId: deletedVideoId },
{ kind: 'video', videoId: animeVideoId },
{ kind: 'anime', animeId: deletedAnimeId },
]);
assert.deepEqual(db.prepare('SELECT session_id FROM imm_sessions').all(), [
{ session_id: retainedSessionId },
]);
assert.deepEqual(db.prepare('SELECT video_id FROM imm_videos').all(), [
{ video_id: retainedVideoId },
]);
assert.deepEqual(db.prepare('SELECT anime_id FROM imm_anime').all(), [
{ anime_id: retainedAnimeId },
]);
assert.equal(
(
db.prepare(`SELECT frequency FROM imm_words WHERE headword = '猫'`).get() as {
frequency: number;
}
).frequency,
1,
);
assert.equal(
(
db.prepare(`SELECT frequency FROM imm_kanji WHERE kanji = '日'`).get() as {
frequency: number;
}
).frequency,
1,
);
assert.deepEqual(
db.prepare('SELECT video_id, total_sessions FROM imm_daily_rollups').all() as Array<{
video_id: number;
total_sessions: number;
}>,
[{ video_id: retainedVideoId, total_sessions: 1 }],
);
assert.deepEqual(
db.prepare('SELECT video_id, total_sessions FROM imm_monthly_rollups').all() as Array<{
video_id: number;
total_sessions: number;
}>,
[{ video_id: retainedVideoId, total_sessions: 1 }],
);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('delete maintenance batch chunks id lists below the SQLite variable limit', () => {
const { db, dbPath } = createDb();
try {
const ids = Array.from({ length: 32_767 }, (_, index) => index + 1);
assert.doesNotThrow(() => {
deleteMaintenanceBatch(db, [
{ kind: 'sessions', sessionIds: ids },
...ids.map((videoId) => ({ kind: 'video' as const, videoId })),
...ids.map((animeId) => ({ kind: 'anime' as const, animeId })),
]);
});
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
@@ -0,0 +1,160 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { DeleteMaintenanceScheduler } from './delete-maintenance-scheduler';
import type { DeleteMaintenanceTask } from './delete-maintenance';
test('scheduler batches same-turn requests and balances busy state', async () => {
const tasks: DeleteMaintenanceTask[] = [];
const states: string[] = [];
const scheduler = new DeleteMaintenanceScheduler({
batchWindowMs: 0,
runTask: async (task) => {
tasks.push(task);
},
onBusy: () => states.push('busy'),
onIdle: () => states.push('idle'),
});
const first = scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 }));
const second = scheduler.enqueue(() => ({ kind: 'sessions', sessionIds: [2, 3] }));
const third = scheduler.enqueue(() => null);
await Promise.all([first, second, third]);
assert.deepEqual(tasks, [
{
kind: 'batch',
tasks: [
{ kind: 'session', sessionId: 1 },
{ kind: 'sessions', sessionIds: [2, 3] },
],
},
]);
assert.deepEqual(states, ['busy', 'idle']);
});
test('scheduler rejects enqueue after destruction without entering busy state', async () => {
let busyCalls = 0;
let runCalls = 0;
const scheduler = new DeleteMaintenanceScheduler({
batchWindowMs: 0,
runTask: async () => {
runCalls += 1;
},
onBusy: () => {
busyCalls += 1;
},
onIdle: () => {},
});
scheduler.destroy();
await assert.rejects(
scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 })),
/shutting down/,
);
assert.equal(busyCalls, 0);
assert.equal(runCalls, 0);
});
test('scheduler rejects every request in a batch when the maintenance task fails', async () => {
const failure = new Error('maintenance failed');
const scheduler = new DeleteMaintenanceScheduler({
batchWindowMs: 0,
runTask: async () => {
throw failure;
},
onBusy: () => {},
onIdle: () => {},
});
const first = scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 }));
const second = scheduler.enqueue(() => ({ kind: 'session', sessionId: 2 }));
const results = await Promise.allSettled([first, second]);
assert.deepEqual(
results.map((result) => (result.status === 'rejected' ? result.reason : null)),
[failure, failure],
);
});
test('scheduler rejects only the request whose task resolution fails', async () => {
const failure = new Error('resolution failed');
const tasks: DeleteMaintenanceTask[] = [];
const scheduler = new DeleteMaintenanceScheduler({
batchWindowMs: 0,
runTask: async (task) => {
tasks.push(task);
},
onBusy: () => {},
onIdle: () => {},
});
const failed = scheduler.enqueue(() => {
throw failure;
});
const succeeded = scheduler.enqueue(() => ({ kind: 'session', sessionId: 2 }));
const results = await Promise.allSettled([failed, succeeded]);
assert.equal(results[0]?.status, 'rejected');
assert.equal(results[0]?.status === 'rejected' ? results[0].reason : null, failure);
assert.equal(results[1]?.status, 'fulfilled');
assert.deepEqual(tasks, [{ kind: 'session', sessionId: 2 }]);
});
test('scheduler does not schedule another drain when the queue is empty', async () => {
const originalSetTimeout = globalThis.setTimeout;
let timerCalls = 0;
globalThis.setTimeout = ((handler: TimerHandler, timeout?: number, ...args: unknown[]) => {
timerCalls += 1;
return originalSetTimeout(handler, timeout, ...args);
}) as typeof setTimeout;
try {
const scheduler = new DeleteMaintenanceScheduler({
batchWindowMs: 0,
runTask: async () => {},
onBusy: () => {},
onIdle: () => {},
});
await scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 }));
assert.equal(timerCalls, 1);
} finally {
globalThis.setTimeout = originalSetTimeout;
}
});
test('scheduler serializes batches and rejects requests queued at destruction', async () => {
const releases: Array<() => void> = [];
let activeTasks = 0;
let maxActiveTasks = 0;
const scheduler = new DeleteMaintenanceScheduler({
batchWindowMs: 0,
runTask: async () => {
activeTasks += 1;
maxActiveTasks = Math.max(maxActiveTasks, activeTasks);
await new Promise<void>((resolve) => releases.push(resolve));
activeTasks -= 1;
},
onBusy: () => {},
onIdle: () => {},
});
const first = scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 }));
const maxPollAttempts = 100;
let pollAttempts = 0;
while (releases.length === 0 && pollAttempts < maxPollAttempts) {
pollAttempts += 1;
await new Promise<void>((resolve) => setTimeout(resolve, 0));
}
assert.ok(
releases.length > 0,
`runTask did not produce a release after ${maxPollAttempts} polling attempts`,
);
const queued = scheduler.enqueue(() => ({ kind: 'session', sessionId: 2 }));
scheduler.destroy();
await assert.rejects(queued, /shutting down/);
releases[0]?.();
await first;
assert.equal(maxActiveTasks, 1);
});
@@ -0,0 +1,105 @@
import type { DeleteMaintenanceOperation, DeleteMaintenanceTask } from './delete-maintenance';
type ResolveDeleteMaintenanceOperation = () =>
| DeleteMaintenanceOperation
| null
| Promise<DeleteMaintenanceOperation | null>;
interface PendingDeleteMaintenanceRequest {
resolveTask: ResolveDeleteMaintenanceOperation;
resolve: () => void;
reject: (error: unknown) => void;
}
interface DeleteMaintenanceSchedulerOptions {
batchWindowMs: number;
runTask: (task: DeleteMaintenanceTask) => Promise<void>;
onBusy: () => void;
onIdle: () => void;
}
export class DeleteMaintenanceScheduler {
private readonly pendingRequests: PendingDeleteMaintenanceRequest[] = [];
private running = false;
private drainTimer: ReturnType<typeof setTimeout> | null = null;
private pendingTaskCount = 0;
private destroyed = false;
constructor(private readonly options: DeleteMaintenanceSchedulerOptions) {}
enqueue(resolveTask: ResolveDeleteMaintenanceOperation): Promise<void> {
if (this.destroyed) {
return Promise.reject(new Error('Immersion tracker is shutting down'));
}
if (this.pendingTaskCount === 0) this.options.onBusy();
this.pendingTaskCount += 1;
const result = new Promise<void>((resolve, reject) => {
this.pendingRequests.push({ resolveTask, resolve, reject });
this.scheduleDrain();
});
return result.finally(() => {
this.pendingTaskCount -= 1;
if (this.pendingTaskCount === 0) this.options.onIdle();
});
}
destroy(): void {
if (this.destroyed) return;
this.destroyed = true;
if (this.drainTimer) {
clearTimeout(this.drainTimer);
this.drainTimer = null;
}
const error = new Error('Immersion tracker is shutting down');
for (const request of this.pendingRequests.splice(0)) request.reject(error);
}
private scheduleDrain(): void {
if (this.destroyed || this.running || this.drainTimer || this.pendingRequests.length === 0) {
return;
}
this.drainTimer = setTimeout(() => {
this.drainTimer = null;
void this.drain();
}, this.options.batchWindowMs);
}
private async drain(): Promise<void> {
if (this.running || this.pendingRequests.length === 0) return;
this.running = true;
const requests = this.pendingRequests.splice(0);
const runnable: Array<{
request: PendingDeleteMaintenanceRequest;
task: DeleteMaintenanceOperation;
}> = [];
for (const request of requests) {
try {
const task = await request.resolveTask();
if (task) runnable.push({ request, task });
else request.resolve();
} catch (error) {
request.reject(error);
}
}
if (runnable.length > 0) {
const task: DeleteMaintenanceTask =
runnable.length === 1
? runnable[0]!.task
: { kind: 'batch', tasks: runnable.map((entry) => entry.task) };
try {
await this.options.runTask(task);
for (const { request } of runnable) request.resolve();
} catch (error) {
for (const { request } of runnable) request.reject(error);
}
}
this.running = false;
this.scheduleDrain();
}
}
@@ -0,0 +1,239 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import {
DeleteMaintenanceWorkerRuntime,
resolveDeleteMaintenanceWorkerPath,
} from './delete-maintenance-worker-runtime';
import { executeDeleteMaintenanceTask } from './delete-maintenance';
import { startSessionRecord } from './session';
import { Database } from './sqlite';
import { applyPragmas, ensureSchema, getOrCreateVideoRecord } from './storage';
type FakeWorkerListener = (value: never) => void;
function createFakeWorker() {
const listeners = new Map<string, FakeWorkerListener>();
const terminationState = { calls: 0 };
const worker = {
once(event: string, listener: FakeWorkerListener) {
listeners.set(event, listener);
return this;
},
terminate: async () => {
terminationState.calls += 1;
return 0;
},
};
return { worker, listeners, terminationState };
}
type FakeWorker = ReturnType<typeof createFakeWorker>['worker'];
test('a delete batch rebuilds lifetime summaries once', () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-delete-batch-test-'));
const dbPath = path.join(tempDir, 'immersion.sqlite');
let db = new Database(dbPath);
try {
applyPragmas(db);
ensureSchema(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/batch-delete.mkv', {
canonicalTitle: 'Batch Delete',
sourcePath: '/tmp/batch-delete.mkv',
sourceUrl: null,
sourceType: 1,
});
const firstSessionId = startSessionRecord(db, videoId, 1_000).sessionId;
const secondSessionId = startSessionRecord(db, videoId, 2_000).sessionId;
const deletedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/batch-delete-video.mkv', {
canonicalTitle: 'Batch Delete Video',
sourcePath: '/tmp/batch-delete-video.mkv',
sourceUrl: null,
sourceType: 1,
});
startSessionRecord(db, deletedVideoId, 3_000);
db.exec(`
CREATE TABLE delete_rebuild_audit (id INTEGER PRIMARY KEY);
CREATE TRIGGER count_delete_lifetime_rebuild
AFTER UPDATE OF last_rebuilt_ms ON imm_lifetime_global
BEGIN
INSERT INTO delete_rebuild_audit (id) VALUES (NULL);
END;
`);
db.close();
executeDeleteMaintenanceTask(dbPath, {
kind: 'batch',
tasks: [
{ kind: 'session', sessionId: firstSessionId },
{ kind: 'video', videoId: deletedVideoId },
],
});
db = new Database(dbPath);
const audit = db.prepare('SELECT COUNT(*) AS total FROM delete_rebuild_audit').get() as {
total: number;
};
const retainedSession = db
.prepare('SELECT session_id AS sessionId FROM imm_sessions WHERE video_id = ?')
.get(videoId) as { sessionId: number } | null;
const deletedVideo = db
.prepare('SELECT video_id AS videoId FROM imm_videos WHERE video_id = ?')
.get(deletedVideoId) as { videoId: number } | null;
assert.equal(retainedSession?.sessionId, secondSessionId);
assert.equal(deletedVideo, undefined);
assert.equal(
audit.total,
2,
'one rebuild performs exactly its reset and final global summary writes',
);
} finally {
try {
db.close();
} catch {
// The setup connection closes before maintenance runs.
}
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
test(
'compiled delete worker removes data through its separate database connection',
{ skip: resolveDeleteMaintenanceWorkerPath() === null },
async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-delete-worker-test-'));
const dbPath = path.join(tempDir, 'immersion.sqlite');
const runtime = new DeleteMaintenanceWorkerRuntime();
let db = new Database(dbPath);
try {
applyPragmas(db);
ensureSchema(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/worker-delete.mkv', {
canonicalTitle: 'Worker Delete',
sourcePath: '/tmp/worker-delete.mkv',
sourceUrl: null,
sourceType: 1,
});
const firstSessionId = startSessionRecord(db, videoId, 1_000).sessionId;
const secondSessionId = startSessionRecord(db, videoId, 2_000).sessionId;
db.close();
await runtime.run(dbPath, {
kind: 'batch',
tasks: [
{ kind: 'session', sessionId: firstSessionId },
{ kind: 'session', sessionId: secondSessionId },
],
});
db = new Database(dbPath);
const row = db
.prepare('SELECT COUNT(*) AS total FROM imm_sessions WHERE video_id = ?')
.get(videoId) as { total: number };
assert.equal(row.total, 0);
} finally {
runtime.destroy();
try {
db.close();
} catch {
// The setup connection is already closed before the worker starts.
}
fs.rmSync(tempDir, { recursive: true, force: true });
}
},
);
test('worker runtime warns before falling back when no emitted worker is available', async () => {
const warnings: unknown[][] = [];
const fallbackTasks: unknown[] = [];
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => null,
warn: (...args) => warnings.push(args),
executeFallback: (_dbPath, task) => fallbackTasks.push(task),
});
await runtime.run('/tmp/fallback.sqlite', { kind: 'session', sessionId: 1 });
assert.equal(warnings.length, 1);
assert.match(String(warnings[0]?.[0]), /worker unavailable/i);
assert.deepEqual(fallbackTasks, [{ kind: 'session', sessionId: 1 }]);
});
test('worker runtime terminates a worker after successful settlement', async () => {
const { worker, listeners, terminationState } = createFakeWorker();
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => '/tmp/delete-worker.js',
createWorker: async () => worker,
});
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
await new Promise<void>((resolve) => setTimeout(resolve, 0));
listeners.get('message')?.({ ok: true } as never);
await result;
assert.equal(terminationState.calls, 1);
});
test('worker runtime terminates a worker after failed settlement', async () => {
const { worker, listeners, terminationState } = createFakeWorker();
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => '/tmp/delete-worker.js',
createWorker: async () => worker,
});
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
await new Promise<void>((resolve) => setTimeout(resolve, 0));
listeners.get('error')?.(new Error('worker failed') as never);
await assert.rejects(result, /worker failed/);
assert.equal(terminationState.calls, 1);
});
test('worker runtime terminates a worker created after shutdown begins', async () => {
const { worker, listeners, terminationState } = createFakeWorker();
const createGate: { resolve?: (worker: FakeWorker) => void } = {};
const fallbackTasks: unknown[] = [];
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => '/tmp/delete-worker.js',
createWorker: () =>
new Promise((resolve) => {
createGate.resolve = resolve;
}),
executeFallback: (_dbPath, task) => fallbackTasks.push(task),
});
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
await new Promise<void>((resolve) => setTimeout(resolve, 0));
runtime.destroy();
createGate.resolve?.(worker);
await assert.rejects(result, /shut down/);
assert.equal(terminationState.calls, 1);
assert.equal(listeners.size, 0);
assert.deepEqual(fallbackTasks, []);
});
test('worker runtime does not fall back when worker creation fails during shutdown', async () => {
const createGate: { reject?: (error: Error) => void } = {};
const fallbackTasks: unknown[] = [];
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => '/tmp/delete-worker.js',
createWorker: () =>
new Promise((_resolve, reject) => {
createGate.reject = reject;
}),
executeFallback: (_dbPath, task) => fallbackTasks.push(task),
});
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
await new Promise<void>((resolve) => setTimeout(resolve, 0));
runtime.destroy();
createGate.reject?.(new Error('creation failed'));
await assert.rejects(result, /shut down/);
assert.deepEqual(fallbackTasks, []);
});
@@ -0,0 +1,121 @@
import fs from 'node:fs';
import path from 'node:path';
import { createLogger } from '../../../logger';
import { executeDeleteMaintenanceTask, type DeleteMaintenanceTask } from './delete-maintenance';
interface DeleteMaintenanceWorkerResponse {
ok?: unknown;
error?: unknown;
}
export type RunDeleteMaintenanceTask = (
dbPath: string,
task: DeleteMaintenanceTask,
) => Promise<void>;
interface DeleteMaintenanceWorkerHandle {
once(event: 'message', listener: (message: DeleteMaintenanceWorkerResponse) => void): this;
once(event: 'error', listener: (error: Error) => void): this;
once(event: 'exit', listener: (code: number) => void): this;
terminate(): Promise<number>;
}
interface DeleteMaintenanceWorkerRuntimeOptions {
resolveWorkerPath?: () => string | null;
createWorker?: (
workerPath: string,
workerData: { dbPath: string; task: DeleteMaintenanceTask },
) => Promise<DeleteMaintenanceWorkerHandle>;
executeFallback?: typeof executeDeleteMaintenanceTask;
warn?: (message: string, ...meta: unknown[]) => void;
}
export function resolveDeleteMaintenanceWorkerPath(): string | null {
const workerPath = path.join(__dirname, 'delete-maintenance-worker-thread.js');
return fs.existsSync(workerPath) ? workerPath : null;
}
const logger = createLogger('main:immersion-tracker:delete-worker');
export class DeleteMaintenanceWorkerRuntime {
private readonly activeWorkers = new Set<DeleteMaintenanceWorkerHandle>();
private destroyed = false;
constructor(private readonly options: DeleteMaintenanceWorkerRuntimeOptions = {}) {}
async run(dbPath: string, task: DeleteMaintenanceTask): Promise<void> {
if (this.destroyed) {
throw new Error('Delete maintenance worker is shut down');
}
let worker: DeleteMaintenanceWorkerHandle;
try {
const workerPath = (this.options.resolveWorkerPath ?? resolveDeleteMaintenanceWorkerPath)();
if (!workerPath) throw new Error('Emitted delete-maintenance worker module was not found');
const createWorker =
this.options.createWorker ??
(async (resolvedPath, workerData) => {
const { Worker } = await import('node:worker_threads');
return new Worker(resolvedPath, { workerData });
});
worker = await createWorker(workerPath, { dbPath, task });
} catch (error) {
if (this.destroyed) {
throw new Error('Delete maintenance worker is shut down');
}
(this.options.warn ?? logger.warn)(
'Delete maintenance worker unavailable; running maintenance on the current thread',
error,
);
(this.options.executeFallback ?? executeDeleteMaintenanceTask)(dbPath, task);
return;
}
if (this.destroyed) {
await worker.terminate().catch(() => undefined);
throw new Error('Delete maintenance worker is shut down');
}
await new Promise<void>((resolve, reject) => {
let settled = false;
this.activeWorkers.add(worker);
const settle = (error?: Error) => {
if (settled) return;
settled = true;
this.activeWorkers.delete(worker);
if (error) reject(error);
else resolve();
void worker.terminate();
};
worker.once('message', (message: DeleteMaintenanceWorkerResponse) => {
if (message.ok === true) {
settle();
return;
}
const detail = typeof message.error === 'string' ? message.error : 'unknown worker error';
settle(new Error(`Delete maintenance failed: ${detail}`));
});
worker.once('error', (error) => settle(error));
worker.once('exit', (code) => {
settle(
new Error(
code === 0
? 'Delete maintenance worker exited without a response'
: `Delete maintenance worker exited with code ${code}`,
),
);
});
});
}
destroy(): void {
if (this.destroyed) return;
this.destroyed = true;
for (const worker of this.activeWorkers) {
void worker.terminate();
}
this.activeWorkers.clear();
}
}
@@ -0,0 +1,22 @@
import { parentPort, workerData } from 'node:worker_threads';
import { executeDeleteMaintenanceTask, type DeleteMaintenanceTask } from './delete-maintenance';
interface DeleteMaintenanceWorkerData {
dbPath: string;
task: DeleteMaintenanceTask;
}
if (!parentPort) {
throw new Error('delete maintenance worker missing parent port');
}
const port = parentPort;
const request = workerData as DeleteMaintenanceWorkerData;
try {
executeDeleteMaintenanceTask(request.dbPath, request.task);
port.postMessage({ ok: true });
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
port.postMessage({ ok: false, error: message });
}
@@ -0,0 +1,47 @@
import { Database } from './sqlite';
import { applyPragmas } from './storage';
import { deleteAnime, deleteSession, deleteSessions, deleteVideo } from './query-maintenance';
import {
deleteMaintenanceBatch,
type DeleteMaintenanceOperation,
} from './query-delete-maintenance';
export type { DeleteMaintenanceOperation } from './query-delete-maintenance';
export type DeleteMaintenanceTask =
| DeleteMaintenanceOperation
| { kind: 'batch'; tasks: DeleteMaintenanceOperation[] };
function executeDeleteMaintenanceOperation(
db: InstanceType<typeof Database>,
task: DeleteMaintenanceOperation,
): void {
switch (task.kind) {
case 'session':
deleteSession(db, task.sessionId);
return;
case 'sessions':
deleteSessions(db, task.sessionIds);
return;
case 'video':
deleteVideo(db, task.videoId);
return;
case 'anime':
deleteAnime(db, task.animeId);
return;
}
}
export function executeDeleteMaintenanceTask(dbPath: string, task: DeleteMaintenanceTask): void {
const db = new Database(dbPath);
try {
applyPragmas(db);
if (task.kind === 'batch') {
deleteMaintenanceBatch(db, task.tasks);
return;
}
executeDeleteMaintenanceOperation(db, task);
} finally {
db.close();
}
}
@@ -0,0 +1,196 @@
import type { DatabaseSync } from './sqlite';
import { rebuildLifetimeSummariesInTransaction } from './lifetime';
import { getRollupGroupsForSessions, refreshRollupsForGroupsInTransaction } from './maintenance';
import {
applyLexicalRemovals,
cleanupUnusedCoverArtBlobHash,
deleteSessionsByIds,
forEachIdChunk,
makePlaceholders,
planLexicalRemovalsForSessions,
SQLITE_ID_CHUNK_SIZE,
type LexicalRemovalPlan,
} from './query-shared';
export type DeleteMaintenanceOperation =
| { kind: 'session'; sessionId: number }
| { kind: 'sessions'; sessionIds: number[] }
| { kind: 'video'; videoId: number }
| { kind: 'anime'; animeId: number };
function addOperationTargets(
operations: DeleteMaintenanceOperation[],
sessionIds: Set<number>,
videoIds: Set<number>,
animeIds: Set<number>,
): void {
for (const operation of operations) {
switch (operation.kind) {
case 'session':
sessionIds.add(operation.sessionId);
break;
case 'sessions':
for (const sessionId of operation.sessionIds) sessionIds.add(sessionId);
break;
case 'video':
videoIds.add(operation.videoId);
break;
case 'anime':
animeIds.add(operation.animeId);
break;
}
}
}
function selectIds(
db: DatabaseSync,
buildSql: (placeholders: string) => string,
params: number[],
column: string,
): number[] {
if (params.length === 0) return [];
const ids: number[] = [];
forEachIdChunk(params, (chunk) => {
const rows = db.prepare(buildSql(makePlaceholders(chunk))).all(...chunk) as Array<
Record<string, number>
>;
for (const row of rows) ids.push(row[column]!);
});
return ids;
}
function planLexicalRemovalsInChunks(db: DatabaseSync, sessionIds: number[]): LexicalRemovalPlan {
const combined: LexicalRemovalPlan = { words: [], kanji: [] };
const merge = (target: LexicalRemovalPlan['words'], source: LexicalRemovalPlan['words']) => {
const byId = new Map(target.map((entry) => [entry.id, entry]));
for (const entry of source) {
const existing = byId.get(entry.id);
if (!existing) {
const added = { ...entry };
target.push(added);
byId.set(entry.id, added);
continue;
}
existing.removedFrequency += entry.removedFrequency;
if (
entry.removedFirstSeenMs !== null &&
(existing.removedFirstSeenMs === null ||
entry.removedFirstSeenMs < existing.removedFirstSeenMs)
) {
existing.removedFirstSeenMs = entry.removedFirstSeenMs;
}
if (
entry.removedLastSeenMs !== null &&
(existing.removedLastSeenMs === null ||
entry.removedLastSeenMs > existing.removedLastSeenMs)
) {
existing.removedLastSeenMs = entry.removedLastSeenMs;
}
}
};
forEachIdChunk(sessionIds, (chunk) => {
const plan = planLexicalRemovalsForSessions(db, chunk);
merge(combined.words, plan.words);
merge(combined.kanji, plan.kanji);
});
return combined;
}
export function deleteMaintenanceBatch(
db: DatabaseSync,
operations: DeleteMaintenanceOperation[],
): void {
if (operations.length === 0) return;
db.exec('BEGIN IMMEDIATE');
try {
const sessionIds = new Set<number>();
const videoIds = new Set<number>();
const animeIds = new Set<number>();
addOperationTargets(operations, sessionIds, videoIds, animeIds);
const animeIdList = [...animeIds];
for (const videoId of selectIds(
db,
(placeholders) => `SELECT video_id FROM imm_videos WHERE anime_id IN (${placeholders})`,
animeIdList,
'video_id',
)) {
videoIds.add(videoId);
}
const videoIdList = [...videoIds];
for (const sessionId of selectIds(
db,
(placeholders) => `SELECT session_id FROM imm_sessions WHERE video_id IN (${placeholders})`,
videoIdList,
'session_id',
)) {
sessionIds.add(sessionId);
}
const sessionIdList = [...sessionIds];
const lexicalRemovals = planLexicalRemovalsInChunks(db, sessionIdList);
const affectedRollupGroups = sessionIdList
.flatMap((_, index) =>
index % SQLITE_ID_CHUNK_SIZE === 0
? getRollupGroupsForSessions(db, sessionIdList.slice(index, index + SQLITE_ID_CHUNK_SIZE))
: [],
)
.filter((group) => !videoIds.has(group.videoId));
const coverBlobHashes = new Set<string>();
if (videoIdList.length > 0) {
forEachIdChunk(videoIdList, (chunk) => {
const placeholders = makePlaceholders(chunk);
const artRows = db
.prepare(
`SELECT cover_blob_hash AS coverBlobHash
FROM imm_media_art
WHERE video_id IN (${placeholders}) AND cover_blob_hash IS NOT NULL`,
)
.all(...chunk) as Array<{ coverBlobHash: string }>;
for (const row of artRows) coverBlobHashes.add(row.coverBlobHash);
});
deleteSessionsByIds(db, sessionIdList);
forEachIdChunk(videoIdList, (chunk) => {
const placeholders = makePlaceholders(chunk);
db.prepare(`DELETE FROM imm_subtitle_lines WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_daily_rollups WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_monthly_rollups WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_media_art WHERE video_id IN (${placeholders})`).run(...chunk);
db.prepare(`DELETE FROM imm_videos WHERE video_id IN (${placeholders})`).run(...chunk);
});
} else {
deleteSessionsByIds(db, sessionIdList);
}
for (const coverBlobHash of coverBlobHashes) {
cleanupUnusedCoverArtBlobHash(db, coverBlobHash);
}
if (animeIdList.length > 0) {
forEachIdChunk(animeIdList, (chunk) => {
const placeholders = makePlaceholders(chunk);
db.prepare(`DELETE FROM imm_lifetime_anime WHERE anime_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_anime WHERE anime_id IN (${placeholders})`).run(...chunk);
});
}
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
}
@@ -80,6 +80,14 @@ export function makePlaceholders(values: number[]): string {
return values.map(() => '?').join(',');
}
export const SQLITE_ID_CHUNK_SIZE = 1_000;
export function forEachIdChunk(ids: number[], callback: (chunk: number[]) => void): void {
for (let start = 0; start < ids.length; start += SQLITE_ID_CHUNK_SIZE) {
callback(ids.slice(start, start + SQLITE_ID_CHUNK_SIZE));
}
}
export function resolvedCoverBlobExpr(mediaAlias: string, blobStoreAlias: string): string {
return `COALESCE(${blobStoreAlias}.cover_blob, CASE WHEN ${mediaAlias}.cover_blob_hash IS NULL THEN ${mediaAlias}.cover_blob ELSE NULL END)`;
}
@@ -490,17 +498,19 @@ export function deleteSessionsByIds(db: DatabaseSync, sessionIds: number[]): voi
return;
}
const placeholders = makePlaceholders(sessionIds);
db.prepare(`DELETE FROM imm_subtitle_lines WHERE session_id IN (${placeholders})`).run(
...sessionIds,
);
db.prepare(`DELETE FROM imm_session_telemetry WHERE session_id IN (${placeholders})`).run(
...sessionIds,
);
db.prepare(`DELETE FROM imm_session_events WHERE session_id IN (${placeholders})`).run(
...sessionIds,
);
db.prepare(`DELETE FROM imm_sessions WHERE session_id IN (${placeholders})`).run(...sessionIds);
forEachIdChunk(sessionIds, (chunk) => {
const placeholders = makePlaceholders(chunk);
db.prepare(`DELETE FROM imm_subtitle_lines WHERE session_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_session_telemetry WHERE session_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_session_events WHERE session_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_sessions WHERE session_id IN (${placeholders})`).run(...chunk);
});
}
export function toDbMs(ms: number | bigint): bigint {
+191
View File
@@ -0,0 +1,191 @@
/*
* Duplicate/animation-burst collapsing for parsed subtitle cues.
*
* Split out of the cue parser so the parsing rules and the "is this run one animation?"
* heuristics can be read -- and tested -- on their own. The parser owns the cue shape;
* this module only decides which cues survive.
*/
import { hasAssTemporalOverride, isAnimatedAssEffectKind } from './ass-text';
import type {
AnnotatedSubtitleCue,
SubtitleCue,
SubtitleSourceFormat,
} from './subtitle-cue-parser';
// Back-to-back frames of the same animation are authored flush against each other; a
// tiny tolerance absorbs the centisecond rounding of the ASS timestamp format.
const DUPLICATE_CUE_GAP_TOLERANCE_SECONDS = 0.05;
// A burst is a *sequence*. Two adjacent events are two events, not an animation --
// characters do repeat each other, and a repeated line can legitimately be short.
const MIN_BURST_EVENTS = 3;
// Real dialogue holds on screen for about a second, so a run with a couple of much
// shorter events among them looks like frames. Used only alongside authoring evidence.
const ANIMATION_FRAME_MAX_SECONDS = 0.3;
// A karaoke run usually ends on a long "hold" frame, so not every event is short.
const MIN_TAGGED_BURST_FRAMES = 2;
// SRT and VTT carry no authoring metadata at all, so timing is the only signal available
// -- which makes it the easiest one to get wrong. ASS->SRT conversion leaves frames at
// ~0.04s, well under any real utterance, and a burst leaves many of them behind. Both
// bounds are deliberately far stricter than the ASS path: a run of ordinary short lines
// (`えっ` traded between characters) must not clear them.
const TIMING_ONLY_FRAME_MAX_SECONDS = 0.1;
const MIN_TIMING_ONLY_FRAMES = 5;
function cueKey(cue: SubtitleCue): string {
return `${cue.startTime}|${cue.endTime}|${cue.text}`;
}
/**
* Identical text over an identical span is redundant however it was authored -- most
* often a layered ASS event stacking a shadow copy under the visible one.
*/
function collapseExactDuplicates(cues: AnnotatedSubtitleCue[]): AnnotatedSubtitleCue[] {
const seen = new Set<string>();
return cues.filter((cue) => {
const key = cueKey(cue);
if (seen.has(key)) {
return false;
}
seen.add(key);
return true;
});
}
function countFramesShorterThan(run: AnnotatedSubtitleCue[], maxSeconds: number): number {
return run.filter((cue) => cue.endTime - cue.startTime < maxSeconds).length;
}
/**
* Evidence that a run of ASS events is one animation rather than several authored lines.
* A static tag says nothing on its own -- three events sharing one `\clip(...)` are three
* signs -- so the tag has to be temporal by nature (`\t`, `\move`, karaoke timing, or
* anything wrapped in `\t(...)`), an animated `Effect` column, or a value that actually
* changes from event to event, which is how per-frame typesetting is authored.
*/
export function hasAssAnimationEvidence(run: AnnotatedSubtitleCue[]): boolean {
if (run.every((cue) => hasAssTemporalOverride(cue.overrides))) {
return true;
}
if (run.every((cue) => isAnimatedAssEffectKind(cue.effectKind))) {
return true;
}
const [first] = run;
const everyEventTypeset = run.every((cue) => cue.overrides.length > 0);
const signatureChanges = run.some((cue) => cue.overrideSignature !== first!.overrideSignature);
return everyEventTypeset && signatureChanges;
}
export function isAnimationBurst(
run: AnnotatedSubtitleCue[],
format: SubtitleSourceFormat,
): boolean {
if (run.length < MIN_BURST_EVENTS) {
return false;
}
if (format === 'srt') {
return (
run.length >= MIN_TIMING_ONLY_FRAMES &&
countFramesShorterThan(run, TIMING_ONLY_FRAME_MAX_SECONDS) === run.length
);
}
if (countFramesShorterThan(run, ANIMATION_FRAME_MAX_SECONDS) < MIN_TAGGED_BURST_FRAMES) {
return false;
}
// One animation belongs to one styled, one named source line. Two characters trading
// the same short word are two styles or two actors, and never merge.
const [first] = run;
if (run.some((cue) => cue.style !== first!.style || cue.name !== first!.name)) {
return false;
}
return hasAssAnimationEvidence(run);
}
/**
* Karaoke and sign typesetting emits one Dialogue event per animation frame, all carrying
* the same visible text over a contiguous span. Collapse each such run into a single cue.
*
* Only runs that look like animation collapse. Two ordinary lines that happen to repeat
* -- several characters each saying `おはよう` in turn, a positioned sign redrawn with a
* different fade -- stay separate, because merging them would destroy real mineable lines.
*/
function collapseAnimationBursts(
cues: AnnotatedSubtitleCue[],
format: SubtitleSourceFormat,
): AnnotatedSubtitleCue[] {
const indicesByText = new Map<string, number[]>();
cues.forEach((cue, index) => {
const bucket = indicesByText.get(cue.text);
if (bucket) {
bucket.push(index);
} else {
indicesByText.set(cue.text, [index]);
}
});
const dropped = new Set<number>();
const extendedEnd = new Map<number, number>();
for (const indices of indicesByText.values()) {
if (indices.length < MIN_BURST_EVENTS) {
continue;
}
let runStart = 0;
while (runStart < indices.length) {
let runEnd = runStart;
let chainEnd = cues[indices[runStart]!]!.endTime;
while (runEnd + 1 < indices.length) {
const next = cues[indices[runEnd + 1]!]!;
if (next.startTime > chainEnd + DUPLICATE_CUE_GAP_TOLERANCE_SECONDS) {
break;
}
chainEnd = Math.max(chainEnd, next.endTime);
runEnd += 1;
}
const run = indices.slice(runStart, runEnd + 1).map((index) => cues[index]!);
if (isAnimationBurst(run, format)) {
for (let i = runStart + 1; i <= runEnd; i += 1) {
dropped.add(indices[i]!);
}
extendedEnd.set(indices[runStart]!, chainEnd);
}
runStart = runEnd + 1;
}
}
if (dropped.size === 0) {
return cues;
}
const merged: AnnotatedSubtitleCue[] = [];
cues.forEach((cue, index) => {
if (dropped.has(index)) {
return;
}
const end = extendedEnd.get(index);
merged.push(end !== undefined && end > cue.endTime ? { ...cue, endTime: end } : cue);
});
return merged;
}
/**
* Collapse redundant cues. Input must already be sorted by non-decreasing `startTime`,
* ties broken by `endTime` then source `order` -- burst detection chains events by
* comparing each one against the running end of the events before it, so an unsorted
* list breaks runs apart and leaves the frames behind.
*/
export function mergeDuplicateCues(
cues: AnnotatedSubtitleCue[],
format: SubtitleSourceFormat,
): AnnotatedSubtitleCue[] {
return collapseAnimationBursts(collapseExactDuplicates(cues), format);
}
+393 -3
View File
@@ -91,6 +91,17 @@ test('parseSrtCues skips malformed timing lines gracefully', () => {
assert.equal(cues[0]!.text, '有効');
});
test('parseSubtitleCues strips complete brace blocks from SRT and VTT text', () => {
const content = ['1', '00:00:01,000 --> 00:00:02,000', '彼は{謎}と言った', ''].join('\n');
for (const filename of ['test.srt', 'test.vtt']) {
const cues = parseSubtitleCues(content, filename);
assert.equal(cues.length, 1, filename);
assert.equal(cues[0]!.text, '彼はと言った', filename);
}
});
test('parseAssCues parses basic ASS dialogue lines', () => {
const content = [
'[Script Info]',
@@ -137,7 +148,9 @@ test('parseAssCues handles text containing commas', () => {
assert.equal(cues[0]!.text, 'はい、そうです、ね');
});
test('parseAssCues handles \\N line breaks', () => {
test('parseAssCues decodes \\N line breaks into real newlines', () => {
// ASS is decoded once, here at ingestion, so cue text matches what mpv hands over for
// the same line played live.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
@@ -146,7 +159,7 @@ test('parseAssCues handles \\N line breaks', () => {
const cues = parseAssCues(content);
assert.equal(cues[0]!.text, '一行目\\N二行目');
assert.equal(cues[0]!.text, '一行目\n二行目');
});
test('parseAssCues strips HTML-like markup while preserving ASS line breaks', () => {
@@ -158,7 +171,46 @@ test('parseAssCues strips HTML-like markup while preserving ASS line breaks', ()
const cues = parseAssCues(content);
assert.equal(cues[0]!.text, '一行目\\N二行目');
assert.equal(cues[0]!.text, '一行目\n二行目');
});
test('parseAssCues drops vector drawing runs enabled by \\p', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 1,0:00:01.00,0:00:04.00,Default,,0,0,0,,{\\an5\\pos(730,1042)\\p1\\blur1}m 20 0 b 10 0 0 10 0 20 b 0 31 10 40 20 40 {\\p0}',
'Dialogue: 0,0:00:05.00,0:00:08.00,Default,,0,0,0,,これは字幕',
].join('\n');
const cues = parseAssCues(content);
assert.equal(cues.length, 1);
assert.equal(cues[0]!.text, 'これは字幕');
});
test('parseAssCues keeps text that follows a \\p0 reset on the same line', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:04.00,Default,,0,0,0,,{\\p1}m 0 0 l 10 10{\\p0}本文{\\p1}m 5 5 l 6 6{\\p0}続き',
].join('\n');
const cues = parseAssCues(content);
assert.equal(cues.length, 1);
assert.equal(cues[0]!.text, '本文続き');
});
test('parseAssCues leaves \\pos untouched when no drawing mode is active', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:04.00,Default,,0,0,0,,{\\pos(960,1068)\\bord3}位置指定',
].join('\n');
const cues = parseAssCues(content);
assert.equal(cues[0]!.text, '位置指定');
});
test('parseAssCues returns empty for content without Events section', () => {
@@ -258,6 +310,344 @@ test('parseSubtitleCues returns cues sorted by start time', () => {
assert.equal(cues[1]!.text, '二番目');
});
test('parseSubtitleCues collapses per-frame karaoke duplicates into one cue', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.05,OP_JP,,0,0,0,,{\\clip(m 1 1)}過ぎ去ってしまう瞬間を',
'Dialogue: 0,0:00:01.05,0:00:01.09,OP_JP,,0,0,0,,{\\clip(m 2 2)}過ぎ去ってしまう瞬間を',
'Dialogue: 0,0:00:01.09,0:00:03.55,OP_JP,,0,0,0,,{\\clip(m 3 3)}過ぎ去ってしまう瞬間を',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 1);
assert.equal(cues[0]!.startTime, 1.0);
assert.equal(cues[0]!.endTime, 3.55);
assert.equal(cues[0]!.text, '過ぎ去ってしまう瞬間を');
});
test('parseSubtitleCues keeps back-to-back plain dialogue repeats separate', () => {
// Several characters greeting in turn: distinct utterances that happen to abut.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:04:05.67,0:04:06.82,Dial_JP,,0,0,0,,おはよう',
'Dialogue: 0,0:04:06.82,0:04:07.56,Dial_JP,,0,0,0,,おはよう',
'Dialogue: 0,0:04:07.56,0:04:08.78,Dial_JP,,0,0,0,,おはよう',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
assert.equal(cues[0]!.endTime, 246.82);
assert.equal(cues[2]!.startTime, 247.56);
});
test('parseSubtitleCues collapses exact duplicate cues even without effect tags', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:04.00,Default,,0,0,0,,重なった行',
'Dialogue: 1,0:00:01.00,0:00:04.00,Default,,0,0,0,,重なった行',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 1);
});
test('parseSubtitleCues collapses tag-less animation frames in converted SRT', () => {
// ASS -> SRT conversion drops override tags, so only the ~0.04s frame timing remains.
const lines = ['1', '00:00:07,870 --> 00:00:07,910', 'Kaguya Wants to be Confessed to', ''];
for (let i = 1; i < 8; i++) {
const start = 7910 + (i - 1) * 40;
const end = start + 40;
const at = (ms: number) =>
`00:00:0${Math.floor(ms / 1000)},${String(ms % 1000).padStart(3, '0')}`;
lines.push(String(i + 1), `${at(start)} --> ${at(end)}`, 'Kaguya Wants to be Confessed to', '');
}
const cues = parseSubtitleCues(lines.join('\n'), 'test.srt');
assert.equal(cues.length, 1);
assert.equal(cues[0]!.startTime, 7.87);
});
test('parseSubtitleCues keeps identical lines that recur far apart', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:02.00,Default,,0,0,0,,なんで',
'Dialogue: 0,0:05:00.00,0:05:01.00,Default,,0,0,0,,なんで',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 2);
assert.equal(cues[0]!.startTime, 1.0);
assert.equal(cues[1]!.startTime, 300.0);
});
test('parseSubtitleCues keeps two positioned signs that repeat the same text', () => {
// Both carry override tags, but `\pos` and `\fad` are static placement, not animation.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:01:00.00,0:01:03.00,Sign,,0,0,0,,{\\pos(960,120)\\fad(200,200)}第一話',
'Dialogue: 0,0:01:03.00,0:01:06.00,Sign,,0,0,0,,{\\pos(960,900)\\fad(200,200)}第一話',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 2);
assert.equal(cues[1]!.startTime, 63.0);
});
test('parseSubtitleCues keeps a run of ordinary positioned lines separate', () => {
// Three events is a sequence, but none of them runs at animation-frame speed.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:01:00.00,0:01:02.00,Sign,,0,0,0,,{\\pos(960,120)\\fad(100,100)}止まれ',
'Dialogue: 0,0:01:02.00,0:01:04.00,Sign,,0,0,0,,{\\pos(960,120)\\fad(100,100)}止まれ',
'Dialogue: 0,0:01:04.00,0:01:06.00,Sign,,0,0,0,,{\\pos(960,120)\\fad(100,100)}止まれ',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
});
test('parseSubtitleCues keeps a short repeated SRT pair without burst evidence', () => {
const content = [
'1',
'00:00:01,000 --> 00:00:01,200',
'えっ',
'',
'2',
'00:00:01,200 --> 00:00:01,400',
'えっ',
'',
].join('\n');
const cues = parseSubtitleCues(content, 'test.srt');
assert.equal(cues.length, 2);
});
test('parseSubtitleCues collapses a burst marked only by the Effect column', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.05,OP_JP,,0,0,0,Karaoke,歌詞',
'Dialogue: 0,0:00:01.05,0:00:01.09,OP_JP,,0,0,0,Karaoke,歌詞',
'Dialogue: 0,0:00:01.09,0:00:03.55,OP_JP,,0,0,0,Karaoke,歌詞',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 1);
assert.equal(cues[0]!.endTime, 3.55);
});
test('parseSubtitleCues keeps a second karaoke burst that starts after a gap', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.05,OP_JP,,0,0,0,,{\\clip(m 1 1)}リフレイン',
'Dialogue: 0,0:00:01.05,0:00:01.09,OP_JP,,0,0,0,,{\\clip(m 2 2)}リフレイン',
'Dialogue: 0,0:00:01.09,0:00:03.00,OP_JP,,0,0,0,,{\\clip(m 3 3)}リフレイン',
'Dialogue: 0,0:00:20.00,0:00:20.05,OP_JP,,0,0,0,,{\\clip(m 1 1)}リフレイン',
'Dialogue: 0,0:00:20.05,0:00:20.09,OP_JP,,0,0,0,,{\\clip(m 2 2)}リフレイン',
'Dialogue: 0,0:00:20.09,0:00:22.00,OP_JP,,0,0,0,,{\\clip(m 3 3)}リフレイン',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 2);
assert.equal(cues[0]!.endTime, 3.0);
assert.equal(cues[1]!.startTime, 20.0);
assert.equal(cues[1]!.endTime, 22.0);
});
test('parseSubtitleCues does not merge a burst into unrelated dialogue between frames', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.05,OP_JP,,0,0,0,,{\\clip(m 1 1)}歌詞',
'Dialogue: 0,0:00:01.02,0:00:03.00,Dial_JP,,0,0,0,,別のセリフ',
'Dialogue: 0,0:00:01.05,0:00:01.09,OP_JP,,0,0,0,,{\\clip(m 2 2)}歌詞',
'Dialogue: 0,0:00:01.09,0:00:03.55,OP_JP,,0,0,0,,{\\clip(m 3 3)}歌詞',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 2);
assert.deepEqual(
cues.map((cue) => cue.text),
['歌詞', '別のセリフ'],
);
assert.equal(cues[0]!.endTime, 3.55);
});
test('parseSubtitleCues keeps rapid ASS lines from different actors separate', () => {
// Three 200ms `えっ` reactions traded between characters. Fast, adjacent and identical,
// but authored as three lines: different styles and different actors.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Dial_A,アリス,0,0,0,,えっ',
'Dialogue: 0,0:00:01.20,0:00:01.40,Dial_B,ボブ,0,0,0,,えっ',
'Dialogue: 0,0:00:01.40,0:00:01.60,Dial_C,キャロル,0,0,0,,えっ',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
});
test('parseSubtitleCues reads the speaker column when it is spelled Actor', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Actor, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Dial_JP,アリス,0,0,0,,えっ',
'Dialogue: 0,0:00:01.20,0:00:01.40,Dial_JP,ボブ,0,0,0,,えっ',
'Dialogue: 0,0:00:01.40,0:00:01.60,Dial_JP,キャロル,0,0,0,,えっ',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
});
test('parseSubtitleCues does not treat a custom Effect name as animation', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Sign,,0,0,0,scrolling-credit,制作',
'Dialogue: 0,0:00:01.20,0:00:01.40,Sign,,0,0,0,scrolling-credit,制作',
'Dialogue: 0,0:00:01.40,0:00:01.60,Sign,,0,0,0,scrolling-credit,制作',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
});
test('parseSubtitleCues keeps rapid ASS lines that share a style but not an actor', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Dial_JP,アリス,0,0,0,,えっ',
'Dialogue: 0,0:00:01.20,0:00:01.40,Dial_JP,ボブ,0,0,0,,えっ',
'Dialogue: 0,0:00:01.40,0:00:01.60,Dial_JP,キャロル,0,0,0,,えっ',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
});
test('parseSubtitleCues keeps untagged rapid ASS repeats separate', () => {
// No overrides at all: timing-only evidence is an SRT/VTT fallback and must not apply
// to ASS, where the absence of typesetting is itself evidence of plain dialogue.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.05,Dial_JP,,0,0,0,,えっ',
'Dialogue: 0,0:00:01.05,0:00:01.10,Dial_JP,,0,0,0,,えっ',
'Dialogue: 0,0:00:01.10,0:00:01.15,Dial_JP,,0,0,0,,えっ',
'Dialogue: 0,0:00:01.15,0:00:01.20,Dial_JP,,0,0,0,,えっ',
'Dialogue: 0,0:00:01.20,0:00:01.25,Dial_JP,,0,0,0,,えっ',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 5);
});
test('parseSubtitleCues keeps repeated signs sharing one static clip', () => {
// `\clip` is a static shape for the event. Three events with the identical clip were
// typeset the same way, so none of them is a frame of the others.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Sign,,0,0,0,,{\\clip(0,0,100,100)}注意',
'Dialogue: 0,0:00:01.20,0:00:01.40,Sign,,0,0,0,,{\\clip(0,0,100,100)}注意',
'Dialogue: 0,0:00:01.40,0:00:01.60,Sign,,0,0,0,,{\\clip(0,0,100,100)}注意',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
});
test('parseSubtitleCues collapses a sign animated through \\t', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Sign,,0,0,0,,{\\pos(10,10)\\t(0,200,\\frz30)}回る',
'Dialogue: 0,0:00:01.20,0:00:01.40,Sign,,0,0,0,,{\\pos(10,10)\\t(0,200,\\frz30)}回る',
'Dialogue: 0,0:00:01.40,0:00:03.00,Sign,,0,0,0,,{\\pos(10,10)\\t(0,200,\\frz30)}回る',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 1);
assert.equal(cues[0]!.endTime, 3.0);
});
test('parseSubtitleCues keeps a short repeated SRT run above the frame threshold', () => {
// Five contiguous 200ms cues: a sequence, but nowhere near animation-frame speed.
const lines: string[] = [];
for (let i = 0; i < 5; i++) {
const start = 1000 + i * 200;
const at = (ms: number) =>
`00:00:0${Math.floor(ms / 1000)},${String(ms % 1000).padStart(3, '0')}`;
lines.push(String(i + 1), `${at(start)} --> ${at(start + 200)}`, 'えっ', '');
}
const cues = parseSubtitleCues(lines.join('\n'), 'test.srt');
assert.equal(cues.length, 5);
});
test('parseSubtitleCues keeps a short SRT frame run below the minimum length', () => {
// Four 40ms frames: frame-speed, but too few to tell an animation from an artefact.
const lines: string[] = [];
for (let i = 0; i < 4; i++) {
const start = 7870 + i * 40;
const at = (ms: number) =>
`00:00:0${Math.floor(ms / 1000)},${String(ms % 1000).padStart(3, '0')}`;
lines.push(String(i + 1), `${at(start)} --> ${at(start + 40)}`, 'タイトル', '');
}
const cues = parseSubtitleCues(lines.join('\n'), 'test.srt');
assert.equal(cues.length, 4);
});
test('parseSubtitleCues applies ASS burst rules to ASS content behind an .srt filename', () => {
// The extension lies, so the SRT parser finds nothing and the content-sniffing fallback
// takes over -- which has to carry the `ass` source format with it, or the far stricter
// timing-only thresholds would let this karaoke burst through as three cues.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Karaoke,,0,0,0,,{\\k20}歌詞',
'Dialogue: 0,0:00:01.20,0:00:01.40,Karaoke,,0,0,0,,{\\k20}歌詞',
'Dialogue: 0,0:00:01.40,0:00:03.00,Karaoke,,0,0,0,,{\\k20}歌詞',
].join('\n');
const cues = parseSubtitleCues(content, 'test.srt');
assert.equal(cues.length, 1);
assert.equal(cues[0]!.startTime, 1.0);
assert.equal(cues[0]!.endTime, 3.0);
assert.equal(cues[0]!.text, '歌詞');
});
test('parseSubtitleCues detects subtitle formats from remote URLs', () => {
const assContent = [
'[Events]',
+160 -34
View File
@@ -1,9 +1,46 @@
import {
assOverrideSignature,
assToPlainText,
collectAssOverrideCommands,
parseAssEffectField,
type AssEffectKind,
type AssOverrideCommand,
} from './ass-text';
import { mergeDuplicateCues } from './subtitle-cue-dedup';
export interface SubtitleCue {
startTime: number;
endTime: number;
text: string;
}
/**
* Everything the parser knows about a source event, shared only with the dedup engine.
* Deduplication needs the authoring context -- which style the line belongs to, which
* override commands it carries, whether the `Effect` column was set -- to tell a karaoke
* burst apart from two characters saying the same word in turn. None of it is meaningful
* outside the parser, so the public API stays `{startTime, endTime, text}`.
*/
export interface AnnotatedSubtitleCue extends SubtitleCue {
/** Text exactly as authored, override blocks and all. */
rawText: string;
style: string;
layer: number;
/** ASS `Name`/`Actor` column. */
name: string;
/** ASS `Effect` column, verbatim. */
effect: string;
effectKind: AssEffectKind;
/** Override commands found in `{...}` blocks, with their arguments. */
overrides: readonly AssOverrideCommand[];
/** Canonical form of `overrides`, for spotting values that change across a run. */
overrideSignature: string;
/** Position in the source file, so sorting by time stays deterministic across layers. */
order: number;
}
export type SubtitleSourceFormat = 'ass' | 'srt';
const HTML_SUBTITLE_TAG_PATTERN = /<\/?[A-Za-z][^>\n]*>/g;
const SRT_TIMING_PATTERN =
@@ -23,12 +60,21 @@ function parseTimestamp(
);
}
/**
* The single ASS decode for the file path: cues leave the parser as plain text with real
* line breaks, matching what mpv hands over for the same line played live. No layer
* downstream decodes ASS again.
*/
function sanitizeSubtitleCueText(text: string): string {
return text.replace(ASS_OVERRIDE_TAG_PATTERN, '').replace(HTML_SUBTITLE_TAG_PATTERN, '').trim();
return assToPlainText(text, '\n').replace(HTML_SUBTITLE_TAG_PATTERN, '').trim();
}
export function parseSrtCues(content: string): SubtitleCue[] {
const cues: SubtitleCue[] = [];
function toPublicCues(cues: AnnotatedSubtitleCue[]): SubtitleCue[] {
return cues.map(({ startTime, endTime, text }) => ({ startTime, endTime, text }));
}
function parseAnnotatedSrtCues(content: string): AnnotatedSubtitleCue[] {
const cues: AnnotatedSubtitleCue[] = [];
const lines = content.split(/\r?\n/);
let i = 0;
@@ -60,20 +106,39 @@ export function parseSrtCues(content: string): SubtitleCue[] {
i += 1;
}
const text = sanitizeSubtitleCueText(textLines.join('\n'));
const rawText = textLines.join('\n');
const text = sanitizeSubtitleCueText(rawText);
if (text) {
cues.push({ startTime, endTime, text });
cues.push({
startTime,
endTime,
text,
rawText,
style: '',
layer: 0,
name: '',
effect: '',
effectKind: 'none',
// SRT and VTT carry no authoring metadata, and the dedup engine never reads
// overrides for those formats -- collecting them would be parsing for nobody.
overrides: [],
overrideSignature: '',
order: cues.length,
});
}
}
return cues;
}
const ASS_OVERRIDE_TAG_PATTERN = /\{[^}]*\}/g;
export function parseSrtCues(content: string): SubtitleCue[] {
return toPublicCues(parseAnnotatedSrtCues(content));
}
const ASS_TIMING_PATTERN = /^(\d+):(\d{2}):(\d{2})\.(\d{1,2})$/;
const ASS_FORMAT_PREFIX = 'Format:';
const ASS_DIALOGUE_PREFIX = 'Dialogue:';
const ASS_NAME_FIELD_ALIASES = ['name', 'actor'];
function parseAssTimestamp(raw: string): number | null {
const match = ASS_TIMING_PATTERN.exec(raw.trim());
@@ -87,13 +152,43 @@ function parseAssTimestamp(raw: string): number | null {
return hours * 3600 + minutes * 60 + seconds + centiseconds / 100;
}
export function parseAssCues(content: string): SubtitleCue[] {
const cues: SubtitleCue[] = [];
function readField(fields: string[], index: number): string {
return index >= 0 && index < fields.length ? fields[index]!.trim() : '';
}
function findFieldIndex(formatFields: string[], aliases: string[]): number {
for (const alias of aliases) {
const index = formatFields.indexOf(alias);
if (index >= 0) {
return index;
}
}
return -1;
}
function parseAnnotatedAssCues(content: string): AnnotatedSubtitleCue[] {
const cues: AnnotatedSubtitleCue[] = [];
const lines = content.split(/\r?\n/);
let inEventsSection = false;
let startFieldIndex = -1;
let endFieldIndex = -1;
let textFieldIndex = -1;
const fieldIndex = {
start: -1,
end: -1,
text: -1,
style: -1,
layer: -1,
name: -1,
effect: -1,
};
const resetFieldIndex = () => {
fieldIndex.start = -1;
fieldIndex.end = -1;
fieldIndex.text = -1;
fieldIndex.style = -1;
fieldIndex.layer = -1;
fieldIndex.name = -1;
fieldIndex.effect = -1;
};
for (const line of lines) {
const trimmed = line.trim();
@@ -101,9 +196,7 @@ export function parseAssCues(content: string): SubtitleCue[] {
if (trimmed.startsWith('[') && trimmed.endsWith(']')) {
inEventsSection = trimmed.toLowerCase() === '[events]';
if (!inEventsSection) {
startFieldIndex = -1;
endFieldIndex = -1;
textFieldIndex = -1;
resetFieldIndex();
}
continue;
}
@@ -117,9 +210,15 @@ export function parseAssCues(content: string): SubtitleCue[] {
.slice(ASS_FORMAT_PREFIX.length)
.split(',')
.map((field) => field.trim().toLowerCase());
startFieldIndex = formatFields.indexOf('start');
endFieldIndex = formatFields.indexOf('end');
textFieldIndex = formatFields.indexOf('text');
fieldIndex.start = formatFields.indexOf('start');
fieldIndex.end = formatFields.indexOf('end');
fieldIndex.text = formatFields.indexOf('text');
fieldIndex.style = formatFields.indexOf('style');
fieldIndex.layer = formatFields.indexOf('layer');
// Aegisub writes the speaker column as `Actor`; the v4+ spec calls it `Name`.
// Missing it costs the burst check its speaker guard, so both spellings count.
fieldIndex.name = findFieldIndex(formatFields, ASS_NAME_FIELD_ALIASES);
fieldIndex.effect = formatFields.indexOf('effect');
continue;
}
@@ -127,34 +226,57 @@ export function parseAssCues(content: string): SubtitleCue[] {
continue;
}
if (startFieldIndex < 0 || endFieldIndex < 0 || textFieldIndex < 0) {
if (fieldIndex.start < 0 || fieldIndex.end < 0 || fieldIndex.text < 0) {
continue;
}
const fields = trimmed.slice(ASS_DIALOGUE_PREFIX.length).split(',');
if (
startFieldIndex >= fields.length ||
endFieldIndex >= fields.length ||
textFieldIndex >= fields.length
fieldIndex.start >= fields.length ||
fieldIndex.end >= fields.length ||
fieldIndex.text >= fields.length
) {
continue;
}
const startTime = parseAssTimestamp(fields[startFieldIndex]!);
const endTime = parseAssTimestamp(fields[endFieldIndex]!);
const startTime = parseAssTimestamp(fields[fieldIndex.start]!);
const endTime = parseAssTimestamp(fields[fieldIndex.end]!);
if (startTime === null || endTime === null) {
continue;
}
const text = sanitizeSubtitleCueText(fields.slice(textFieldIndex).join(','));
if (text) {
cues.push({ startTime, endTime, text });
const rawText = fields.slice(fieldIndex.text).join(',');
const text = sanitizeSubtitleCueText(rawText);
if (!text) {
continue;
}
const effect = readField(fields, fieldIndex.effect);
const layer = Number(readField(fields, fieldIndex.layer));
const overrides = collectAssOverrideCommands(rawText);
cues.push({
startTime,
endTime,
text,
rawText,
style: readField(fields, fieldIndex.style),
layer: Number.isFinite(layer) ? layer : 0,
name: readField(fields, fieldIndex.name),
effect,
effectKind: parseAssEffectField(effect),
overrides,
overrideSignature: assOverrideSignature(overrides),
order: cues.length,
});
}
return cues;
}
export function parseAssCues(content: string): SubtitleCue[] {
return toPublicCues(parseAnnotatedAssCues(content));
}
function detectSubtitleFormat(source: string): 'srt' | 'vtt' | 'ass' | 'ssa' | null {
const [normalizedSource = source] =
(() => {
@@ -173,27 +295,31 @@ function detectSubtitleFormat(source: string): 'srt' | 'vtt' | 'ass' | 'ssa' | n
export function parseSubtitleCues(content: string, filename: string): SubtitleCue[] {
const format = detectSubtitleFormat(filename);
let cues: SubtitleCue[];
let cues: AnnotatedSubtitleCue[];
let sourceFormat: SubtitleSourceFormat = 'srt';
switch (format) {
case 'srt':
case 'vtt':
cues = parseSrtCues(content);
cues = parseAnnotatedSrtCues(content);
break;
case 'ass':
case 'ssa':
cues = parseAssCues(content);
cues = parseAnnotatedAssCues(content);
sourceFormat = 'ass';
break;
default:
cues = [];
}
if (cues.length === 0) {
const assCues = parseAssCues(content);
const srtCues = parseSrtCues(content);
cues = assCues.length >= srtCues.length ? assCues : srtCues;
const assCues = parseAnnotatedAssCues(content);
const srtCues = parseAnnotatedSrtCues(content);
const preferAss = assCues.length >= srtCues.length;
cues = preferAss ? assCues : srtCues;
sourceFormat = preferAss && assCues.length > 0 ? 'ass' : 'srt';
}
cues.sort((a, b) => a.startTime - b.startTime);
return cues;
cues.sort((a, b) => a.startTime - b.startTime || a.endTime - b.endTime || a.order - b.order);
return toPublicCues(mergeDuplicateCues(cues, sourceFormat));
}
@@ -115,6 +115,21 @@ test('subtitle processing does not emit plain payload for cached lines', async (
assert.deepEqual(emitted, [{ text: '字幕', tokens: [] }]);
});
test('text that normalizes to nothing is never cached', () => {
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
emitSubtitle: () => {},
});
// Two different inputs both reduce to an empty key; sharing one entry would serve the
// first one's tokens for the second.
controller.preCacheTokenization(' ', { text: ' ', tokens: [] });
assert.equal(controller.hasCachedSubtitle(' '), false);
assert.equal(controller.hasCachedSubtitle('\\n'), false);
assert.equal(controller.consumeCachedSubtitle('\\n'), null);
});
test('subtitle processing shows plain line while tokenization is still pending', async () => {
const emitted: SubtitleData[] = [];
let resolveTokenization: ((value: SubtitleData | null) => void) | undefined;
@@ -1,4 +1,5 @@
import type { SubtitleData } from '../../types';
import { normalizePlainSubtitleText } from './ass-text';
export interface SubtitleProcessingControllerDeps {
tokenizeSubtitle: (text: string) => Promise<SubtitleData | null>;
@@ -47,8 +48,15 @@ export interface SubtitleProcessingController {
hasCachedSubtitle: (text: string) => boolean;
}
/**
* Prefetched cues and live mpv text are both already decoded from ASS, so the key only
* has to settle whitespace for one authored line to resolve to one entry.
*
* An empty key is not a line: it is whatever normalization reduced to nothing. Callers
* must skip the cache for it rather than let every such input share one entry.
*/
export function normalizeSubtitleCacheKey(text: string): string {
return text.replace(/\r\n/g, '\n').replace(/\\N/g, '\n').replace(/\\n/g, '\n').trim();
return normalizePlainSubtitleText(text);
}
export function createSubtitleProcessingController(
@@ -72,6 +80,9 @@ export function createSubtitleProcessingController(
const getCachedTokenization = (text: string): SubtitleData | null => {
const cacheKey = normalizeSubtitleCacheKey(text);
if (!cacheKey) {
return null;
}
const cached = tokenizationCache.get(cacheKey);
if (!cached) {
return null;
@@ -83,7 +94,11 @@ export function createSubtitleProcessingController(
};
const setCachedTokenization = (text: string, payload: SubtitleData): void => {
tokenizationCache.set(normalizeSubtitleCacheKey(text), payload);
const cacheKey = normalizeSubtitleCacheKey(text);
if (!cacheKey) {
return;
}
tokenizationCache.set(cacheKey, payload);
while (tokenizationCache.size > SUBTITLE_TOKENIZATION_CACHE_LIMIT) {
const firstKey = tokenizationCache.keys().next().value;
if (firstKey !== undefined) {
@@ -254,7 +269,8 @@ export function createSubtitleProcessingController(
return cached;
},
hasCachedSubtitle: (text: string) => {
return tokenizationCache.has(normalizeSubtitleCacheKey(text));
const cacheKey = normalizeSubtitleCacheKey(text);
return cacheKey.length > 0 && tokenizationCache.has(cacheKey);
},
};
}
+4 -2
View File
@@ -1651,9 +1651,11 @@ test('tokenizeSubtitle clears JLPT level from standalone Yomitan particle token'
assert.equal(result.tokens?.[0]?.jlptLevel, undefined);
});
test('tokenizeSubtitle returns null tokens for empty normalized text', async () => {
test('tokenizeSubtitle returns the normalized text when it comes out empty', async () => {
// Handing back the original would push whatever normalization dropped into app state
// as if it were subtitle text.
const result = await tokenizeSubtitle(' \\n ', makeDeps());
assert.deepEqual(result, { text: ' \\n ', tokens: null });
assert.deepEqual(result, { text: '', tokens: null });
});
test('tokenizeSubtitle normalizes newlines before Yomitan parse request', async () => {
+7 -6
View File
@@ -27,6 +27,7 @@ import {
} from './tokenizer/yomitan-parser-runtime';
import type { YomitanTermFrequency } from './tokenizer/yomitan-parser-runtime';
import { isKanaChar } from './tokenizer/token-classification';
import { normalizePlainSubtitleText } from './ass-text';
const logger = createLogger('main:tokenizer');
@@ -886,14 +887,14 @@ export async function tokenizeSubtitle(
text: string,
deps: TokenizerServiceDeps,
): Promise<SubtitleData> {
const displayText = text
.replace(/\r\n/g, '\n')
.replace(/\\N/g, '\n')
.replace(/\\n/g, '\n')
.trim();
const displayText = normalizePlainSubtitleText(text);
// ASS decoding already happened upstream (cue parser for files, mpv for live text), so
// all this drops is whitespace -- but a whitespace-only line still normalizes to empty.
// Return the normalized form anyway: handing back the original would put a blank line
// into application state as if it were subtitle text.
if (!displayText) {
return { text, tokens: null };
return { text: displayText, tokens: null };
}
const tokenizeText = displayText
+68 -1
View File
@@ -1,6 +1,6 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { shouldForceX11ElectronBackend } from './electron-backend';
import { resolveX11ElectronRelaunchArgs, shouldForceX11ElectronBackend } from './electron-backend';
function withPlatform(platform: NodeJS.Platform, run: () => void): void {
const original = Object.getOwnPropertyDescriptor(process, 'platform');
@@ -32,3 +32,70 @@ test('shouldForceX11ElectronBackend is false off Linux', () => {
assert.equal(shouldForceX11ElectronBackend({}), false);
});
});
test('resolveX11ElectronRelaunchArgs adds the raw X11 Ozone argument on unsupported Linux', () => {
assert.deepEqual(
resolveX11ElectronRelaunchArgs(
['--start'],
{
DISPLAY: ':1',
WAYLAND_DISPLAY: 'wayland-0',
XDG_CURRENT_DESKTOP: 'KDE',
},
'linux',
),
['--start', '--ozone-platform=x11'],
);
});
test('resolveX11ElectronRelaunchArgs avoids loops and preserves native Wayland backends', () => {
const kdeWayland = {
DISPLAY: ':1',
WAYLAND_DISPLAY: 'wayland-0',
XDG_CURRENT_DESKTOP: 'KDE',
};
assert.equal(
resolveX11ElectronRelaunchArgs(['--start', '--ozone-platform=x11'], kdeWayland, 'linux'),
null,
);
assert.equal(
resolveX11ElectronRelaunchArgs(
['--start'],
{ ...kdeWayland, HYPRLAND_INSTANCE_SIGNATURE: 'hypr' },
'linux',
),
null,
);
assert.equal(resolveX11ElectronRelaunchArgs(['--start'], kdeWayland, 'darwin'), null);
assert.equal(
resolveX11ElectronRelaunchArgs(
[],
{
...kdeWayland,
SUBMINER_APP_ARGC: '1',
SUBMINER_APP_ARG_0: '--start',
},
'linux',
)?.at(-1),
'--ozone-platform=x11',
);
assert.equal(
resolveX11ElectronRelaunchArgs([], { ...kdeWayland, SUBMINER_X11_BOOTSTRAPPED: '1' }, 'linux'),
null,
);
});
test('resolveX11ElectronRelaunchArgs replaces an explicit unsupported Wayland argument', () => {
assert.deepEqual(
resolveX11ElectronRelaunchArgs(
['--start', '--ozone-platform', 'wayland'],
{
DISPLAY: ':1',
WAYLAND_DISPLAY: 'wayland-0',
XDG_CURRENT_DESKTOP: 'KDE',
},
'linux',
),
['--start', '--ozone-platform=x11'],
);
});
+36 -2
View File
@@ -4,6 +4,9 @@ import { isSupportedWaylandCompositor } from '../../shared/mpv-x11-backend';
const logger = createLogger('core:electron-backend');
export const X11_ELECTRON_BOOTSTRAP_ENV = 'SUBMINER_X11_BOOTSTRAPPED';
const X11_ELECTRON_OZONE_ARG = '--ozone-platform=x11';
function getElectronOzonePlatformHint(env: NodeJS.ProcessEnv = process.env): string | null {
const hint = env.ELECTRON_OZONE_PLATFORM_HINT?.trim().toLowerCase();
if (hint) return hint;
@@ -24,11 +27,42 @@ function getElectronOzonePlatformHint(env: NodeJS.ProcessEnv = process.env): str
* Electron Wayland backend is unsupported); the Hyprland/Sway case is left untouched so
* {@link enforceUnsupportedWaylandMode} can report it.
*/
export function shouldForceX11ElectronBackend(env: NodeJS.ProcessEnv = process.env): boolean {
if (process.platform !== 'linux') return false;
export function shouldForceX11ElectronBackend(
env: NodeJS.ProcessEnv = process.env,
platform: NodeJS.Platform = process.platform,
): boolean {
if (platform !== 'linux') return false;
return !isSupportedWaylandCompositor(env);
}
export function resolveX11ElectronRelaunchArgs(
args: string[],
env: NodeJS.ProcessEnv = process.env,
platform: NodeJS.Platform = process.platform,
): string[] | null {
if (!shouldForceX11ElectronBackend(env, platform)) return null;
if (env[X11_ELECTRON_BOOTSTRAP_ENV] === '1') return null;
const retainedArgs: string[] = [];
let alreadyForced = false;
for (let index = 0; index < args.length; index += 1) {
const arg = args[index];
if (arg === '--ozone-platform') {
const value = args[index + 1];
alreadyForced = value?.trim().toLowerCase() === 'x11';
if (value && !value.startsWith('--')) index += 1;
continue;
}
if (arg?.startsWith('--ozone-platform=')) {
alreadyForced = arg.slice('--ozone-platform='.length).trim().toLowerCase() === 'x11';
continue;
}
if (arg) retainedArgs.push(arg);
}
return alreadyForced ? null : [...retainedArgs, X11_ELECTRON_OZONE_ARG];
}
export function forceX11Backend(args: CliArgs): void {
if (!shouldStartApp(args)) return;
if (!shouldForceX11ElectronBackend()) return;
+17 -1
View File
@@ -52,9 +52,15 @@ function resolveRuntimeDefaultNotificationIconPath(): string | null {
});
}
/**
* Live notifications keyed by `replaceId`. Electron exposes no native "replace this notification"
* flag, so a repeated status closes its predecessor instead of stacking a fresh toast per update.
*/
const notificationsByReplaceId = new Map<string, Electron.Notification>();
export function showDesktopNotification(
title: string,
options: { body?: string; icon?: string },
options: { body?: string; icon?: string; replaceId?: string },
): void {
const notificationOptions: {
title: string;
@@ -98,5 +104,15 @@ export function showDesktopNotification(
}
const notification = new Notification(notificationOptions);
const replaceId = options.replaceId?.trim();
if (replaceId) {
notificationsByReplaceId.get(replaceId)?.close();
notificationsByReplaceId.set(replaceId, notification);
notification.once('close', () => {
if (notificationsByReplaceId.get(replaceId) === notification) {
notificationsByReplaceId.delete(replaceId);
}
});
}
notification.show();
}
+16
View File
@@ -25,8 +25,24 @@ import {
applyBackgroundBootstrapCommandLineSwitches,
applyEarlyLinuxCommandLineSwitches,
resolveLinuxPasswordStoreValue,
spawnDetachedApp,
} from './main-entry-runtime';
test('detached app launch policy stays in the startup runtime utilities', () => {
const entrySource = fs.readFileSync(path.join(process.cwd(), 'src/main-entry.ts'), 'utf8');
const runtimeSource = fs.readFileSync(
path.join(process.cwd(), 'src/main-entry-runtime.ts'),
'utf8',
);
assert.equal(typeof spawnDetachedApp, 'function');
assert.doesNotMatch(entrySource, /function spawnDetachedApp/);
assert.match(
runtimeSource,
/child\.once\('error', \(error\) => \{\s*console\.error\([^;]*error\);\s*\}\);\s*child\.unref\(\)/,
);
});
test('background bootstrap exits through Electron so Chromium children shut down', () => {
const exitCodes: number[] = [];
exitBackgroundBootstrap({ exit: (code) => exitCodes.push(code) });
+21
View File
@@ -1,7 +1,9 @@
import fs from 'node:fs';
import os from 'node:os';
import { spawn } from 'node:child_process';
import { CliArgs, hasExplicitCommand, parseArgs, shouldStartApp } from './cli/args';
import { resolveConfigDir } from './config/path-resolution';
import { resolveAppImageMountKeepaliveInvocation } from './main/appimage-mount-keepalive';
const BACKGROUND_ARG = '--background';
const START_ARG = '--start';
@@ -265,6 +267,25 @@ export function exitBackgroundBootstrap(app: BackgroundBootstrapAppLike): void {
app.exit(0);
}
export function spawnDetachedApp(childArgs: string[], env: NodeJS.ProcessEnv): void {
const keepalive = resolveAppImageMountKeepaliveInvocation(env);
const child = keepalive
? spawn(keepalive.command, [...keepalive.args, ...childArgs], {
detached: true,
stdio: 'ignore',
env,
})
: spawn(process.execPath, childArgs, {
detached: true,
stdio: 'ignore',
env,
});
child.once('error', (error) => {
console.error('Failed to spawn detached SubMiner app:', error);
});
child.unref();
}
export function shouldHandleHelpOnlyAtEntry(argv: string[], env: NodeJS.ProcessEnv): boolean {
if (env.ELECTRON_RUN_AS_NODE === '1') return false;
const args = parseCliArgs(argv);
+19 -16
View File
@@ -1,5 +1,4 @@
import os from 'node:os';
import { spawn } from 'node:child_process';
import { app, dialog, shell } from 'electron';
import { printHelp } from './cli/help';
import {
@@ -20,9 +19,9 @@ import {
shouldHandleHelpOnlyAtEntry,
shouldHandleLaunchMpvAtEntry,
shouldHandleStatsDaemonCommandAtEntry,
spawnDetachedApp,
} from './main-entry-runtime';
import { requestSingleInstanceLockEarly } from './main/early-single-instance';
import { resolveAppImageMountKeepaliveInvocation } from './main/appimage-mount-keepalive';
import { readConfiguredWindowsMpvLaunch } from './main-entry-launch-config';
import { isAppControlServerAvailable, sendAppControlCommand } from './shared/app-control-client';
import {
@@ -44,6 +43,10 @@ import {
resolveDefaultLogFilePath,
type LogRotation,
} from './shared/log-files';
import {
resolveX11ElectronRelaunchArgs,
X11_ELECTRON_BOOTSTRAP_ENV,
} from './core/utils/electron-backend';
const DEFAULT_TEXTHOOKER_PORT = 5174;
@@ -296,26 +299,26 @@ async function runEntryProcess(): Promise<void> {
return;
}
const childArgs = hasTransportedStartupArgs(process.env) ? [] : process.argv.slice(1);
const x11ChildArgs = resolveX11ElectronRelaunchArgs(childArgs, process.env);
if (shouldDetachBackgroundLaunch(process.argv, process.env)) {
const childArgs = hasTransportedStartupArgs(process.env) ? [] : process.argv.slice(1);
const keepalive = resolveAppImageMountKeepaliveInvocation(process.env);
const child = keepalive
? spawn(keepalive.command, [...keepalive.args, ...childArgs], {
detached: true,
stdio: 'ignore',
env: sanitizeBackgroundEnv(process.env),
})
: spawn(process.execPath, childArgs, {
detached: true,
stdio: 'ignore',
env: sanitizeBackgroundEnv(process.env),
});
child.unref();
const childEnv = sanitizeBackgroundEnv(process.env);
if (x11ChildArgs) childEnv[X11_ELECTRON_BOOTSTRAP_ENV] = '1';
spawnDetachedApp(x11ChildArgs ?? childArgs, childEnv);
// Let Electron stop bootstrap Chromium children before its AppImage mount is released.
exitBackgroundBootstrap(app);
return;
}
if (x11ChildArgs) {
const childEnv = sanitizeStartupEnv(process.env);
childEnv[X11_ELECTRON_BOOTSTRAP_ENV] = '1';
spawnDetachedApp(x11ChildArgs, childEnv);
exitBackgroundBootstrap(app);
return;
}
startMainProcess();
}
-10
View File
@@ -256,7 +256,6 @@ import {
import {
enforceUnsupportedWaylandMode,
forceX11Backend,
shouldForceX11ElectronBackend,
generateDefaultConfigFile,
resolveConfiguredShortcuts,
resolveKeybindings,
@@ -597,15 +596,6 @@ if (process.platform === 'linux') {
);
app.commandLine.appendSwitch('password-store', passwordStore);
createLogger('main').debug(`Applied --password-store ${passwordStore}`);
// Pin the overlay to XWayland on unsupported Wayland sessions (everything except
// Hyprland/Sway). `setAlwaysOnTop`/`moveTop` are no-ops under a native Wayland surface,
// so the overlay can only stay above mpv under X11/XWayland. The command-line switch is
// applied at module load (before app init) so it reliably wins over the late env-var
// fallback in forceX11Backend().
if (shouldForceX11ElectronBackend(process.env)) {
app.commandLine.appendSwitch('ozone-platform-hint', 'x11');
createLogger('main').debug('Forced ozone-platform-hint=x11 for XWayland overlay stacking');
}
}
app.setName('SubMiner');
@@ -1903,7 +1903,7 @@ test('generateForCurrentMedia logs progress while resolving and rebuilding snaps
'[dictionary] current anime guess: The Eminence in Shadow (episode 5)',
'[dictionary] AniList match: The Eminence in Shadow -> AniList 130298',
'[dictionary] snapshot miss for AniList 130298, fetching characters',
'[dictionary] downloaded AniList character page 1 for AniList 130298',
'[dictionary] downloaded AniList character page 1 for AniList 130298 (1 characters)',
'[dictionary] downloading 1 images for AniList 130298',
'[dictionary] stored snapshot for AniList 130298: 16 terms',
'[dictionary] building ZIP for AniList 130298',
+48 -4
View File
@@ -64,6 +64,7 @@ export type {
CharacterDictionarySnapshotProgress,
CharacterDictionarySnapshotProgressCallbacks,
CharacterDictionarySnapshotResult,
CharacterDictionarySnapshotStageProgress,
MergedCharacterDictionaryBuildResult,
} from './character-dictionary-runtime/types';
@@ -363,19 +364,28 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
deps.logInfo?.(`[dictionary] snapshot stale for AniList ${mediaId}: ${refreshReason}`);
}
const progressMediaTitle = mediaTitleHint || `AniList ${mediaId}`;
progress?.onGenerating?.({
mediaId,
mediaTitle: mediaTitleHint || `AniList ${mediaId}`,
mediaTitle: progressMediaTitle,
});
deps.logInfo?.(`[dictionary] snapshot miss for AniList ${mediaId}, fetching characters`);
const { mediaTitle: fetchedMediaTitle, characters } = await fetchCharactersForMedia(
mediaId,
beforeRequest,
(page) => {
(page, charactersSoFar) => {
deps.logInfo?.(
`[dictionary] downloaded AniList character page ${page} for AniList ${mediaId}`,
`[dictionary] downloaded AniList character page ${page} for AniList ${mediaId} (${charactersSoFar} characters)`,
);
progress?.onGenerateProgress?.({
mediaId,
mediaTitle: progressMediaTitle,
stage: 'characters',
completed: charactersSoFar,
total: null,
page,
});
},
);
if (characters.length === 0) {
@@ -403,12 +413,26 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
);
}
let hasAttemptedImageDownload = false;
let attemptedImageCount = 0;
for (const entry of allImageUrls) {
if (hasAttemptedImageDownload) {
await sleepMs(CHARACTER_IMAGE_DOWNLOAD_DELAY_MS);
}
hasAttemptedImageDownload = true;
const image = await downloadCharacterImage(entry.url, entry.id);
attemptedImageCount += 1;
progress?.onGenerateProgress?.({
mediaId,
mediaTitle: progressMediaTitle,
stage: 'images',
completed: attemptedImageCount,
total: allImageUrls.length,
});
if (attemptedImageCount % 100 === 0) {
deps.logInfo?.(
`[dictionary] downloaded ${attemptedImageCount}/${allImageUrls.length} images for AniList ${mediaId}`,
);
}
if (!image) continue;
if (entry.kind === 'character') {
imagesByCharacterId.set(entry.id, {
@@ -425,11 +449,31 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
const nameSplitTokenizerAvailable = isNameSplitTokenizerAvailable();
const resolvedNameSplits = nameSplitTokenizerAvailable
? await resolveJapaneseNameSplits(characters, deps.tokenizeJapaneseName!, deps.logWarn)
? await resolveJapaneseNameSplits(
characters,
deps.tokenizeJapaneseName!,
deps.logWarn,
(completed, total) => {
progress?.onGenerateProgress?.({
mediaId,
mediaTitle: progressMediaTitle,
stage: 'names',
completed,
total,
});
},
)
: undefined;
const nameSplitSource =
resolvedNameSplits && resolvedNameSplits.size > 0 ? 'mecab' : 'heuristic';
progress?.onGenerateProgress?.({
mediaId,
mediaTitle: progressMediaTitle,
stage: 'saving',
completed: 0,
total: null,
});
const snapshot = buildSnapshotFromCharacters(
mediaId,
fetchedMediaTitle || mediaTitleHint || `AniList ${mediaId}`,
@@ -278,7 +278,7 @@ export async function fetchAniListMediaCandidateById(
export async function fetchCharactersForMedia(
mediaId: number,
beforeRequest?: () => Promise<void>,
onPageFetched?: (page: number) => void,
onPageFetched?: (page: number, charactersSoFar: number) => void,
): Promise<{
mediaTitle: string;
characters: CharacterRecord[];
@@ -345,7 +345,6 @@ export async function fetchCharactersForMedia(
},
beforeRequest,
);
onPageFetched?.(page);
const media = data.Media;
if (!media) {
@@ -415,6 +414,8 @@ export async function fetchCharactersForMedia(
});
}
onPageFetched?.(page, characters.length);
const hasNextPage = Boolean(media.characters?.pageInfo?.hasNextPage);
if (!hasNextPage) {
break;
@@ -86,8 +86,10 @@ export async function resolveJapaneseNameSplits(
characters: CharacterRecord[],
tokenize: NameSplitTokenizer,
logWarn?: (message: string) => void,
onCharacterResolved?: (completed: number, total: number) => void,
): Promise<Map<string, ResolvedNameSplit>> {
const splits = new Map<string, ResolvedNameSplit>();
let resolvedCharacters = 0;
for (const character of characters) {
const familyHintReading = buildReadingFromHint(character.lastNameHint?.trim() || '');
const givenHintReading = buildReadingFromHint(character.firstNameHint?.trim() || '');
@@ -113,6 +115,8 @@ export async function resolveJapaneseNameSplits(
splits.set(name, { family, given });
}
}
resolvedCharacters += 1;
onCharacterResolved?.(resolvedCharacters, characters.length);
}
return splits;
}
@@ -122,9 +122,24 @@ export type CharacterDictionarySnapshotProgress = {
mediaTitle: string;
};
export type CharacterDictionarySnapshotStage = 'characters' | 'images' | 'names' | 'saving';
/**
* Fine-grained generation progress. `total` is null while the work size is still unknown (AniList
* paginates characters, so the character count only settles on the last page).
*/
export type CharacterDictionarySnapshotStageProgress = CharacterDictionarySnapshotProgress & {
stage: CharacterDictionarySnapshotStage;
completed: number;
total: number | null;
/** AniList page currently being downloaded; only set during the `characters` stage. */
page?: number;
};
export type CharacterDictionarySnapshotProgressCallbacks = {
onChecking?: (progress: CharacterDictionarySnapshotProgress) => void;
onGenerating?: (progress: CharacterDictionarySnapshotProgress) => void;
onGenerateProgress?: (progress: CharacterDictionarySnapshotStageProgress) => void;
};
export type MergedCharacterDictionaryBuildResult = {
@@ -3,7 +3,7 @@ import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { buildDictionaryZip } from './zip';
import { buildDictionaryZip, readDictionaryZipRevision } from './zip';
import type { CharacterDictionaryTermEntry } from './types';
function makeTempDir(): string {
@@ -105,3 +105,63 @@ test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', ()
cleanupDir(tempDir);
}
});
test('readDictionaryZipRevision reads the built revision and rejects foreign archives', () => {
const dir = makeTempDir();
try {
const zipPath = path.join(dir, 'merged.zip');
buildDictionaryZip(
zipPath,
'SubMiner Character Dictionary',
'Character names',
'rev-42',
[
{
term: 'ルフィ',
reading: 'ルフィ',
role: 'main',
glossary: [],
} as unknown as CharacterDictionaryTermEntry,
],
[],
);
assert.equal(readDictionaryZipRevision(zipPath), 'rev-42');
assert.equal(readDictionaryZipRevision(path.join(dir, 'missing.zip')), null);
const archive = fs.readFileSync(zipPath);
const truncatedPath = path.join(dir, 'truncated.zip');
fs.writeFileSync(truncatedPath, archive.subarray(0, 40));
assert.equal(readDictionaryZipRevision(truncatedPath), null);
// An archive cut short after index.json still holds a readable revision, but importing it
// would hand Yomitan a half-written file: the missing end-of-central-directory record has to
// reject it. One byte off the end is enough to make the record incomplete.
for (const missingBytes of [1, 22, archive.length - 200]) {
const cutPath = path.join(dir, `cut-${missingBytes}.zip`);
fs.writeFileSync(cutPath, archive.subarray(0, archive.length - missingBytes));
assert.equal(readDictionaryZipRevision(cutPath), null, `cut of ${missingBytes} bytes`);
}
// Same size, corrupt directory: a record overwritten in place has to be rejected too.
const centralStart = archive.readUInt32LE(archive.length - 22 + 16);
const brokenSignaturePath = path.join(dir, 'broken-signature.zip');
const brokenSignature = Buffer.from(archive);
brokenSignature.writeUInt32LE(0xdeadbeef, centralStart);
fs.writeFileSync(brokenSignaturePath, brokenSignature);
assert.equal(readDictionaryZipRevision(brokenSignaturePath), null);
const brokenLengthPath = path.join(dir, 'broken-length.zip');
const brokenLength = Buffer.from(archive);
// Name length that runs the walk past the end of the directory.
brokenLength.writeUInt16LE(0xffff, centralStart + 28);
fs.writeFileSync(brokenLengthPath, brokenLength);
assert.equal(readDictionaryZipRevision(brokenLengthPath), null);
const foreignPath = path.join(dir, 'foreign.zip');
fs.writeFileSync(foreignPath, Buffer.from('not a zip at all', 'utf8'));
assert.equal(readDictionaryZipRevision(foreignPath), null);
} finally {
cleanupDir(dir);
}
});
+18 -1
View File
@@ -1,5 +1,5 @@
import * as path from 'path';
import { writeStoredZip } from '../../shared/stored-zip';
import { readStoredZipFirstFile, writeStoredZip } from '../../shared/stored-zip';
import { ensureDir } from './fs-utils';
import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } from './types';
@@ -31,6 +31,23 @@ function createTagBank(): Array<[string, string, number, string, number]> {
];
}
/**
* Revision recorded inside a built dictionary ZIP, or null when the archive is missing, truncated,
* or not one of ours. `index.json` is always the first entry written by {@link buildDictionaryZip}.
*/
export function readDictionaryZipRevision(zipPath: string): string | null {
const firstFile = readStoredZipFirstFile(zipPath);
if (!firstFile || firstFile.name !== 'index.json') {
return null;
}
try {
const index = JSON.parse(firstFile.data.toString('utf8')) as { revision?: unknown };
return typeof index.revision === 'string' && index.revision.length > 0 ? index.revision : null;
} catch {
return null;
}
}
export function buildDictionaryZip(
outputPath: string,
dictionaryTitle: string,
@@ -0,0 +1,90 @@
import type { CharacterDictionarySnapshotStageProgress } from '../character-dictionary-runtime';
export function buildSyncingMessage(mediaTitle: string): string {
return `Updating character dictionary for ${mediaTitle}...`;
}
export function buildCheckingMessage(mediaTitle: string): string {
return `Checking character dictionary for ${mediaTitle}...`;
}
export function buildGeneratingMessage(mediaTitle: string, detail?: string): string {
return detail
? `Generating character dictionary for ${mediaTitle} (${detail})...`
: `Generating character dictionary for ${mediaTitle}...`;
}
export function formatCharacterDictionaryProgressDetail(
progress: CharacterDictionarySnapshotStageProgress,
remainingMs: number | null,
): string {
if (progress.stage === 'saving') {
return 'saving snapshot';
}
if (progress.stage === 'names') {
return progress.total !== null && progress.total > 0
? `name ${progress.completed}/${progress.total}`
: `${progress.completed} names`;
}
if (progress.stage === 'images') {
const counted =
progress.total !== null && progress.total > 0
? `image ${progress.completed}/${progress.total}`
: `${progress.completed} images`;
return remainingMs !== null
? `${counted}, ~${formatRemainingDuration(remainingMs)} left`
: counted;
}
const page = typeof progress.page === 'number' ? `page ${progress.page}, ` : '';
return `${page}${progress.completed} characters`;
}
export function formatElapsedDuration(elapsedMs: number): string {
const totalSeconds = Math.max(0, Math.floor(elapsedMs / 1000));
const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60;
return minutes > 0 ? `${minutes}m ${String(seconds).padStart(2, '0')}s` : `${seconds}s`;
}
/** Coarser than the elapsed clock: an estimate that ticks every second reads as precision it lacks. */
export function formatRemainingDuration(remainingMs: number): string {
const totalSeconds = Math.max(1, Math.round(remainingMs / 1000));
if (totalSeconds >= 60) {
return `${Math.max(1, Math.round(totalSeconds / 60))}m`;
}
return `${Math.max(5, Math.ceil(totalSeconds / 5) * 5)}s`;
}
/**
* The elapsed clock is the part that proves the app is alive: a stalled network fetch freezes the
* counts, but the clock keeps moving.
*/
export function joinGeneratingDetail(detail: string | null, elapsedMs: number): string | undefined {
const parts = [detail, elapsedMs >= 5_000 ? formatElapsedDuration(elapsedMs) : null].filter(
(part): part is string => typeof part === 'string' && part.length > 0,
);
return parts.length > 0 ? parts.join(' · ') : undefined;
}
export function buildImportingMessage(mediaTitle: string, elapsedMs?: number): string {
const elapsed =
typeof elapsedMs === 'number' && elapsedMs >= 1000
? ` (${formatElapsedDuration(elapsedMs)})`
: '';
return `Importing character dictionary for ${mediaTitle}${elapsed}...`;
}
export function buildBuildingMessage(mediaTitle: string): string {
return `Building character dictionary for ${mediaTitle}...`;
}
export function buildReadyMessage(mediaTitle: string): string {
return `Character dictionary ready for ${mediaTitle}`;
}
export function buildFailedMessage(mediaTitle: string | null, errorMessage: string): string {
if (mediaTitle) {
return `Character dictionary sync failed for ${mediaTitle}: ${errorMessage}`;
}
return `Character dictionary sync failed: ${errorMessage}`;
}
@@ -214,3 +214,69 @@ test('auto sync notifications let startup sequencer own osd-system desktop deliv
assert.deepEqual(calls, ['osd:importing', 'desktop:SubMiner:importing']);
});
test('auto sync desktop notifications reuse one replace id across every phase', () => {
const replaceIds: Array<string | undefined> = [];
const deps = {
getNotificationType: () => 'system' as const,
showOsd: () => undefined,
showDesktopNotification: (_title: string, options: { body?: string; replaceId?: string }) => {
replaceIds.push(options.replaceId);
},
};
for (const phase of ['checking', 'generating', 'importing', 'ready'] as const) {
notifyCharacterDictionaryAutoSyncStatus(makeEvent(phase, phase), deps);
}
assert.deepEqual(replaceIds, [
'character-dictionary-auto-sync',
'character-dictionary-auto-sync',
'character-dictionary-auto-sync',
'character-dictionary-auto-sync',
]);
});
test('overlay-unavailable desktop fallback shares the same replace id', () => {
const replaceIds: Array<string | undefined> = [];
notifyCharacterDictionaryAutoSyncStatus(makeEvent('generating', 'generating'), {
getNotificationType: () => 'overlay',
showOsd: () => undefined,
showDesktopNotification: (_title, options) => {
replaceIds.push(options.replaceId);
},
});
assert.deepEqual(replaceIds, ['character-dictionary-auto-sync']);
});
test('startup lanes keep one desktop notification per lane', () => {
const calls: Array<{ body?: string; replaceId?: string }> = [];
const sequencer = createStartupOsdSequencer({
getNotificationType: () => 'system',
showOsd: () => undefined,
showDesktopNotification: (_title, options) => {
calls.push(options);
},
});
sequencer.markTokenizationReady();
notifyCharacterDictionaryAutoSyncStatus(makeEvent('generating', 'generating one'), {
getNotificationType: () => 'osd',
showOsd: () => undefined,
showDesktopNotification: () => undefined,
startupOsdSequencer: sequencer,
});
notifyCharacterDictionaryAutoSyncStatus(makeEvent('generating', 'generating two'), {
getNotificationType: () => 'osd',
showOsd: () => undefined,
showDesktopNotification: () => undefined,
startupOsdSequencer: sequencer,
});
assert.deepEqual(
calls.map((call) => call.replaceId),
['startup-status', 'startup-status'],
);
});
@@ -9,7 +9,7 @@ export interface CharacterDictionaryAutoSyncNotificationDeps {
getNotificationType: () => NotificationType | undefined;
showOsd: (message: string) => boolean | void;
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
showDesktopNotification: (title: string, options: { body?: string }) => void;
showDesktopNotification: (title: string, options: { body?: string; replaceId?: string }) => void;
startupOsdSequencer?: {
notifyCharacterDictionaryStatus: (
event: StartupOsdSequencerCharacterDictionaryEvent,
@@ -17,6 +17,10 @@ export interface CharacterDictionaryAutoSyncNotificationDeps {
};
}
// One live desktop notification for the whole sync: progress updates replace each other and the
// terminal ready/failed message replaces the last progress one, matching the overlay toast.
const CHARACTER_DICTIONARY_DESKTOP_NOTIFICATION_ID = 'character-dictionary-auto-sync';
function isTerminalPhase(phase: CharacterDictionaryAutoSyncNotificationEvent['phase']): boolean {
return phase === 'ready' || phase === 'failed';
}
@@ -53,7 +57,10 @@ export function notifyCharacterDictionaryAutoSyncStatus(
persistent: !isTerminalPhase(event.phase),
});
} else if (!shouldShowDesktop(type)) {
deps.showDesktopNotification('SubMiner', { body: event.message });
deps.showDesktopNotification('SubMiner', {
body: event.message,
replaceId: CHARACTER_DICTIONARY_DESKTOP_NOTIFICATION_ID,
});
}
}
@@ -69,6 +76,9 @@ export function notifyCharacterDictionaryAutoSyncStatus(
}
if (shouldShowDesktop(type) && !startupSequencerShown) {
deps.showDesktopNotification('SubMiner', { body: event.message });
deps.showDesktopNotification('SubMiner', {
body: event.message,
replaceId: CHARACTER_DICTIONARY_DESKTOP_NOTIFICATION_ID,
});
}
}
@@ -3,6 +3,7 @@ import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import test from 'node:test';
import { buildDictionaryZip } from '../character-dictionary-runtime/zip';
import {
createCharacterDictionaryAutoSyncRuntimeService,
getCharacterDictionaryManagerSnapshot,
@@ -14,6 +15,14 @@ function makeTempDir(): string {
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-char-dict-auto-sync-'));
}
async function waitUntil(predicate: () => boolean, label: string): Promise<void> {
for (let attempt = 0; attempt < 500; attempt += 1) {
if (predicate()) return;
await new Promise((resolve) => setTimeout(resolve, 0));
}
throw new Error(`Timed out waiting for ${label}`);
}
function createDeferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
let resolve!: (value: T) => void;
const promise = new Promise<T>((nextResolve) => {
@@ -187,7 +196,7 @@ test('auto sync imports merged dictionary and persists MRU state', async () => {
'[dictionary:auto-sync] syncing current anime snapshot',
'[dictionary:auto-sync] active AniList media set: 130298 - The Eminence in Shadow',
'[dictionary:auto-sync] rebuilding merged dictionary for active anime set',
'[dictionary:auto-sync] importing merged dictionary: /tmp/subminer-character-dictionary.zip',
'[dictionary:auto-sync] importing merged dictionary: /tmp/subminer-character-dictionary.zip (timeout 120000ms)',
'[dictionary:auto-sync] applying Yomitan settings for SubMiner Character Dictionary',
'[dictionary:auto-sync] synced AniList 130298: SubMiner Character Dictionary (2544 entries)',
]);
@@ -367,7 +376,14 @@ test('auto sync reimports existing merged zip without rebuilding on unchanged re
const userDataPath = makeTempDir();
const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
fs.mkdirSync(dictionariesDir, { recursive: true });
fs.writeFileSync(path.join(dictionariesDir, 'merged.zip'), 'cached-zip', 'utf8');
buildDictionaryZip(
path.join(dictionariesDir, 'merged.zip'),
'SubMiner Character Dictionary',
'Character names',
'rev-7',
[{ term: 'フリーレン', reading: 'フリーレン', role: 'main', glossary: [] } as never],
[],
);
const mergedBuilds: number[][] = [];
const imports: string[] = [];
let importedRevision: string | null = null;
@@ -958,11 +974,9 @@ test('auto sync emits building while merged dictionary generation is in flight',
});
const syncPromise = runtime.runSyncNow();
await Promise.resolve();
assert.equal(
events.some((event) => event.phase === 'building'),
true,
await waitUntil(
() => events.some((event) => event.phase === 'building'),
'the building status event',
);
buildDeferred.resolve({
@@ -1029,8 +1043,7 @@ test('auto sync waits for tokenization-ready gate before Yomitan mutations', asy
});
const syncPromise = runtime.runSyncNow();
await Promise.resolve();
await Promise.resolve();
await waitUntil(() => calls.includes('wait'), 'the tokenization-ready gate');
assert.deepEqual(calls, ['build', 'wait']);
@@ -1039,3 +1052,395 @@ test('auto sync waits for tokenization-ready gate before Yomitan mutations', asy
assert.deepEqual(calls, ['build', 'wait', 'info', 'import', 'settings']);
});
test('auto sync scales the import timeout with the merged dictionary size', async () => {
const userDataPath = makeTempDir();
const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
fs.mkdirSync(dictionariesDir, { recursive: true });
const zipPath = path.join(dictionariesDir, 'merged.zip');
// 2 MB of merged dictionary buys ~12s of import budget on top of the base.
fs.writeFileSync(zipPath, Buffer.alloc(2 * 1024 * 1024));
const events: Array<{ phase: string; message: string }> = [];
let importedRevision: string | null = null;
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath,
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async () => ({
mediaId: 21,
mediaTitle: 'ONE PIECE',
entryCount: 4000,
fromCache: false,
updatedAt: 1000,
}),
buildMergedDictionary: async () => ({
zipPath,
revision: 'rev-21',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 4000,
}),
getYomitanDictionaryInfo: async () =>
importedRevision
? [{ title: 'SubMiner Character Dictionary', revision: importedRevision }]
: [],
importYomitanDictionary: async () => {
// Far longer than the quick-operation budget, well inside the size-scaled one.
await new Promise((resolve) => setTimeout(resolve, 400));
importedRevision = 'rev-21';
return true;
},
deleteYomitanDictionary: async () => true,
upsertYomitanDictionarySettings: async () => true,
now: () => 1000,
// Comfortable for the stubs that resolve immediately, still far under the import's 400ms.
operationTimeoutMs: 100,
dictionaryImportTimeoutBaseMs: 20,
onSyncStatus: (event) => {
events.push({ phase: event.phase, message: event.message });
},
});
await runtime.runSyncNow();
assert.equal(
events.some((event) => event.phase === 'failed'),
false,
);
assert.deepEqual(events.at(-1), {
phase: 'ready',
message: 'Character dictionary ready for ONE PIECE',
});
});
test('auto sync reports the scaled budget when an import really does hang', async () => {
const userDataPath = makeTempDir();
const events: Array<{ phase: string; message: string }> = [];
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath,
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async () => ({
mediaId: 21,
mediaTitle: 'ONE PIECE',
entryCount: 4000,
fromCache: false,
updatedAt: 1000,
}),
buildMergedDictionary: async () => ({
zipPath: path.join(userDataPath, 'character-dictionaries', 'missing.zip'),
revision: 'rev-21',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 4000,
}),
getYomitanDictionaryInfo: async () => [],
importYomitanDictionary: () => new Promise<boolean>(() => {}),
deleteYomitanDictionary: async () => true,
upsertYomitanDictionarySettings: async () => true,
now: () => 1000,
dictionaryImportTimeoutBaseMs: 20,
onSyncStatus: (event) => {
events.push({ phase: event.phase, message: event.message });
},
});
await assert.rejects(
runtime.runSyncNow(),
/importYomitanDictionary\(missing\.zip\) timed out after 20ms/,
);
assert.equal(events.at(-1)?.phase, 'failed');
});
test('auto sync ticks the importing notification while the import runs', async () => {
const userDataPath = makeTempDir();
const events: Array<{ phase: string; message: string }> = [];
const scheduled: Array<() => void> = [];
const importDeferred = createDeferred<boolean>();
let clock = 1000;
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath,
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async () => ({
mediaId: 21,
mediaTitle: 'ONE PIECE',
entryCount: 4000,
fromCache: false,
updatedAt: 1000,
}),
buildMergedDictionary: async () => ({
zipPath: '/tmp/merged.zip',
revision: 'rev-21',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 4000,
}),
getYomitanDictionaryInfo: async () => [],
importYomitanDictionary: () => importDeferred.promise,
deleteYomitanDictionary: async () => true,
upsertYomitanDictionarySettings: async () => true,
now: () => clock,
schedule: (fn) => {
scheduled.push(fn);
return 0 as unknown as ReturnType<typeof setTimeout>;
},
clearSchedule: () => undefined,
onSyncStatus: (event) => {
events.push({ phase: event.phase, message: event.message });
},
});
const syncPromise = runtime.runSyncNow();
await waitUntil(
() => events.some((event) => event.phase === 'importing'),
'the importing status event',
);
clock = 1000 + 65_000;
// The importing heartbeat is the most recently scheduled tick.
scheduled.at(-1)!();
assert.deepEqual(events.at(-1), {
phase: 'importing',
message: 'Importing character dictionary for ONE PIECE (1m 05s)...',
});
importDeferred.resolve(true);
await syncPromise;
assert.equal(events.at(-1)?.phase, 'ready');
});
test('auto sync reports character and image counts while generating a snapshot', async () => {
const userDataPath = makeTempDir();
const events: Array<{ phase: string; message: string }> = [];
let clock = 1000;
let importedRevision: string | null = null;
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath,
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async (_targetPath, progress) => {
progress?.onGenerating?.({ mediaId: 21, mediaTitle: 'ONE PIECE' });
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'characters',
completed: 50,
total: null,
page: 12,
});
// Same stage, same clock tick: throttled away so a 33-page fetch cannot spam the overlay.
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'characters',
completed: 100,
total: null,
page: 13,
});
// A stage change always reports, throttle window or not.
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'images',
completed: 1,
total: 1220,
});
clock += 2000;
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'images',
completed: 240,
total: 1220,
});
clock += 6000;
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'names',
completed: 800,
total: 1220,
});
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'saving',
completed: 0,
total: null,
});
return {
mediaId: 21,
mediaTitle: 'ONE PIECE',
entryCount: 4000,
fromCache: false,
updatedAt: 1000,
};
},
buildMergedDictionary: async () => ({
zipPath: '/tmp/merged.zip',
revision: 'rev-21',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 4000,
}),
getYomitanDictionaryInfo: async () =>
importedRevision
? [{ title: 'SubMiner Character Dictionary', revision: importedRevision }]
: [],
importYomitanDictionary: async () => {
importedRevision = 'rev-21';
return true;
},
deleteYomitanDictionary: async () => true,
upsertYomitanDictionarySettings: async () => true,
now: () => clock,
onSyncStatus: (event) => {
events.push({ phase: event.phase, message: event.message });
},
});
await runtime.runSyncNow();
assert.deepEqual(
events.filter((event) => event.phase === 'generating').map((event) => event.message),
[
'Generating character dictionary for ONE PIECE...',
'Generating character dictionary for ONE PIECE (page 12, 50 characters)...',
'Generating character dictionary for ONE PIECE (image 1/1220)...',
'Generating character dictionary for ONE PIECE (image 240/1220, ~10s left)...',
'Generating character dictionary for ONE PIECE (name 800/1220 · 8s)...',
'Generating character dictionary for ONE PIECE (saving snapshot · 8s)...',
],
);
});
test('auto sync keeps the generating clock ticking when a stage stalls', async () => {
const userDataPath = makeTempDir();
const events: Array<{ phase: string; message: string }> = [];
const scheduled: Array<() => void> = [];
const snapshotDeferred = createDeferred<{
mediaId: number;
mediaTitle: string;
entryCount: number;
fromCache: boolean;
updatedAt: number;
}>();
let clock = 1000;
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath,
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async (_targetPath, progress) => {
progress?.onGenerating?.({ mediaId: 21, mediaTitle: 'ONE PIECE' });
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'images',
completed: 240,
total: 1220,
});
return await snapshotDeferred.promise;
},
buildMergedDictionary: async () => ({
zipPath: '/tmp/merged.zip',
revision: 'rev-21',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 4000,
}),
getYomitanDictionaryInfo: async () => [],
importYomitanDictionary: async () => true,
deleteYomitanDictionary: async () => true,
upsertYomitanDictionarySettings: async () => true,
now: () => clock,
schedule: (fn) => {
scheduled.push(fn);
return 0 as unknown as ReturnType<typeof setTimeout>;
},
clearSchedule: () => undefined,
onSyncStatus: (event) => {
events.push({ phase: event.phase, message: event.message });
},
});
const syncPromise = runtime.runSyncNow();
await waitUntil(() => scheduled.length > 0, 'the generating heartbeat');
// No further progress arrives: only the clock moves.
clock += 95_000;
scheduled.at(-1)!();
assert.deepEqual(events.at(-1), {
phase: 'generating',
message: 'Generating character dictionary for ONE PIECE (image 240/1220 · 1m 35s)...',
});
snapshotDeferred.resolve({
mediaId: 21,
mediaTitle: 'ONE PIECE',
entryCount: 4000,
fromCache: false,
updatedAt: 1000,
});
await syncPromise;
assert.equal(events.at(-1)?.phase, 'ready');
});
test('auto sync rebuilds instead of importing a cached merged ZIP with a mismatched revision', async () => {
const userDataPath = makeTempDir();
const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
fs.mkdirSync(dictionariesDir, { recursive: true });
const statePath = path.join(dictionariesDir, 'auto-sync-state.json');
fs.writeFileSync(
statePath,
JSON.stringify({
activeMediaIds: ['7 - Frieren'],
mergedRevision: 'rev-7',
mergedDictionaryTitle: 'SubMiner Character Dictionary',
}),
'utf8',
);
// Left over from an interrupted run: the archive on disk is not the revision state recorded.
buildDictionaryZip(
path.join(dictionariesDir, 'merged.zip'),
'SubMiner Character Dictionary',
'Character names',
'rev-stale',
[{ term: 'フリーレン', reading: 'フリーレン', role: 'main', glossary: [] } as never],
[],
);
const mergedBuilds: number[][] = [];
const imports: string[] = [];
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath,
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async () => ({
mediaId: 7,
mediaTitle: 'Frieren',
entryCount: 100,
fromCache: true,
updatedAt: 1000,
}),
buildMergedDictionary: async (mediaIds) => {
mergedBuilds.push([...mediaIds]);
return {
zipPath: '/tmp/rebuilt-merged.zip',
revision: 'rev-7',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 100,
};
},
// Yomitan does not have the dictionary, so the sync has to import despite the cached state.
getYomitanDictionaryInfo: async () => [],
importYomitanDictionary: async (zipPath) => {
imports.push(zipPath);
return true;
},
deleteYomitanDictionary: async () => true,
upsertYomitanDictionarySettings: async () => true,
now: () => 1000,
});
await runtime.runSyncNow();
assert.deepEqual(mergedBuilds, [[7]]);
assert.deepEqual(imports, ['/tmp/rebuilt-merged.zip']);
});
@@ -1,13 +1,29 @@
import * as fs from 'fs';
import * as path from 'path';
import { ensureDir } from '../../shared/fs-utils';
import {
buildBuildingMessage,
buildCheckingMessage,
buildFailedMessage,
buildGeneratingMessage,
buildImportingMessage,
buildReadyMessage,
formatCharacterDictionaryProgressDetail,
joinGeneratingDetail,
} from './character-dictionary-auto-sync-messages';
import { readDictionaryZipRevision } from '../character-dictionary-runtime/zip';
import type { AnilistCharacterDictionaryProfileScope } from '../../types';
import type {
CharacterDictionarySnapshotProgressCallbacks,
CharacterDictionarySnapshotResult,
CharacterDictionarySnapshotStageProgress,
MergedCharacterDictionaryBuildResult,
} from '../character-dictionary-runtime';
const DEFAULT_IMPORT_TIMEOUT_BASE_MS = 120_000;
const IMPORT_TIMEOUT_MS_PER_MB = 6_000;
const IMPORT_TIMEOUT_MAX_MS = 1_800_000;
type AutoSyncMediaEntry = {
mediaId: number;
label: string;
@@ -72,7 +88,15 @@ export interface CharacterDictionaryAutoSyncRuntimeDeps {
now: () => number;
schedule?: (fn: () => void, delayMs: number) => ReturnType<typeof setTimeout>;
clearSchedule?: (timer: ReturnType<typeof setTimeout>) => void;
/** Budget for the quick Yomitan queries (dictionary info, settings upsert). */
operationTimeoutMs?: number;
/**
* Base budget for the slow Yomitan mutations (delete + import). The effective budget grows with
* the merged ZIP size, because importing a large dictionary can take several minutes.
*/
dictionaryImportTimeoutBaseMs?: number;
heartbeatMs?: number;
progressThrottleMs?: number;
logInfo?: (message: string) => void;
logWarn?: (message: string) => void;
onSyncStatus?: (event: CharacterDictionaryAutoSyncStatusEvent) => void;
@@ -345,37 +369,6 @@ function sameMembership(left: number[], right: number[]): boolean {
return arraysEqual(leftSorted, rightSorted);
}
function buildSyncingMessage(mediaTitle: string): string {
return `Updating character dictionary for ${mediaTitle}...`;
}
function buildCheckingMessage(mediaTitle: string): string {
return `Checking character dictionary for ${mediaTitle}...`;
}
function buildGeneratingMessage(mediaTitle: string): string {
return `Generating character dictionary for ${mediaTitle}...`;
}
function buildImportingMessage(mediaTitle: string): string {
return `Importing character dictionary for ${mediaTitle}...`;
}
function buildBuildingMessage(mediaTitle: string): string {
return `Building character dictionary for ${mediaTitle}...`;
}
function buildReadyMessage(mediaTitle: string): string {
return `Character dictionary ready for ${mediaTitle}`;
}
function buildFailedMessage(mediaTitle: string | null, errorMessage: string): string {
if (mediaTitle) {
return `Character dictionary sync failed for ${mediaTitle}: ${errorMessage}`;
}
return `Character dictionary sync failed: ${errorMessage}`;
}
export function createCharacterDictionaryAutoSyncRuntimeService(
deps: CharacterDictionaryAutoSyncRuntimeDeps,
): {
@@ -389,21 +382,31 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
const clearSchedule = deps.clearSchedule ?? ((timer) => clearTimeout(timer));
const debounceMs = 800;
const operationTimeoutMs = Math.max(1, Math.floor(deps.operationTimeoutMs ?? 7_000));
const dictionaryImportTimeoutBaseMs = Math.max(
1,
Math.floor(deps.dictionaryImportTimeoutBaseMs ?? DEFAULT_IMPORT_TIMEOUT_BASE_MS),
);
const heartbeatMs = Math.max(1, Math.floor(deps.heartbeatMs ?? 5_000));
const progressThrottleMs = Math.max(0, Math.floor(deps.progressThrottleMs ?? 1_000));
let debounceTimer: ReturnType<typeof setTimeout> | null = null;
let syncInFlight = false;
let runQueued = false;
let activeCurrentMediaId: number | null = null;
const withOperationTimeout = async <T>(label: string, promise: Promise<T>): Promise<T> => {
const withTimeout = async <T>(
label: string,
promise: Promise<T>,
timeoutMs: number,
): Promise<T> => {
let timer: ReturnType<typeof setTimeout> | null = null;
try {
return await Promise.race([
promise,
new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => {
reject(new Error(`${label} timed out after ${operationTimeoutMs}ms`));
}, operationTimeoutMs);
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
}, timeoutMs);
}),
]);
} finally {
@@ -413,6 +416,57 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
}
};
const withOperationTimeout = <T>(label: string, promise: Promise<T>): Promise<T> =>
withTimeout(label, promise, operationTimeoutMs);
/**
* Importing a merged dictionary means Yomitan writing every term and image into IndexedDB, which
* scales with the ZIP: a single-season dictionary lands in seconds, One Piece takes minutes. Size
* the budget off the archive instead of failing a healthy import on a flat deadline.
*/
const resolveImportTimeoutMs = (zipPath: string | null): number => {
let bytes = 0;
if (zipPath) {
try {
bytes = fs.statSync(zipPath).size;
} catch {
bytes = 0;
}
}
const sizeAllowanceMs = (bytes / (1024 * 1024)) * IMPORT_TIMEOUT_MS_PER_MB;
return Math.min(
IMPORT_TIMEOUT_MAX_MS,
Math.max(
dictionaryImportTimeoutBaseMs,
Math.round(dictionaryImportTimeoutBaseMs + sizeAllowanceMs),
),
);
};
/** Keeps the persistent notification ticking so a multi-minute import never looks hung. */
const withHeartbeat = async <T>(
run: () => Promise<T>,
onTick: (elapsedMs: number) => void,
): Promise<T> => {
const startedAt = deps.now();
let stopped = false;
let timer: ReturnType<typeof setTimeout> | null = null;
const tick = (): void => {
if (stopped) return;
onTick(deps.now() - startedAt);
timer = schedule(tick, heartbeatMs);
};
timer = schedule(tick, heartbeatMs);
try {
return await run();
} finally {
stopped = true;
if (timer !== null) {
clearSchedule(timer);
}
}
};
const runSyncOnce = async (): Promise<void> => {
const config = deps.getConfig();
if (!config.enabled) {
@@ -422,33 +476,112 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
let currentMediaId: number | undefined;
let currentMediaTitle: string | null = null;
let lastProgressAt = Number.NEGATIVE_INFINITY;
let lastProgressStage: CharacterDictionarySnapshotStageProgress['stage'] | null = null;
let generating: {
mediaId: number;
mediaTitle: string;
startedAt: number;
detail: string | null;
} | null = null;
let imageRate: { startedAt: number; startCompleted: number } | null = null;
const emitGeneratingStatus = (): void => {
if (!generating) {
return;
}
deps.onSyncStatus?.({
phase: 'generating',
mediaId: generating.mediaId,
mediaTitle: generating.mediaTitle,
message: buildGeneratingMessage(
generating.mediaTitle,
joinGeneratingDetail(generating.detail, deps.now() - generating.startedAt),
),
});
};
// Image downloads are serial and evenly paced, so the observed rate predicts the tail well.
const estimateRemainingImageMs = (
progress: CharacterDictionarySnapshotStageProgress,
nowMs: number,
): number | null => {
if (progress.stage !== 'images' || progress.total === null || imageRate === null) {
return null;
}
const done = progress.completed - imageRate.startCompleted;
const elapsedMs = nowMs - imageRate.startedAt;
const remaining = progress.total - progress.completed;
if (done <= 0 || elapsedMs <= 0 || remaining <= 0) {
return null;
}
return Math.round((elapsedMs / done) * remaining);
};
try {
deps.logInfo?.('[dictionary:auto-sync] syncing current anime snapshot');
const snapshot = await deps.getOrCreateCurrentSnapshot(undefined, {
onChecking: ({ mediaId, mediaTitle }) => {
currentMediaId = mediaId;
currentMediaTitle = mediaTitle;
activeCurrentMediaId = mediaId;
deps.onSyncStatus?.({
phase: 'checking',
mediaId,
mediaTitle,
message: buildCheckingMessage(mediaTitle),
});
},
onGenerating: ({ mediaId, mediaTitle }) => {
currentMediaId = mediaId;
currentMediaTitle = mediaTitle;
activeCurrentMediaId = mediaId;
deps.onSyncStatus?.({
phase: 'generating',
mediaId,
mediaTitle,
message: buildGeneratingMessage(mediaTitle),
});
},
});
const snapshot = await withHeartbeat(
() =>
deps.getOrCreateCurrentSnapshot(undefined, {
onChecking: ({ mediaId, mediaTitle }) => {
currentMediaId = mediaId;
currentMediaTitle = mediaTitle;
activeCurrentMediaId = mediaId;
deps.onSyncStatus?.({
phase: 'checking',
mediaId,
mediaTitle,
message: buildCheckingMessage(mediaTitle),
});
},
onGenerating: ({ mediaId, mediaTitle }) => {
currentMediaId = mediaId;
currentMediaTitle = mediaTitle;
activeCurrentMediaId = mediaId;
lastProgressAt = Number.NEGATIVE_INFINITY;
lastProgressStage = null;
imageRate = null;
generating = { mediaId, mediaTitle, startedAt: deps.now(), detail: null };
emitGeneratingStatus();
},
// Long-running work (AniList character pages, then one image download per character
// and voice actor, then MeCab name splits) reports counts so the notification shows
// movement instead of a frozen "Generating..." for the minutes a large series takes.
onGenerateProgress: (progress) => {
const nowMs = deps.now();
if (!generating) {
generating = {
mediaId: progress.mediaId,
mediaTitle: progress.mediaTitle,
startedAt: nowMs,
detail: null,
};
}
if (progress.stage === 'images' && imageRate === null) {
imageRate = { startedAt: nowMs, startCompleted: progress.completed };
}
// Stage changes and the last item of a stage always report; the throttle only thins
// out the run of identical-looking updates in between.
const isFinal = progress.total !== null && progress.completed >= progress.total;
const isStageChange = progress.stage !== lastProgressStage;
if (!isFinal && !isStageChange && nowMs - lastProgressAt < progressThrottleMs) {
return;
}
lastProgressAt = nowMs;
lastProgressStage = progress.stage;
generating.mediaId = progress.mediaId;
generating.mediaTitle = progress.mediaTitle;
generating.detail = formatCharacterDictionaryProgressDetail(
progress,
estimateRemainingImageMs(progress, nowMs),
);
emitGeneratingStatus();
},
}),
// Ticks even when a step stalls, so the message keeps moving while the counts do not.
() => emitGeneratingStatus(),
);
generating = null;
currentMediaId = snapshot.mediaId;
currentMediaTitle = snapshot.mediaTitle;
activeCurrentMediaId = snapshot.mediaId;
@@ -531,33 +664,61 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
mediaTitle: snapshot.mediaTitle,
message: buildImportingMessage(snapshot.mediaTitle),
});
if (existing !== null) {
await withOperationTimeout(
`deleteYomitanDictionary(${dictionaryTitle})`,
deps.deleteYomitanDictionary(dictionaryTitle),
);
}
if (merged === null) {
const existingMergedZipPath = path.join(dictionariesDir, 'merged.zip');
if (fs.existsSync(existingMergedZipPath)) {
merged = {
zipPath: existingMergedZipPath,
revision,
dictionaryTitle,
entryCount: snapshot.entryCount,
};
} else {
merged = await deps.buildMergedDictionary(nextActiveMediaIdValues);
}
}
deps.logInfo?.(`[dictionary:auto-sync] importing merged dictionary: ${merged.zipPath}`);
const imported = await withOperationTimeout(
`importYomitanDictionary(${path.basename(merged.zipPath)})`,
deps.importYomitanDictionary(merged.zipPath),
await withHeartbeat(
async () => {
const importTimeoutMs = resolveImportTimeoutMs(
merged?.zipPath ?? path.join(dictionariesDir, 'merged.zip'),
);
if (existing !== null) {
await withTimeout(
`deleteYomitanDictionary(${dictionaryTitle})`,
deps.deleteYomitanDictionary(dictionaryTitle),
importTimeoutMs,
);
}
if (merged === null) {
// The cached archive only stands in for the recorded state when its own index.json
// agrees. A stale or half-written ZIP would be imported under the wrong revision,
// and every later sync would then see a revision mismatch and re-import it.
const existingMergedZipPath = path.join(dictionariesDir, 'merged.zip');
const existingMergedRevision = readDictionaryZipRevision(existingMergedZipPath);
if (existingMergedRevision === revision) {
merged = {
zipPath: existingMergedZipPath,
revision,
dictionaryTitle,
entryCount: snapshot.entryCount,
};
} else {
deps.logInfo?.(
`[dictionary:auto-sync] cached merged ZIP unusable (revision ${existingMergedRevision ?? 'unreadable'}, expected ${revision}); rebuilding`,
);
merged = await deps.buildMergedDictionary(nextActiveMediaIdValues);
}
}
const mergedZipPath = merged.zipPath;
const mergedImportTimeoutMs = resolveImportTimeoutMs(mergedZipPath);
deps.logInfo?.(
`[dictionary:auto-sync] importing merged dictionary: ${mergedZipPath} (timeout ${mergedImportTimeoutMs}ms)`,
);
const imported = await withTimeout(
`importYomitanDictionary(${path.basename(mergedZipPath)})`,
deps.importYomitanDictionary(mergedZipPath),
mergedImportTimeoutMs,
);
if (!imported) {
throw new Error(`Failed to import dictionary ZIP: ${merged.zipPath}`);
}
},
(elapsedMs) => {
deps.onSyncStatus?.({
phase: 'importing',
mediaId: snapshot.mediaId,
mediaTitle: snapshot.mediaTitle,
message: buildImportingMessage(snapshot.mediaTitle, elapsedMs),
});
},
);
if (!imported) {
throw new Error(`Failed to import dictionary ZIP: ${merged.zipPath}`);
}
changed = true;
}
@@ -47,6 +47,29 @@ WINDOW=44040194
assert.deepEqual(reader.getCursorScreenPoint({ x: 877, y: 718 }), { x: 1700, y: 1050 });
});
test('createLinuxX11CursorPointReader converts physical X11 coordinates to Electron DIP', async () => {
const convertedPoints: Array<{ x: number; y: number }> = [];
const reader = createLinuxX11CursorPointReader({
env: { DISPLAY: ':1' },
platform: 'linux',
runCommand: async () => `X=1424
Y=697
SCREEN=0
WINDOW=44040194
`,
screenToDipPoint: (point) => {
convertedPoints.push(point);
return { x: 1139, y: 557 };
},
});
reader.refresh();
await new Promise((resolve) => setImmediate(resolve));
assert.deepEqual(reader.getCursorScreenPoint({ x: 0, y: 0 }), { x: 1139, y: 557 });
assert.deepEqual(convertedPoints, [{ x: 1424, y: 697 }]);
});
test('createLinuxX11CursorPointReader does not spawn off X11 Linux', () => {
const calls: string[] = [];
const reader = createLinuxX11CursorPointReader({
+3 -1
View File
@@ -36,11 +36,13 @@ export function createLinuxX11CursorPointReader(options?: {
now?: () => number;
platform?: NodeJS.Platform;
runCommand?: CommandRunner;
screenToDipPoint?: (point: PointerPoint) => PointerPoint;
}) {
const env = options?.env ?? process.env;
const now = options?.now ?? (() => Date.now());
const platform = options?.platform ?? process.platform;
const runCommand = options?.runCommand ?? execFileUtf8;
const screenToDipPoint = options?.screenToDipPoint ?? ((point: PointerPoint) => point);
let latest: { point: PointerPoint; updatedAtMs: number } | null = null;
let inFlight = false;
let retryAfterMs = 0;
@@ -63,7 +65,7 @@ export function createLinuxX11CursorPointReader(options?: {
retryAfterMs = now() + COMMAND_FAILURE_RETRY_DELAY_MS;
return;
}
latest = { point, updatedAtMs: now() };
latest = { point: screenToDipPoint(point), updatedAtMs: now() };
retryAfterMs = 0;
})
.catch(() => {
+6 -2
View File
@@ -10,7 +10,7 @@ export interface StartupOsdSequencerDeps {
getNotificationType?: () => NotificationType | undefined;
showOsd: (message: string) => boolean | void;
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
showDesktopNotification?: (title: string, options: { body?: string }) => void;
showDesktopNotification?: (title: string, options: { body?: string; replaceId?: string }) => void;
}
interface StartupStatusNotificationOptions {
@@ -62,7 +62,11 @@ export function createStartupOsdSequencer(deps: StartupOsdSequencerDeps): {
shown = deps.showOsd(options.message) !== false || shown;
}
if (options.desktop !== false && shouldShowDesktop(type)) {
deps.showDesktopNotification?.('SubMiner', { body: options.message });
// Each startup lane keeps one live desktop notification instead of one per update.
deps.showDesktopNotification?.('SubMiner', {
body: options.message,
replaceId: options.id,
});
shown = true;
}
return shown;
@@ -289,7 +289,9 @@ export function createVisibleOverlayInteractionRuntime(deps: VisibleOverlayInter
if (initialArgs && isHeadlessInitialCommand(initialArgs)) {
return null;
}
return createWindowTrackerCore(override, targetMpvSocketPath);
return createWindowTrackerCore(override, targetMpvSocketPath, (point) =>
screen.screenToDipPoint(point),
);
}
function bindVisibleOverlayOwner(): void {
@@ -627,7 +629,9 @@ export function createVisibleOverlayInteractionRuntime(deps: VisibleOverlayInter
ensureWindowsVisibleOverlayForegroundPollLoop();
const linuxX11CursorPointReader = createLinuxX11CursorPointReader();
const linuxX11CursorPointReader = createLinuxX11CursorPointReader({
screenToDipPoint: (point) => screen.screenToDipPoint(point),
});
function getLinuxOverlayPointerMeasurement() {
const measurement = overlayContentMeasurementStore.getLatestByLayer('visible');
+18
View File
@@ -1004,6 +1004,24 @@ test('normalizeSubtitle collapses explicit line breaks when collapseLineBreaks i
);
});
test('normalizeSubtitle leaves already-decoded text alone', () => {
// Primary subtitle text is decoded from ASS once, upstream: by mpv for live lines and
// by the cue parser for prefetched ones. A brace that survives that is literal text.
assert.equal(normalizeSubtitle('本文{\\pos(1,2)'), '本文{\\pos(1,2)');
assert.equal(normalizeSubtitle(' 余白 ', false), ' 余白 ');
});
test('prepareSecondarySubtitleLines drops ASS vector drawing runs', () => {
assert.deepEqual(
prepareSecondarySubtitleLines(
'{\\an5\\pos(730,1042)\\p1\\blur1}m 20 0 b 10 0 0 10 0 20 b 0 31 10 40 20 40 {\\p0}',
),
[],
);
assert.deepEqual(prepareSecondarySubtitleLines('{\\p1}m 0 0 l 10 10{\\p0}本文'), ['本文']);
assert.deepEqual(prepareSecondarySubtitleLines('{\\pos(960,1068)\\bord3}位置指定'), ['位置指定']);
});
test('shouldRenderTokenizedSubtitle enables token rendering when tokens exist', () => {
assert.equal(shouldRenderTokenizedSubtitle(5), true);
assert.equal(shouldRenderTokenizedSubtitle(0), false);
+8 -11
View File
@@ -5,6 +5,7 @@ import type {
SubtitleData,
SubtitleRendererStyleConfig,
} from '../types';
import { assToPlainText, normalizePlainSubtitleText } from '../core/services/ass-text.js';
import type { RendererContext } from './context';
import { PRIMARY_SUB_VISIBLE_ON_YOMITAN_POPUP_CLASS } from './yomitan-popup.js';
@@ -42,17 +43,10 @@ function isWhitespaceOnly(value: string): boolean {
return value.trim().length === 0;
}
// Text reaching the overlay has already been decoded from ASS -- by mpv for live lines,
// by the cue parser for prefetched ones -- so this only settles line breaks.
export function normalizeSubtitle(text: string, trim = true, collapseLineBreaks = false): string {
if (!text) return '';
let normalized = text.replace(/\\N/g, '\n').replace(/\\n/g, '\n');
normalized = normalized.replace(/\{[^}]*\}/g, '');
if (collapseLineBreaks) {
normalized = normalized.replace(/\n/g, ' ');
normalized = normalized.replace(/\s+/g, ' ');
}
return trim ? normalized.trim() : normalized;
return normalizePlainSubtitleText(text, { trim, collapseLineBreaks });
}
const HEX_COLOR_PATTERN = /^#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{4}|[0-9a-fA-F]{6}|[0-9a-fA-F]{8})$/;
@@ -672,7 +666,10 @@ function isKaraokeLikeLineSet(lines: string[]): boolean {
}
export function prepareSecondarySubtitleLines(text: string): string[] {
const normalized = normalizeSubtitle(text, true, false);
// The one display-side ASS decode: secondary text also reaches the overlay from
// websocket clients that forward their source line untouched, so unlike the primary
// path it cannot assume mpv already decoded it.
const normalized = assToPlainText(text).trim();
if (!normalized) return [];
+5 -4
View File
@@ -99,9 +99,10 @@ test('applyX11EnvOverrides strips Wayland hints and pins session type to x11', (
assert.equal(result.XDG_SESSION_TYPE, 'x11');
});
test('MPV_X11_BACKEND_ARGS pins the GPU stack to X11', () => {
assert.deepEqual(
[...MPV_X11_BACKEND_ARGS],
['--vo=gpu', '--gpu-api=opengl', '--gpu-context=x11egl,x11'],
test('MPV_X11_BACKEND_ARGS pins the window context to X11 without overriding the renderer', () => {
assert.deepEqual([...MPV_X11_BACKEND_ARGS], ['--gpu-context=x11vk,x11egl,x11']);
assert.equal(
MPV_X11_BACKEND_ARGS.some((arg) => arg.startsWith('--vo=') || arg.startsWith('--gpu-api=')),
false,
);
});
+12 -6
View File
@@ -12,12 +12,18 @@
so the gate and the mpv backend args stay in one place.
*/
/** mpv args that pin the GPU/windowing stack to X11/XWayland (libGL via EGL on X11). */
export const MPV_X11_BACKEND_ARGS = [
'--vo=gpu',
'--gpu-api=opengl',
'--gpu-context=x11egl,x11',
] as const;
/**
* mpv args that pin the *windowing* stack to X11/XWayland, in Vulkan-then-OpenGL order.
* mpv walks the list and skips contexts that do not match the configured `--gpu-api`,
* so this works for both a Vulkan and an OpenGL config.
*
* Deliberately does NOT set `--vo`/`--gpu-api`: forcing `--vo=gpu --gpu-api=opengl` here
* used to drop configs off `vo=gpu-next` onto the legacy renderer, where user shaders
* written for gpu-next (e.g. ArtCNN, `//!COMPONENTS 4` LUMA hooks) abort mpv with
* `copy_image: Assertion '*offset + count < sizeof(dst)' failed` as soon as their
* upscale-only `//!WHEN` condition turns on, i.e. on the first fullscreen toggle.
*/
export const MPV_X11_BACKEND_ARGS = ['--gpu-context=x11vk,x11egl,x11'] as const;
export type LinuxDesktopEnv = {
xdgCurrentDesktop: string;
+139
View File
@@ -148,6 +148,145 @@ function createEndOfCentralDirectory(
return end;
}
const LOCAL_FILE_HEADER_SIGNATURE = 0x04034b50;
const LOCAL_FILE_HEADER_SIZE = 30;
const END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054b50;
const END_OF_CENTRAL_DIRECTORY_SIZE = 22;
const CENTRAL_FILE_HEADER_SIGNATURE = 0x02014b50;
const CENTRAL_FILE_HEADER_SIZE = 46;
// 65535 entries with names of a few dozen bytes stay far under this; the cap only stops a corrupt
// record length from asking for an allocation the size of the archive.
const MAX_CENTRAL_DIRECTORY_BYTES = 16 * 1024 * 1024;
/**
* Walks every declared central-directory record, checking each signature and keeping the
* variable-length name/extra/comment fields inside the directory. The walk has to land exactly on
* the end of the directory, so a record that was overwritten in place fails even though the file
* kept its size.
*/
function isCentralDirectoryIntact(
fd: number,
centralStart: number,
centralSize: number,
entryCount: number,
): boolean {
if (centralSize === 0 || centralSize > MAX_CENTRAL_DIRECTORY_BYTES) {
return false;
}
const central = Buffer.alloc(centralSize);
if (fs.readSync(fd, central, 0, centralSize, centralStart) !== centralSize) {
return false;
}
let cursor = 0;
for (let index = 0; index < entryCount; index += 1) {
if (cursor + CENTRAL_FILE_HEADER_SIZE > centralSize) {
return false;
}
if (central.readUInt32LE(cursor) !== CENTRAL_FILE_HEADER_SIGNATURE) {
return false;
}
const nameLength = central.readUInt16LE(cursor + 28);
const extraLength = central.readUInt16LE(cursor + 30);
const commentLength = central.readUInt16LE(cursor + 32);
cursor += CENTRAL_FILE_HEADER_SIZE + nameLength + extraLength + commentLength;
if (cursor > centralSize) {
return false;
}
}
return cursor === centralSize;
}
/**
* Start of the central directory, or null when the archive is not a complete one of ours. The
* end-of-central-directory record is written last, so finding an intact one is what separates a
* finished archive from a half-written one.
*/
function readCentralDirectoryStart(fd: number, fileSize: number): number | null {
if (fileSize < END_OF_CENTRAL_DIRECTORY_SIZE) {
return null;
}
const end = Buffer.alloc(END_OF_CENTRAL_DIRECTORY_SIZE);
const endOffset = fileSize - END_OF_CENTRAL_DIRECTORY_SIZE;
if (fs.readSync(fd, end, 0, end.length, endOffset) !== end.length) {
return null;
}
// writeStoredZip never writes an archive comment, so the record is exactly the last 22 bytes.
if (end.readUInt32LE(0) !== END_OF_CENTRAL_DIRECTORY_SIGNATURE || end.readUInt16LE(20) !== 0) {
return null;
}
const entryCount = end.readUInt16LE(10);
if (entryCount === 0) {
return null;
}
const centralSize = end.readUInt32LE(12);
const centralStart = end.readUInt32LE(16);
if (centralStart + centralSize !== endOffset) {
return null;
}
if (!isCentralDirectoryIntact(fd, centralStart, centralSize, entryCount)) {
return null;
}
return centralStart;
}
/**
* Reads the first entry of an archive written by {@link writeStoredZip}: every entry is stored
* uncompressed with no extra field and no data descriptor, so the leading local header is enough.
* Returns null for anything it does not recognize, so callers treat a corrupt, truncated, or
* foreign archive the same as a missing one.
*/
export function readStoredZipFirstFile(zipPath: string): StoredZipFile | null {
let fd: number;
try {
fd = fs.openSync(zipPath, 'r');
} catch {
return null;
}
try {
const fileSize = fs.fstatSync(fd).size;
const centralStart = readCentralDirectoryStart(fd, fileSize);
if (centralStart === null) {
return null;
}
const header = Buffer.alloc(LOCAL_FILE_HEADER_SIZE);
if (fs.readSync(fd, header, 0, header.length, 0) !== header.length) {
return null;
}
if (header.readUInt32LE(0) !== LOCAL_FILE_HEADER_SIGNATURE) {
return null;
}
// Compression method 0 (stored) is the only thing writeStoredZip emits.
if (header.readUInt16LE(8) !== 0) {
return null;
}
const entrySize = header.readUInt32LE(18);
const nameLength = header.readUInt16LE(26);
const extraLength = header.readUInt16LE(28);
const dataOffset = LOCAL_FILE_HEADER_SIZE + nameLength + extraLength;
if (dataOffset + entrySize > centralStart) {
return null;
}
const name = Buffer.alloc(nameLength);
if (
nameLength > 0 &&
fs.readSync(fd, name, 0, nameLength, LOCAL_FILE_HEADER_SIZE) !== nameLength
) {
return null;
}
const data = Buffer.alloc(entrySize);
if (entrySize > 0 && fs.readSync(fd, data, 0, entrySize, dataOffset) !== entrySize) {
return null;
}
return { name: name.toString('utf8'), data };
} catch {
return null;
} finally {
fs.closeSync(fd);
}
}
function writeBuffer(fd: number, buffer: Buffer): void {
let written = 0;
while (written < buffer.length) {
+6 -13
View File
@@ -16,6 +16,8 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { normalizePlainSubtitleText } from './core/services/ass-text';
interface TimingEntry {
startTime: number;
endTime: number;
@@ -191,23 +193,14 @@ export class SubtitleTimingTracker {
return costs[shorter.length] || 0;
}
// Both sides take text mpv has already decoded from ASS; only whitespace differs
// between the lookup key (single line) and the display form (line breaks kept).
private normalizeText(text: string): string {
return text
.replace(/\\N/g, ' ')
.replace(/\\n/g, ' ')
.replace(/\n/g, ' ')
.replace(/{[^}]*}/g, '')
.replace(/\s+/g, ' ')
.trim();
return normalizePlainSubtitleText(text, { collapseLineBreaks: true });
}
private prepareDisplayText(text: string): string {
// Convert ASS/SSA newlines to real newlines, strip tags
return text
.replace(/\\N/g, '\n')
.replace(/\\n/g, '\n')
.replace(/{[^}]*}/g, '')
.trim();
return normalizePlainSubtitleText(text);
}
private startCleanup(): void {
+7 -1
View File
@@ -20,6 +20,7 @@ import { BaseWindowTracker } from './base-tracker';
import { HyprlandWindowTracker } from './hyprland-tracker';
import { SwayWindowTracker } from './sway-tracker';
import { X11WindowTracker } from './x11-tracker';
import type { ScreenToDipPoint } from './x11-tracker';
import { MacOSWindowTracker } from './macos-tracker';
import { WindowsWindowTracker } from './windows-tracker';
import { createLogger } from '../logger';
@@ -51,6 +52,7 @@ function normalizeCompositor(value: string): Compositor | null {
export function createWindowTracker(
override?: string | null,
targetMpvSocketPath?: string | null,
screenToDipPoint?: ScreenToDipPoint,
): BaseWindowTracker | null {
let compositor = detectCompositor();
@@ -70,7 +72,11 @@ export function createWindowTracker(
case 'sway':
return new SwayWindowTracker(targetMpvSocketPath?.trim() || undefined);
case 'x11':
return new X11WindowTracker(targetMpvSocketPath?.trim() || undefined);
return new X11WindowTracker(
targetMpvSocketPath?.trim() || undefined,
undefined,
screenToDipPoint,
);
case 'macos':
return new MacOSWindowTracker(targetMpvSocketPath?.trim() || undefined);
case 'windows':
+41
View File
@@ -82,6 +82,47 @@ Height: 360`;
});
});
test('X11WindowTracker converts both physical rectangle corners to Electron DIP', async () => {
const convertedPoints: Array<{ x: number; y: number }> = [];
const tracker = new X11WindowTracker(
undefined,
async (command, args) => {
if (command === 'xdotool' && args[0] === 'search') {
return '123';
}
if (command === 'xdotool' && args[0] === 'getactivewindow') {
return '123';
}
if (command === 'xwininfo') {
return `Absolute upper-left X: 2000
Absolute upper-left Y: 125
Width: 1000
Height: 750`;
}
return '';
},
(point) => {
convertedPoints.push(point);
if (point.x === 2000) return { x: 1600, y: 100 };
return { x: 2400, y: 700 };
},
);
(tracker as unknown as { pollGeometry: () => void }).pollGeometry();
await new Promise((resolve) => setTimeout(resolve, 0));
assert.deepEqual(convertedPoints, [
{ x: 2000, y: 125 },
{ x: 3000, y: 875 },
]);
assert.deepEqual(tracker.getGeometry(), {
x: 1600,
y: 100,
width: 800,
height: 600,
});
});
test('X11WindowTracker updates target focus from active X11 window', async () => {
let activeWindowId = '999';
const tracker = new X11WindowTracker(undefined, async (command, args) => {
+26 -3
View File
@@ -20,6 +20,9 @@ import { execFile } from 'child_process';
import { BaseWindowTracker } from './base-tracker';
type CommandRunner = (command: string, args: string[]) => Promise<string>;
export type ScreenToDipPoint = (point: { x: number; y: number }) => { x: number; y: number };
const preservePoint: ScreenToDipPoint = (point) => point;
function execFileUtf8(command: string, args: string[]): Promise<string> {
return new Promise((resolve, reject) => {
@@ -87,16 +90,22 @@ export class X11WindowTracker extends BaseWindowTracker {
private pollInterval: ReturnType<typeof setInterval> | null = null;
private readonly targetMpvSocketPath: string | null;
private readonly runCommand: CommandRunner;
private readonly screenToDipPoint: ScreenToDipPoint;
private targetWindowId: string | null = null;
private targetWindowPid: number | null = null;
private pollInFlight = false;
private currentPollIntervalMs = 750;
private readonly stablePollIntervalMs = 250;
constructor(targetMpvSocketPath?: string, runCommand: CommandRunner = execFileUtf8) {
constructor(
targetMpvSocketPath?: string,
runCommand: CommandRunner = execFileUtf8,
screenToDipPoint: ScreenToDipPoint = preservePoint,
) {
super();
this.targetMpvSocketPath = targetMpvSocketPath?.trim() || null;
this.runCommand = runCommand;
this.screenToDipPoint = screenToDipPoint;
}
start(): void {
@@ -196,11 +205,25 @@ export class X11WindowTracker extends BaseWindowTracker {
this.targetWindowPid = targetPid;
const winInfo = await this.runCommand('xwininfo', ['-id', windowId]);
const geometry = parseX11WindowGeometry(winInfo);
if (!geometry) {
const physicalGeometry = parseX11WindowGeometry(winInfo);
if (!physicalGeometry) {
this.updateGeometry(null);
return;
}
const topLeft = this.screenToDipPoint({
x: physicalGeometry.x,
y: physicalGeometry.y,
});
const bottomRight = this.screenToDipPoint({
x: physicalGeometry.x + physicalGeometry.width,
y: physicalGeometry.y + physicalGeometry.height,
});
const geometry = {
x: topLeft.x,
y: topLeft.y,
width: bottomRight.x - topLeft.x,
height: bottomRight.y - topLeft.y,
};
const focused = await this.isWindowActive(windowId, targetPid);
this.updateGeometry(geometry, focused);