Compare commits

..

17 Commits

Author SHA1 Message Date
sudacode 4a7c9b04f1 fix(stats): enforce cleanup mode exclusivity and JSON requests
- Reject conflicting explicit cleanup modes
- Require application/json for duplicate-line maintenance requests
2026-08-12 00:49:32 -07:00
sudacode 9fd2dfd7a0 fix(cli): preserve stats cleanup lookback validation
- Parse the full inline lookback-days value before validation
2026-08-11 22:31:56 -07:00
sudacode 046a87a826 fix(stats): harden duplicate-line cleanup and tracking
- Reject malformed requests and sub-day cleanup windows
- Reset deduplication across media and subtitle-track changes
- Handle karaoke bursts ending with a long hold frame
2026-08-11 22:20:50 -07:00
sudacode fa4c734e30 docs(stats): document duplicate-line cleanup flag rules and forwarded app flags 2026-08-11 18:41:29 -07:00
sudacode b64e1264dc test(stats): click backdrop, not disabled button, in cleanup close test
- Assert the Close button is disabled while apply is in flight
- Click the always-enabled backdrop instead, since that's the path that actually reaches the close guard
2026-08-11 00:16:46 -07:00
sudacode c2561f8c4c fix(stats): fix duplicate-line cleanup reload tracking
- Track reload-owed state separately from the displayed result so a later scan or window change can't erase it before the reload runs
- Refuse to close mid-apply so the pending reload isn't dropped
- Add tests covering these edge cases
2026-08-11 00:04:18 -07:00
sudacode 039aed79c3 fix(stats): fix duplicate-line cleanup edge cases
- Drain the full write queue before scanning, not just one batch, so pending burst rows aren't missed
- Floor lookback-days before the positivity check so a sub-day value no longer collapses to a zero-day window
- Let the parsed cue list override the streaming dedup heuristic wherever it covers a line
- Reject combining --lifetime with --duplicate-lines and non-positive --lookback-days values
- Widen the burst frame bound to catch heavier typesetting while still sparing longer-spaced runs
- Keep the cleanup modal open until the user closes it so they can read the result before it unmounts
2026-08-10 23:50:48 -07:00
sudacode 684ab9eaff fix(stats): stop counting duplicate typeset subtitle lines
- Collapse animation-burst subtitle lines (karaoke OPs, animated signs) at ingest time using the same dedup rules the subtitle sidebar already applies, so repeated frames no longer flood "Top Repeated Words"
- Add retroactive cleanup for stats already affected: a "Duplicates" scanner/cleaner in the Vocabulary tab and `subminer stats cleanup --duplicate-lines` (`--dry-run`, `--lookback-days`) on the CLI
- Only subtitle lines and the vocabulary counts they feed are touched; watch time and lines-seen totals are left as recorded
2026-08-10 23:18:43 -07:00
sudacode 7b0fbdf254 fix(subtitles): collapse duplicate ASS events and decode text once (#186) 2026-08-10 22:21:44 -07:00
sudacode 2fefc83e3f fix(playback): stop forcing legacy OpenGL renderer on X11 mpv backend (#188) 2026-08-06 23:52:35 -07:00
sudacode dbdf578c68 perf(tokenizer): single-pass Yomitan scan with cross-line caching and prefetch fixes (#185) 2026-08-06 21:44:09 -07:00
sudacode 441ecf3c04 feat(overlay): add in-app changelog modal (#187) 2026-08-05 22:19:13 -07:00
sudacode a0dde4ee3e chore(release): v0.19.2 2026-08-04 18:51:15 -07:00
sudacode fe4dacc1e7 fix(overlay): show plain subtitle line immediately on tokenization cache miss (#184) 2026-08-04 01:55:52 -07:00
sudacode b08cd0db35 fix(streaming): keep subtitle tokenization prefetch warm for full episodes (#183) 2026-08-03 21:22:18 -07:00
sudacode bffb1c5982 fix(logging): surface subtitle processing debug/warn logs (#182) 2026-08-03 20:44:39 -07:00
sudacode 5b8848518a feat(subsync): add reference and target subtitle track picker (#181) 2026-08-03 01:00:14 -07:00
173 changed files with 13474 additions and 2354 deletions
+18
View File
@@ -1,5 +1,23 @@
# Changelog
## v0.19.2 (2026-08-04)
### Changed
- Subsync: The sync modal now lets you choose both the reference subtitle (correct timing) and the out-of-sync subtitle to retime, for both alass and ffsubsync. alass can also use the loaded video's audio as a reference for local files. Retiming the secondary track now reloads the result into the secondary slot instead of overwriting the primary subtitle.
### Fixed
- Streaming Subtitle Tokenization: Jellyfin streams now seed subtitle tokenization directly from the downloaded subtitle file instead of relying on an mpv event that could be missed, and prefetching now runs to the end of the file and clears between episodes. The tokenization cache was raised from 256 to 2500 lines, and parsed cues are no longer lost when the active subtitle track briefly can't be resolved (e.g. switching to an embedded track). Together these prevent episodes from falling back to slow, line-by-line tokenization during playback.
- Overlay: Subtitle lines now appear immediately at their cue time even on a tokenization cache miss, upgrading in place once tokens and annotations are ready, instead of waiting on a line still being processed. A failed tokenization is no longer cached as plain text, so repeated lines get another chance at annotations.
- Background Logging: Background startup now respects the configured logging level when no explicit log level is passed.
<details>
<summary>Internal changes</summary>
### Internal
- Patched three high-severity dependency advisories (`undici`, `brace-expansion`, `fast-uri`).
</details>
## v0.19.1 (2026-08-01)
### Added
+7 -9
View File
@@ -26,7 +26,7 @@
"eslint": "^10.8.0",
"prettier": "^3.8.1",
"typescript": "^5.9.3",
"undici": "7.28.0",
"undici": "7.29.0",
},
},
},
@@ -36,8 +36,9 @@
"overrides": {
"@xmldom/xmldom": "0.8.13",
"app-builder-lib": "26.15.3",
"brace-expansion": "5.0.8",
"brace-expansion": "5.0.9",
"electron-builder-squirrel-windows": "26.15.3",
"fast-uri": "3.1.5",
"form-data": "4.0.6",
"ip-address": "10.2.0",
"js-yaml": "4.3.0",
@@ -46,6 +47,7 @@
"picomatch": "4.0.4",
"tar": "7.5.21",
"tmp": "0.2.7",
"undici": "7.29.0",
},
"packages": {
"@discordjs/collection": ["@discordjs/collection@2.1.1", "", {}, "sha512-LiSusze9Tc7qF03sLCujF5iZp7K+vRNEDBZ86FT9aQAv3vxMLihUvKvpsCWiQ2DJq1tVckopKm1rxomgNUc9hg=="],
@@ -266,7 +268,7 @@
"boolean": ["boolean@3.2.0", "", {}, "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw=="],
"brace-expansion": ["brace-expansion@5.0.8", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg=="],
"brace-expansion": ["brace-expansion@5.0.9", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg=="],
"buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="],
@@ -404,7 +406,7 @@
"fast-levenshtein": ["fast-levenshtein@2.0.6", "", {}, "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="],
"fast-uri": ["fast-uri@3.1.4", "", {}, "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw=="],
"fast-uri": ["fast-uri@3.1.5", "", {}, "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw=="],
"fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="],
@@ -714,7 +716,7 @@
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
"undici": ["undici@7.28.0", "", {}, "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA=="],
"undici": ["undici@7.29.0", "", {}, "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw=="],
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
@@ -754,8 +756,6 @@
"@discordjs/rest/discord-api-types": ["discord-api-types@0.38.43", "", {}, "sha512-sSoBf/nK6m7BGtw65mi+QBuvEWaHE8MMziFLqWL+gT6ME/BLg34dRSVKS3Husx40uU06bvxUc3/X+D9Y6/zAbw=="],
"@discordjs/rest/undici": ["undici@6.27.0", "", {}, "sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg=="],
"@discordjs/util/discord-api-types": ["discord-api-types@0.38.43", "", {}, "sha512-sSoBf/nK6m7BGtw65mi+QBuvEWaHE8MMziFLqWL+gT6ME/BLg34dRSVKS3Husx40uU06bvxUc3/X+D9Y6/zAbw=="],
"@electron/asar/commander": ["commander@5.1.0", "", {}, "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg=="],
@@ -808,8 +808,6 @@
"node-gyp/env-paths": ["env-paths@2.2.1", "", {}, "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A=="],
"node-gyp/undici": ["undici@6.27.0", "", {}, "sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg=="],
"node-gyp/which": ["which@6.0.1", "", { "dependencies": { "isexe": "^4.0.0" }, "bin": { "node-which": "bin/which.js" } }, "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg=="],
"pkijs/@noble/hashes": ["@noble/hashes@1.4.0", "", {}, "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg=="],
+7
View File
@@ -0,0 +1,7 @@
type: added
area: overlay
- Added an in-app changelog modal, opened from the tray ("View Changelog") or the "What's New" button on the update-available notification, which now stays on screen so "Update" is still reachable after reading the notes. It renders inside the player bounds when a video is playing and in its own window otherwise, the same as the help modal.
- The changelog is fetched from the newest published release, so release notes for versions newer than the installed build are visible; a failed download falls back to the changelog bundled with the install and says so in the modal.
- Versions are foldable: the current `0.x` line is expanded and older lines are folded, matching the docs-site changelog. A badge marks the installed version and newer versions are tagged "New".
- Keyboard: `J`/`K` or arrows move between versions, `Enter` folds/unfolds, `R` refetches, `Esc` closes.
+5
View File
@@ -0,0 +1,5 @@
type: fixed
area: stats
- Typeset subtitles no longer flood the stats. Karaoke openings and animated signs are authored as one subtitle event per animation frame, and immersion tracking counted every frame, which was enough to put an OP lyric at the top of "Top Repeated Words" for good. Lines are now collapsed on the way in using the same rules the subtitle sidebar already applies: matching parsed timings record exactly the cues the sidebar shows, while shifted, changing, or unparsed sources use a strict fallback where identical, contiguous, sub-0.1s lines stop counting after a few frames. Ordinary repeated dialogue and rewatches are unaffected.
- Added a cleanup for stats already affected. The Vocabulary tab has a **Duplicates** button that scans a chosen window (7 days through all time), shows the bursts it found and the word and kanji counts they added, and collapses each run to one line once confirmed. `subminer stats cleanup --duplicate-lines` does the same from the terminal, with `--dry-run` and `--lookback-days <n>`. Only subtitle lines and the vocabulary counts they feed are touched; watch time and lines-seen totals are left as recorded.
+5
View File
@@ -0,0 +1,5 @@
type: fixed
area: subtitles
- Heavily typeset ASS scripts (karaoke OP/ED, sign work) no longer fill the subtitle sidebar with garbage. Vector drawing runs (`\p1``\p0`) are no longer shown as subtitle text (e.g. `m 20 0 b 10 0 0 10 0 20 …`), and duplicate events in a parsed subtitle file collapse into one cue: identical text over an identical span (layered "shadow" copies), and per-frame animation bursts. An ASS burst has to prove itself with authoring evidence — a temporal tag (`\t`, `\move`, karaoke timing), an animated `Effect` column, or override values that change from event to event — plus one shared style and actor, so three rapid `えっ` reactions from three characters, or a sign repeated with the same static `\clip`, stay separate. SRT and VTT carry no such metadata, so there the run has to be at least five contiguous events all shorter than 0.1s, which is where ASS-to-SRT conversion leaves karaoke frames.
- Subtitle text is now decoded from ASS exactly once, where it enters the app, and matches how mpv renders the same line (including `\N`, `\n`, `\h`, unclosed `{`, and the fact that `\{` is not an escape). Renderer, timing tracker, tokenizer and the tokenization cache take that decoded text as-is instead of each re-deriving it, so one authored line can no longer produce two different cache keys, and a cue that normalizes to nothing is no longer stored as subtitle text or cached under an empty key.
@@ -0,0 +1,20 @@
type: changed
area: subtitles
- Subtitle tokenization no longer runs a duplicate full `parseText` pass per line: the termsFind scanner walk is now the only tokenizer and emits its own hoverable filler runs for unmatched text (parseText is kept only as an error fallback). This roughly halves the dictionary work per line.
- The Yomitan scanning helpers are now installed once per parser window (`__subminerYomitanScan`) instead of re-shipping and re-parsing a ~500-line script for every subtitle line; each line only evaluates a tiny call.
- termsFind lookups are cached across subtitle lines in a window-persistent LRU keyed by substring, so repeated particles and verb forms stop costing backend round trips. The cache invalidates on dictionary/settings changes and window reloads.
- The scanner walk now skips lookups at punctuation and whitespace positions (latin letters and digits still look up, e.g. Tシャツ). The shrinking-window retry ladder keeps following the consumed lengths the backend reports, and only blind guesses (windows the backend consumed whole, which tell it nothing) are capped at four per position. A line that hits that cap escalates to a single `parseText` for the whole line, so a hard line still resolves to dictionary tokens instead of an unparsed run, without letting the ladder run to one lookup per window length.
- Tokenizer runtime dependencies are built once instead of per line, fixing a JLPT lookup cache that never hit (it was keyed on a per-call closure identity and leaked a Map per line) and a `which mecab` availability check that re-ran synchronously on every line when MeCab is absent.
- Subtitle changes no longer restart the prefetch run per line (which discarded in-flight tokenization work); prefetch now only pauses for the live line and restarts on real seeks, cache invalidation, or option changes. Prefetch also stays paused for the whole time the subtitle processing controller is working on the line, including the provisional raw emit that precedes tokenization, so it never competes with the on-screen line for the parser window. The pause is released when the controller reports it has settled, which also covers the lines that finish without an emit (a suppressed duplicate or a failed tokenization) and used to leave prefetching paused indefinitely.
- Added per-stage debug timings (`scanMs`, `mecabMs`, `frequencyMs`, `annotateMs`) to the subtitle tokenization pipeline log.
- Fixed a reading that stopped covering its surface when an unmatched kana run extended the preceding token (for example a trailing る on 待ち合わせ), which silently disabled the known-word reading fallback for those tokens.
- Subtitle prefetching no longer stays paused for the rest of a cue when the same subtitle text is reported twice and there is nothing to tokenize. This covers the startup and overlay priming paths as well as ordinary subtitle changes.
- Character name and image lookups are now refreshed centrally whenever a character dictionary sync changes its content, so a newly added name can no longer be skipped by a stale candidate list.
- A subtitle that was on screen when its annotations were invalidated (by mining a card, for example) is now re-annotated instead of staying plain for the rest of the line.
- Character name annotations no longer cost a dictionary lookup at every position in a line. The scanner now knows which name forms the current title's character dictionary actually contains and only checks where one can start, which removes the whole overhead of having the character dictionary enabled (measured: 21 lookups per line down to 10, the same as with it disabled). Titles with no cached character data keep the previous exhaustive scan, so a missing snapshot costs speed rather than a missing name.
- The cross-line termsFind cache is now bounded by the number of retained dictionary entries as well as by key count, so a run of lookups that each carry hundreds of entries with full glossaries cannot grow the parser window's memory without limit. The budget is re-checked when a lookup resolves, so a single oversized response is dropped rather than parked in the cache and reused.
- The unnamed-mob disambiguator filter (Girl A / Girl B) now only drops a single letter or digit split off a name, instead of every one-character term: a name that is genuinely one character keeps its terms whatever the script (𠮷, あ, 별 김, ア・ベ). The character dictionary and the scanner's name pre-pass also share one Han code-point table now, so a name the dictionary accepts is a name the scanner will look for.
- A character name written in halfwidth katakana takes part in the greedy name pre-pass again, so a longer generic word can no longer swallow the start of it, and it now carries a reading (it used to come out blank, which disables known-word matching and frequency lookups for the token). Voiced halfwidth kana compose properly, so ガク reads ガク rather than ガク, and kana normalization folds halfwidth throughout so those tokens compare equal to the same word written fullwidth. Because the fold makes halfwidth text indexable, the character-name prefilter now judges halfwidth spellings like any other, and a position only bypasses it when an unfoldable voiced mark sits inside the lookup window. That covers a name that starts on a kanji and turns halfwidth later (山ガク), and one stretched out with emphatic characters in between (山ーーーーーーガク).
- Dictionary-entry classification (source dictionaries, character-dictionary media ids) is memoized per entry object for as long as the entry is cached, instead of being recomputed for every headword comparison and every retry window.
- Autoplay priming no longer broadcasts the plain subtitle twice: it tells the processing controller the line has already been painted, so the controller goes straight to the annotated payload.
@@ -0,0 +1,4 @@
type: fixed
area: playback
- XWayland/X11 mode (`--backend=x11`, or the automatic fallback on non-Hyprland/Sway Wayland sessions) no longer forces mpv onto `--vo=gpu --gpu-api=opengl`. It now only pins the window context (`--gpu-context=x11vk,x11egl,x11`), so a `vo=gpu-next` config keeps its renderer, API, and user shaders. Forcing the legacy OpenGL renderer crashed mpv on the first fullscreen toggle for anyone using a gpu-next user shader that emits a 4-component LUMA hook (ArtCNN and friends), which asserts in mpv's old renderer (`copy_image: *offset + count < sizeof(dst)`) as soon as the shader's upscale-only condition turns on.
+2 -2
View File
@@ -75,8 +75,8 @@ src/
renderer/ # Overlay renderer (modularized UI/runtime)
handlers/ # Keyboard/mouse/gamepad interaction modules
modals/ # Modal flows (Jimaku, Kiku, subsync, runtime options, session help,
# character dictionary, playlist browser, subtitle sidebar,
# YouTube track picker, controller config/debug/select)
# changelog, character dictionary, playlist browser, subtitle
# sidebar, YouTube track picker, controller config/debug/select)
positioning/ # Subtitle position controller (drag-to-reposition)
settings/ # Settings window UI (model, controls, markup)
types/ # Domain type modules (anki, config, integrations, ...)
+18
View File
@@ -1,5 +1,23 @@
# Changelog
## v0.19.2 (2026-08-04)
**Changed**
- Subsync: The sync modal now lets you choose both the reference subtitle (correct timing) and the out-of-sync subtitle to retime, for both alass and ffsubsync. alass can also use the loaded video's audio as a reference for local files. Retiming the secondary track now reloads the result into the secondary slot instead of overwriting the primary subtitle.
**Fixed**
- Streaming Subtitle Tokenization: Jellyfin streams now seed subtitle tokenization directly from the downloaded subtitle file instead of relying on an mpv event that could be missed, and prefetching now runs to the end of the file and clears between episodes. The tokenization cache was raised from 256 to 2500 lines, and parsed cues are no longer lost when the active subtitle track briefly can't be resolved (e.g. switching to an embedded track). Together these prevent episodes from falling back to slow, line-by-line tokenization during playback.
- Overlay: Subtitle lines now appear immediately at their cue time even on a tokenization cache miss, upgrading in place once tokens and annotations are ready, instead of waiting on a line still being processed. A failed tokenization is no longer cached as plain text, so repeated lines get another chance at annotations.
- Background Logging: Background startup now respects the configured logging level when no explicit log level is passed.
<details>
<summary>Internal changes</summary>
**Internal**
- Patched three high-severity dependency advisories (`undici`, `brace-expansion`, `fast-uri`).
</details>
## v0.19.1 (2026-08-01)
**Added**
+2 -2
View File
@@ -1196,9 +1196,9 @@ See [TsukiHime Integration](/tsukihime-integration) for the modal workflow, lang
### Subtitle Sync
Sync the active subtitle track from the overlay picker using `alass` or `ffsubsync`. Both are **optional external tools** that must be installed separately and available on your `PATH` (or configured via the path options below).
Sync a subtitle track from the overlay picker using `alass` or `ffsubsync`. The picker lets you choose which track gets retimed (the active primary track by default) and, for alass, which reference it is aligned against (the secondary subtitle track by default). Both are **optional external tools** that must be installed separately and available on your `PATH` (or configured via the path options below).
- [`alass`](https://github.com/kaegi/alass) - fast, audio-independent sync using a secondary subtitle as reference
- [`alass`](https://github.com/kaegi/alass) - fast, audio-independent sync using another subtitle as reference; it can also take the local video file as reference (alass extracts the audio itself)
- [`ffsubsync`](https://github.com/smacke/ffsubsync) - audio-based sync using the video file as reference
```json
+27 -1
View File
@@ -34,7 +34,7 @@ The same immersion data powers the stats dashboard.
- In-app overlay: focus the visible overlay, then press the key from `stats.toggleKey` (default: `` ` `` / `Backquote`).
- Launcher command: run `subminer stats` to start the local stats server on demand (it also opens the dashboard in your browser when `stats.autoOpenBrowser` is enabled; the default is `false`).
- Background server: run `subminer stats -b` to start or reuse a dedicated background stats daemon without keeping the launcher attached, and `subminer stats -s` to stop that daemon.
- Maintenance commands: run `subminer stats cleanup` or `subminer stats cleanup -v` to backfill/repair vocabulary metadata (`headword`, `reading`, POS) and purge stale or excluded rows from `imm_words` on demand; `subminer stats cleanup -l` repairs lifetime summary tables. `subminer stats rebuild` and `subminer stats backfill` rebuild or backfill rollup data.
- Maintenance commands: run `subminer stats cleanup` or `subminer stats cleanup -v` to backfill/repair vocabulary metadata (`headword`, `reading`, POS) and purge stale or excluded rows from `imm_words` on demand; `subminer stats cleanup -l` repairs lifetime summary tables; `subminer stats cleanup --duplicate-lines` collapses repeated lines left behind by typeset subtitles (see [Repeated Line Cleanup](#repeated-line-cleanup)). `subminer stats rebuild` and `subminer stats backfill` rebuild or backfill rollup data.
- Browser page: open `http://127.0.0.1:6969` directly if the local stats server is already running.
### Dashboard Tabs
@@ -125,6 +125,32 @@ Secondary subtitle text (typically English translations) is stored alongside pri
The Vocabulary tab toolbar includes an **Exclusions** button for hiding words from all vocabulary views. Excluded words are stored in the immersion database, with older browser localStorage exclusions imported on first load after upgrade. They can be managed (restored or cleared) from the exclusion modal. Exclusions affect stat cards, charts, the frequency rank table, and the word list.
### Repeated Line Cleanup
Karaoke openings and animated signs are authored as one subtitle event per animation frame, all carrying the same text. Playback reports every one of those frames, so a single OP lyric could be recorded hundreds of times and dominate "Top Repeated Words".
Recording now collapses those runs as they happen, matching what the subtitle sidebar shows:
- When the active subtitle source has been parsed, its cue list has already had duplicate events and animation bursts merged. A line landing inside a surviving cue but after that cue's start is a frame the sidebar merged away, and is not recorded.
- When no parsed cue covers the live timing, including while a subtitle source is changing or shifted, the strict metadata-free rule applies: a run of identical, contiguous lines each shorter than 0.1s stops being recorded after a few frames. Ordinary repeated dialogue, and lines held for a normal beat, always record.
For stats recorded before this, the Vocabulary tab toolbar has a **Duplicates** button:
- Pick how far back to look (7 days, 30 days, 90 days, 1 year, or all time). A narrower window does less work and keeps older history untouched.
- **Scan** reports the bursts found, the lines they added, and the word and kanji counts they inflated, without writing anything.
- **Clean Up** applies exactly what the scan reported: each run collapses to its first line (extended to cover the run), and the removed lines' word and kanji occurrences are subtracted from the vocabulary aggregates.
The same thing runs from the terminal:
```bash
subminer stats cleanup --duplicate-lines --dry-run --lookback-days 30
subminer stats cleanup --duplicate-lines --lookback-days 30
```
`--duplicate-lines` (short: `-d`) picks the cleanup mode, so it cannot be combined with `--vocab` or `--lifetime`, and `--dry-run` and `--lookback-days <days>` only apply to it. Omitting `--lookback-days` scans all history; the value must be at least one day.
Runs never cross a session boundary, so rewatching an episode keeps both watches. Session telemetry (watch time, lines seen, tokens seen) and the rollups derived from it are left as recorded: they are cumulative samples taken during playback, and cannot be recomputed for sessions whose raw rows have since been pruned.
## Retention Defaults
By default, SubMiner keeps all retention tables and raw data (`0` means keep all) while continuing daily/monthly rollup maintenance:
+1
View File
@@ -151,6 +151,7 @@ subminer stats -b # start background stats daemon
| `subminer stats` | Start the stats server (opens the dashboard when `stats.autoOpenBrowser` is on) |
| `subminer stats -b` / `-s` | Start/reuse or stop the background stats daemon |
| `subminer stats cleanup` | Backfill vocabulary metadata and prune stale rows (`-v` vocab, `-l` lifetime summaries) |
| `subminer stats cleanup -d` | Collapse repeated lines from typeset subs (`--dry-run`, `--lookback-days <n>`) |
| `subminer stats rebuild` / `backfill` | Rebuild or backfill rollup data |
| `subminer doctor` | Dependency + config + socket diagnostics (`--refresh-known-words` refreshes the known-word cache) |
| `subminer settings` | Open the SubMiner settings window |
+6 -3
View File
@@ -161,10 +161,13 @@ If your subtitle file is out of sync with the audio, SubMiner can resynchronize
1. Open the subsync modal from the overlay.
2. Select the sync engine (alass or ffsubsync).
3. For alass, select a reference subtitle track from the video.
4. SubMiner runs the sync and reloads the corrected subtitle.
3. For alass, pick the **reference** - the subtitle with correct timing. This defaults to the secondary subtitle track. The loaded video file can also be used as the reference (alass extracts the audio itself), but it is never the default.
4. Pick the **out-of-sync subtitle** - the track that gets retimed. This defaults to the active primary subtitle track and applies to both engines.
5. SubMiner runs the sync and reloads the corrected subtitle into the slot the out-of-sync track came from: retiming the secondary track keeps it secondary and leaves the primary track selected.
For remote streams, including Jellyfin playback, the modal only offers alass. Jellyfin subtitle URLs are cached as temporary subtitle files so alass can read them, but the video stream is not downloaded. ffsubsync needs direct access to the local media file and is unavailable for stream URLs.
The reference and the out-of-sync subtitle must be different tracks; the reference list hides whichever track is selected as the target.
For remote streams, including Jellyfin playback, the modal only offers alass with a subtitle reference. Jellyfin subtitle URLs are cached as temporary subtitle files so alass can read them, but the video stream is not downloaded. ffsubsync and the video-file reference need direct access to the local media file and are unavailable for stream URLs.
When you mine a sentence card from the stats dashboard, SubMiner can also use `alass` automatically to align a local English sidecar against the matching local Japanese sidecar before filling the card translation field. The source subtitle files are not modified; SubMiner writes a temporary retimed copy and reuses it while the stats server is running.
+3 -3
View File
@@ -227,7 +227,7 @@ Install ffsubsync or configure the path:
If subtitle sync fails (the error message is prefixed with the engine name):
- Ensure the reference subtitle track exists in the video (alass requires a source track).
- Ensure a reference is selected (alass needs either a second subtitle track or the local video file, and it cannot be the same track that is being retimed).
- Check that `ffmpeg` is available (used to extract the internal subtitle track).
- Try running the sync tool manually to see detailed error output.
- ffsubsync requires local files and cannot handle remote media streams (e.g., streaming URLs).
@@ -406,7 +406,7 @@ On any Wayland session that is not Hyprland or Sway (KDE Plasma, GNOME, and othe
SubMiner handles this automatically:
- It launches its own window under XWayland (it sets `--ozone-platform-hint=x11`).
- Every mpv it launches (via the `subminer` launcher, Jellyfin, or YouTube) is pinned to XWayland too - Wayland environment hints are stripped and an X11 GPU context (`--gpu-context=x11egl,x11`) is applied.
- Every mpv it launches (via the `subminer` launcher, Jellyfin, or YouTube) is pinned to XWayland too - Wayland environment hints are stripped and an X11 GPU context (`--gpu-context=x11vk,x11egl,x11`) is applied. Only the window context is overridden; your `vo`/`gpu-api` and user shaders are left alone.
- While mpv is windowed, the overlay is a managed X11 window owned by the tracked mpv window (`WM_TRANSIENT_FOR`), so it stays above mpv while other foreground X11/Xwayland apps can still cover both windows.
- While tracked mpv is fullscreen, SubMiner swaps the visible overlay to a focusable-false X11 override-redirect window. That path can stay above the active fullscreen mpv window without requiring a KDE/KWin-specific rule, and SubMiner hides/releases it when mpv is no longer the active X11/Xwayland window.
- The visible overlay is shown inactive on Linux, so normal hover should not steal keyboard focus from mpv.
@@ -420,7 +420,7 @@ Requirements: `xdotool`, `xprop`, and `xwininfo` must be installed. SubMiner use
This almost always means mpv came up as a **native Wayland** window that the XWayland overlay cannot cover. It happens when mpv is launched **manually** (your own command), because SubMiner can only force XWayland on the mpv processes it launches itself. Fix it one of these ways:
- Launch playback through SubMiner (the `subminer` launcher or the tray), which forces XWayland for you, or
- Force XWayland in your own mpv invocation, e.g. `mpv --gpu-context=x11egl …`, or launch with `WAYLAND_DISPLAY= mpv …`, or set `gpu-context=x11egl` in your `mpv.conf`.
- Force XWayland in your own mpv invocation, e.g. `mpv --gpu-context=x11vk,x11egl,x11 …`, or launch with `WAYLAND_DISPLAY= mpv …`, or set `gpu-context=x11vk` (Vulkan) / `gpu-context=x11egl` (OpenGL) in your `mpv.conf`.
To confirm mpv is on XWayland, `xdotool search --class mpv` should return a window id (a native Wayland mpv returns nothing).
+10 -2
View File
@@ -95,6 +95,8 @@ subminer texthooker # Texthooker-only mode (-o also opens the brow
subminer stats -b # Start/reuse the background stats daemon
subminer stats -s # Stop the background stats daemon
subminer stats cleanup # Backfill vocabulary metadata, prune stale rows
subminer stats cleanup -d --dry-run # Preview cleanup of repeated typeset subtitle lines
subminer stats cleanup -d --lookback-days 30 # Clean only lines recorded in the last 30 days
subminer stats rebuild # Rebuild rollup data
subminer doctor --refresh-known-words # Refresh the known-word cache
subminer logs -e # Export a sanitized log ZIP and print its path
@@ -107,6 +109,8 @@ subminer app --stop # Stop the background app
subminer --version # Print the launcher's version
```
`stats cleanup` runs one mode per invocation: `-v`/`--vocab` (the default), `-l`/`--lifetime`, or `-d`/`--duplicate-lines`; explicitly selected modes cannot be combined. `--dry-run` and `--lookback-days <days>` apply to `--duplicate-lines` only and are rejected without it; `--lookback-days` must be at least one day, and leaving it off scans all history.
Jellyfin, cross-machine sync, and character-dictionary commands have their own sections: [Jellyfin](/jellyfin-integration), [Sync Between Machines](/launcher-script#sync-between-machines), and [Character Dictionary](/character-dictionary).
</details>
@@ -137,7 +141,7 @@ SubMiner.AppImage --start --log-level debug # Verbose logging without dev mode
SubMiner.AppImage --help # Show all options
```
The remaining flags are internal or scripting-only surfaces: the `--jellyfin-*` family (login, library listing, item playback, cast announce), `--sync-cli` (the app's headless sync entrypoint that `subminer sync` proxies to), `--dictionary-candidates` / `--dictionary-select`, and `--playback-feedback <text>`. Run `SubMiner.AppImage --help` for the complete list. The previous `--open-animetosho` flag is still accepted as a deprecated alias for `--open-tsukihime`.
The remaining flags are internal or scripting-only surfaces: the `--jellyfin-*` family (login, library listing, item playback, cast announce), `--sync-cli` (the app's headless sync entrypoint that `subminer sync` proxies to), the `--stats-cleanup-*` family that `subminer stats cleanup` forwards (`--stats-cleanup-vocab`, `--stats-cleanup-lifetime`, `--stats-cleanup-duplicate-lines`, and its `--stats-cleanup-dry-run` / `--stats-cleanup-lookback-days <days>` modifiers), `--dictionary-candidates` / `--dictionary-select`, and `--playback-feedback <text>`. Run `SubMiner.AppImage --help` for the complete list. The previous `--open-animetosho` flag is still accepted as a deprecated alias for `--open-tsukihime`.
</details>
@@ -145,11 +149,13 @@ The tray menu includes `Export Logs`, which creates the same sanitized local-dat
Once Jellyfin is configured, the tray menu includes `Jellyfin Discovery` for starting or stopping cast discovery in the current app session without changing config.
The tray menu also includes `View Changelog`, which opens the in-app changelog modal. It fetches the changelog from the newest published release, so you see release notes for versions newer than the one you run; if the download fails it falls back to the changelog bundled with your install and says so. Versions in the current `0.x` line are expanded by default and older lines are folded, matching this site's [Changelog](/changelog). A badge marks the version you have installed, and newer versions are tagged `New`. The same modal opens from the `What's New` button on the update-available overlay notification.
### Logging and App Mode
- `--log-level` controls logger verbosity.
- `--dev` and `--debug` are app/dev-mode switches; they are not log-level aliases.
- `--background` defaults to quieter logging (`warn`) unless `--log-level` is set.
- `--background` starts at the default quieter logging level (`warn`), then follows `logging.level` after config loads. An explicit `--log-level` remains the override.
- `--background` launched from a terminal detaches and returns the prompt; stop it with tray Quit or `SubMiner.AppImage --stop` (`SubMiner.exe --stop` on Windows).
- Linux desktop launcher starts SubMiner with `--background` by default (via electron-builder `linux.executableArgs`).
- On Hyprland and other Wayland compositors, the tray icon appears only when your panel provides a StatusNotifier/AppIndicator tray host.
@@ -368,6 +374,8 @@ Press `V` to cycle the primary SubMiner subtitle bar through hidden → visible
`Ctrl/Cmd+/` opens the session help modal with the current overlay and mpv keybindings. The same help view is also available through the `y-h` chord in mpv.
The changelog modal (tray > `View Changelog`) works the same way: it renders over mpv when a video is playing and in its own window otherwise. Use `J`/`K` or the arrow keys to move between versions, `Enter` to fold or unfold one, `R` to refetch, and `Esc` to close.
Hovering over subtitle text pauses mpv by default; leaving resumes it. Yomitan popups also pause playback by default. Set `subtitleStyle.autoPauseVideoOnHover: false` or `subtitleStyle.autoPauseVideoOnYomitanPopup: false` to disable either behavior.
### Drag-and-Drop
+3 -1
View File
@@ -64,7 +64,7 @@ Use the basic subtitle websocket when you only need the current subtitle line as
- **Client auth:** none
- **Reconnects:** client-managed
When a client connects, SubMiner immediately sends the latest subtitle payload if one is available. After that, it pushes a new message each time the current subtitle changes.
When a client connects, SubMiner immediately sends the latest subtitle payload if one is available. After that, it pushes a new message each time the current subtitle changes. Annotation-only upgrades do not repeat the same line on this basic stream.
#### Message shape
@@ -96,6 +96,8 @@ Use the annotation websocket for custom clients that want the same structured to
In practice, if you are building a new client, prefer `annotationWebsocket` unless you specifically need compatibility with an existing `websocket` consumer.
On a tokenization cache miss, this stream first sends the cue as plain text with an empty `tokens` array, then sends the annotated replacement when tokenization finishes. Treat each message as the complete current state, replacing the previous payload.
#### Message shape
```json
@@ -64,18 +64,23 @@ External subtitle files only (SRT, VTT, ASS). Embedded subtitle tracks are out o
A cue parser extracts both timing and text content from subtitle files for prefetching.
**Parsed cue structure:**
```typescript
interface SubtitleCue {
startTime: number; // seconds
endTime: number; // seconds
text: string; // raw subtitle text
startTime: number; // seconds
endTime: number; // seconds
text: string; // plain text, decoded from the source format
}
```
**Supported formats:**
- SRT/VTT: Regex-based parsing of timing lines + text content between timing blocks.
- ASS: Parse `[Events]` section, extract `Dialogue:` lines, split on the first 9 commas only (ASS v4+ has 10 fields; the last field is Text which can itself contain commas). Strip ASS override tags (`{\...}`) from the text before storing.
ASS text fields contain inline override tags like `{\b1}`, `{\an8}`, `{\fad(200,300)}`. The cue parser strips these during extraction so the tokenizer receives clean text.
- ASS: Parse `[Events]` section, extract `Dialogue:` lines, read the field order from the `Format:` row, and take everything after the Text field index as the text (Text can itself contain commas).
**ASS decoding.** The parser is where ASS text is decoded, once, via `assToPlainText()` in `src/core/services/ass-text.ts`. That decoder mirrors mpv's `ass_to_plaintext` so a cue read from a file reads identically to the same line arriving live on `sub-text`: `{...}` override blocks are markup, `\pN … \p0` vector drawing runs are dropped rather than shown as text, `\N`/`\n`/`\h` are the only escapes (`\{`, `\}` and `\\` are not), and an unclosed `{` is rendered verbatim. Every layer downstream — renderer, timing tracker, tokenizer, tokenization cache keys — receives plain text and uses `normalizePlainSubtitleText()` for whitespace only, so nothing decodes the same string twice and one authored line always maps to one cache key.
**Duplicate collapsing.** Typeset scripts emit one `Dialogue:` event per animation frame, plus layered copies of the same line. The parser collapses identical text over an identical span unconditionally, and collapses contiguous same-text runs of at least three events when the run looks like an animation. For ASS that means shared style and actor plus authoring evidence: a temporal tag (`\t`, `\move`, `\k`/`\kf`/`\ko`/`\K`, or anything wrapped in `\t(...)`), an animated `Effect` column (`Karaoke`, `Banner`, `Scroll`), or override values that change across the run. Static tags shared by every event (`\pos`, an identical `\clip`) are not evidence. SRT/VTT carry no such metadata, so there collapsing needs at least five contiguous events all under 0.1s — the frame timing left behind by ASS-to-SRT conversion. The parser keeps this authoring metadata (style, actor, layer, `Effect`, parsed override commands, source order) private; `parseSubtitleCues()` returns only `SubtitleCue`.
#### Prefetch Service Lifecycle
@@ -153,6 +158,7 @@ tokens (already have frequencyRank values from parser-level applyFrequencyRanks)
### Dependency Analysis
All annotations either depend on MeCab POS data or benefit from running after it:
- **Known word marking:** Needs base tokens (surface/headword). No POS dependency, but no reason to run separately.
- **Frequency filtering:** Uses `pos1Exclusions` and `pos2Exclusions` to clear frequency ranks on excluded tokens (particles, noise). Depends on MeCab POS data.
- **JLPT marking:** Uses `shouldIgnoreJlptForMecabPos1` to filter. Depends on MeCab POS data.
@@ -169,18 +175,14 @@ function annotateTokens(tokens, deps, options): MergedToken[] {
// Single pass: known word + frequency filtering + JLPT computed together
const annotated = tokens.map((token) => {
const isKnown = nPlusOneEnabled
? token.isKnown || computeIsKnown(token, deps)
: false;
const isKnown = nPlusOneEnabled ? token.isKnown || computeIsKnown(token, deps) : false;
// Filter frequency rank using POS exclusions (rank values already set at parser level)
const frequencyRank = frequencyEnabled
? filterFrequencyRank(token, pos1Exclusions, pos2Exclusions)
: undefined;
const jlptLevel = jlptEnabled
? computeJlptLevel(token, deps.getJlptLevel)
: undefined;
const jlptLevel = jlptEnabled ? computeJlptLevel(token, deps.getJlptLevel) : undefined;
return { ...token, isKnown, frequencyRank, jlptLevel };
});
@@ -221,6 +223,7 @@ Replace `document.createElement('span')` calls in the renderer with `templateSpa
### Current Behavior
In `renderWithTokens` (`subtitle-render.ts`), each render cycle:
1. Clears DOM with `innerHTML = ''`
2. Creates a `DocumentFragment`
3. Calls `document.createElement('span')` for each token (~10-15 per subtitle)
@@ -256,27 +259,30 @@ Full recycling (collecting old nodes, clearing attributes, reusing them) require
## Combined Impact Summary
| Scenario | Before | After | Improvement |
|----------|--------|-------|-------------|
| Normal playback (prefetch-warmed) | ~200-320ms | ~30-50ms | ~80-85% |
| Cache hit (repeated subtitle) | ~72ms | ~55-65ms | ~10-20% |
| Cache miss (immediate seek) | ~200-320ms | ~150-260ms | ~20-25% |
| Scenario | Before | After | Improvement |
| --------------------------------- | ---------- | ---------- | ----------- |
| Normal playback (prefetch-warmed) | ~200-320ms | ~30-50ms | ~80-85% |
| Cache hit (repeated subtitle) | ~72ms | ~55-65ms | ~10-20% |
| Cache miss (immediate seek) | ~200-320ms | ~150-260ms | ~20-25% |
---
## Files Summary
### New Files
- `src/core/services/subtitle-prefetch.ts`
- `src/core/services/subtitle-cue-parser.ts`
### Modified Files
- `src/core/services/subtitle-processing-controller.ts` (expose `preCacheTokenization`)
- `src/core/services/tokenizer/annotation-stage.ts` (batched single-pass)
- `src/renderer/subtitle-render.ts` (template cloneNode)
- `src/main.ts` (wire up prefetch service)
### Test Files
- New tests for subtitle cue parser (SRT, VTT, ASS formats)
- New tests for subtitle prefetch service (priority window, seek, pause/resume)
- Updated tests for annotation stage (same behavior, new implementation)
+34 -9
View File
@@ -3,7 +3,7 @@
# Subtitle Overlay Priming
Status: active
Last verified: 2026-06-14
Last verified: 2026-08-04
Owner: Kyle Yasuda
Read when: debugging subtitle state or blank Linux/X11 overlay windows when the visible overlay is shown or recreated
@@ -47,18 +47,43 @@ subtitles do not draw.
`emitSubtitle(payload)` and `refreshCurrentSubtitle(text)`, then prime secondary subtitles.
6. Tokenization cache hit: call `consumeCachedSubtitle(text)`, `onSubtitleChange(text)`, and
`emitSubtitle(cachedPayload)`, then prime secondary subtitles.
7. Cache miss: call `refreshCurrentSubtitle(text)` and let normal tokenization emit the final
payload.
7. Cache miss: call `refreshCurrentSubtitle(text)`. Normal processing emits a plain payload
synchronously, then replaces it with the tokenized payload when ready.
In `src/main.ts`, both `onSubtitleChange` and `refreshCurrentSubtitle` pause
`subtitlePrefetchService`, notify it with `onSeek(lastObservedTimePos)`, and then call the matching
`subtitleProcessingController` method. This gives the visible overlay priority over background
prefetch work and re-centers prefetch around the live playback time.
Both `onSubtitleChange` and `refreshCurrentSubtitle` pause `subtitlePrefetchService` and then call
the matching `subtitleProcessingController` method, giving the visible overlay priority over
background prefetch work. Prefetch is not re-centered here: restarting the run per line
(`onSeek`) discarded the in-flight tokenization every time the subtitle changed, so only real
seeks restart it (see `onTimePosUpdate` in `src/main.ts`).
On an uncached autoplay prime the raw payload is emitted here and reported to the controller with
`notePlainSubtitleEmitted`, so the controller skips its own plain emit for that line and the
overlay receives one plain payload followed by the annotated one.
The pause is released by the controller's `onProcessingSettled` callback, which fires once it has
no work left. Emits do not release it: the first emit for an uncached line is the plain payload
that precedes tokenization, and a run can finish without emitting at all (a suppressed duplicate,
a failed tokenization). Both controller methods return whether processing is now pending, and the
caller resumes immediately when it is not — a repeated subtitle schedules no work, so no settle is
coming and prefetching would otherwise idle for the rest of the cue.
## Live Cue Delivery
- A tokenization cache miss emits the plain cue synchronously. Tokenization remains serialized so
live work does not contend for Yomitan state.
- If a newer cue arrives while an older line is still tokenizing, the newer plain cue or empty
clear payload is emitted immediately. The older tokenization result is dropped before it can
replace the current cue.
- The current cue upgrades in place when its tokens and annotations are ready. This can reflow text
or character images, but cue visibility does not wait for that work.
## Emitted State
- `emitSubtitle(payload)` maps to `emitSubtitlePayload(payload)`, which sends the normal
annotated subtitle payload to overlay windows and subtitle websocket listeners.
- `emitSubtitle(payload)` maps to `emitSubtitlePayload(payload)`. Overlay windows and annotation
websocket listeners receive both the immediate plain cue and its later annotation upgrade.
- The basic subtitle websocket receives the immediate plain cue only. Because its serialized
payload discards annotations, the later upgrade would be an identical duplicate and is skipped
when text and cue timing match.
- Secondary priming reads mpv `secondary-sub-text`, stores it in
`mpvClient.currentSecondarySubText`, and broadcasts `secondary-subtitle:set` to overlay windows.
- If secondary `requestProperty` fails, the primary flow stays complete and only a debug line is
+9
View File
@@ -157,6 +157,15 @@ export async function runStatsCommand(
if (args.statsCleanupLifetime) {
forwarded.push('--stats-cleanup-lifetime');
}
if (args.statsCleanupDuplicateLines) {
forwarded.push('--stats-cleanup-duplicate-lines');
}
if (args.statsCleanupDryRun) {
forwarded.push('--stats-cleanup-dry-run');
}
if (args.statsCleanupLookbackDays) {
forwarded.push('--stats-cleanup-lookback-days', String(args.statsCleanupLookbackDays));
}
if (shouldForwardLogLevel(args.logLevel)) {
forwarded.push('--log-level', args.logLevel);
}
+12
View File
@@ -134,6 +134,9 @@ test('applyInvocationsToArgs maps config and jellyfin invocation state', () => {
statsCleanup: false,
statsCleanupVocab: false,
statsCleanupLifetime: false,
statsCleanupDuplicateLines: false,
statsCleanupDryRun: false,
statsCleanupLookbackDays: null,
statsLogLevel: null,
syncTriggered: false,
syncCliTokens: [],
@@ -185,6 +188,9 @@ test('applyInvocationsToArgs maps settings invocation to settings window', () =>
statsCleanup: false,
statsCleanupVocab: false,
statsCleanupLifetime: false,
statsCleanupDuplicateLines: false,
statsCleanupDryRun: false,
statsCleanupLookbackDays: null,
statsLogLevel: null,
syncTriggered: false,
syncCliTokens: [],
@@ -229,6 +235,9 @@ test('applyInvocationsToArgs fails when config invocation has no action', () =>
statsCleanup: false,
statsCleanupVocab: false,
statsCleanupLifetime: false,
statsCleanupDuplicateLines: false,
statsCleanupDryRun: false,
statsCleanupLookbackDays: null,
statsLogLevel: null,
syncTriggered: false,
syncCliTokens: [],
@@ -271,6 +280,9 @@ test('applyInvocationsToArgs maps texthooker browser-open request', () => {
statsCleanup: false,
statsCleanupVocab: false,
statsCleanupLifetime: false,
statsCleanupDuplicateLines: false,
statsCleanupDryRun: false,
statsCleanupLookbackDays: null,
statsLogLevel: null,
syncTriggered: false,
syncCliTokens: [],
+7
View File
@@ -162,6 +162,8 @@ export function createDefaultArgs(
statsCleanup: false,
statsCleanupVocab: false,
statsCleanupLifetime: false,
statsCleanupDuplicateLines: false,
statsCleanupDryRun: false,
doctor: false,
doctorRefreshKnownWords: false,
logsExport: false,
@@ -258,6 +260,11 @@ export function applyInvocationsToArgs(parsed: Args, invocations: CliInvocations
if (invocations.statsCleanup) parsed.statsCleanup = true;
if (invocations.statsCleanupVocab) parsed.statsCleanupVocab = true;
if (invocations.statsCleanupLifetime) parsed.statsCleanupLifetime = true;
if (invocations.statsCleanupDuplicateLines) parsed.statsCleanupDuplicateLines = true;
if (invocations.statsCleanupDryRun) parsed.statsCleanupDryRun = true;
if (invocations.statsCleanupLookbackDays !== null) {
parsed.statsCleanupLookbackDays = invocations.statsCleanupLookbackDays;
}
if (invocations.dictionaryTarget) {
parsed.dictionaryTarget = parseDictionaryTarget(invocations.dictionaryTarget);
} else if (
+47 -3
View File
@@ -37,6 +37,9 @@ export interface CliInvocations {
statsCleanup: boolean;
statsCleanupVocab: boolean;
statsCleanupLifetime: boolean;
statsCleanupDuplicateLines: boolean;
statsCleanupDryRun: boolean;
statsCleanupLookbackDays: number | null;
statsLogLevel: string | null;
syncTriggered: boolean;
syncCliTokens: string[];
@@ -53,6 +56,16 @@ export interface CliInvocations {
texthookerOpenBrowser: boolean;
}
/** `--lookback-days` narrows the duplicate-line cleanup; fractions are floored. */
function parseStatsLookbackDays(value: unknown): number | null {
if (typeof value !== 'string' && typeof value !== 'number') return null;
const days = Number(value);
if (!Number.isFinite(days) || days < 1) {
throw new Error('Stats --lookback-days must be at least one day.');
}
return Math.floor(days);
}
function applyRootOptions(program: Command): void {
program
.option(
@@ -169,6 +182,9 @@ export function parseCliPrograms(
let statsCleanup = false;
let statsCleanupVocab = false;
let statsCleanupLifetime = false;
let statsCleanupDuplicateLines = false;
let statsCleanupDryRun = false;
let statsCleanupLookbackDays: number | null = null;
let statsLogLevel: string | null = null;
let syncTriggered = false;
let syncCliTokens: string[] = [];
@@ -269,6 +285,9 @@ export function parseCliPrograms(
.option('-s, --stop', 'Stop the background stats server')
.option('-v, --vocab', 'Clean vocabulary rows in the stats database')
.option('-l, --lifetime', 'Rebuild lifetime summary rows from retained data')
.option('-d, --duplicate-lines', 'Collapse repeated subtitle lines from typeset animations')
.option('--dry-run', 'Report what a cleanup would remove without changing anything')
.option('--lookback-days <days>', 'Only clean lines recorded in the last N days')
.option('--log-level <level>', 'Log level')
.action((action: string | undefined, options: Record<string, unknown>) => {
statsTriggered = true;
@@ -289,13 +308,35 @@ export function parseCliPrograms(
if (normalizedAction && (statsBackground || statsStop)) {
throw new Error('Stats background and stop flags cannot be combined with stats actions.');
}
if (normalizedAction !== 'cleanup' && (options.vocab === true || options.lifetime === true)) {
throw new Error('Stats --vocab and --lifetime flags require the cleanup action.');
if (
normalizedAction !== 'cleanup' &&
(options.vocab === true || options.lifetime === true || options.duplicateLines === true)
) {
throw new Error(
'Stats --vocab, --lifetime and --duplicate-lines flags require the cleanup action.',
);
}
if (
options.duplicateLines !== true &&
(options.dryRun === true || options.lookbackDays !== undefined)
) {
throw new Error('Stats --dry-run and --lookback-days require --duplicate-lines.');
}
if (normalizedAction === 'cleanup') {
statsCleanup = true;
statsCleanupLifetime = options.lifetime === true;
statsCleanupVocab = statsCleanupLifetime ? false : options.vocab !== false;
statsCleanupDuplicateLines = options.duplicateLines === true;
const explicitModeCount = [options.vocab, options.lifetime, options.duplicateLines].filter(
(value) => value === true,
).length;
if (explicitModeCount > 1) {
throw new Error('Stats cleanup runs one mode at a time.');
}
// Vocabulary cleanup stays the default so `stats cleanup` keeps its old meaning.
statsCleanupVocab =
statsCleanupLifetime || statsCleanupDuplicateLines ? false : options.vocab !== false;
statsCleanupDryRun = options.dryRun === true;
statsCleanupLookbackDays = parseStatsLookbackDays(options.lookbackDays);
} else if (normalizedAction === 'rebuild' || normalizedAction === 'backfill') {
statsCleanup = true;
statsCleanupLifetime = true;
@@ -483,6 +524,9 @@ export function parseCliPrograms(
statsCleanup,
statsCleanupVocab,
statsCleanupLifetime,
statsCleanupDuplicateLines,
statsCleanupDryRun,
statsCleanupLookbackDays,
statsLogLevel,
syncTriggered,
syncCliTokens,
+5 -7
View File
@@ -222,7 +222,7 @@ test('buildMpvEnv preserves native Wayland env for supported Hyprland and Sway a
});
});
test('buildMpvBackendArgs forces an explicit X11 renderer stack when backend resolves to x11', () => {
test('buildMpvBackendArgs pins the X11 window context when backend resolves to x11', () => {
withPlatform('linux', () => {
assert.deepEqual(
buildMpvBackendArgs(makeArgs({ backend: 'x11' }), {
@@ -230,12 +230,12 @@ test('buildMpvBackendArgs forces an explicit X11 renderer stack when backend res
WAYLAND_DISPLAY: 'wayland-0',
XDG_SESSION_TYPE: 'wayland',
}),
['--vo=gpu', '--gpu-api=opengl', '--gpu-context=x11egl,x11'],
['--gpu-context=x11vk,x11egl,x11'],
);
});
});
test('buildMpvBackendArgs forces the same X11 renderer stack for unsupported Wayland auto fallback', () => {
test('buildMpvBackendArgs pins the same X11 window context for unsupported Wayland auto fallback', () => {
withPlatform('linux', () => {
assert.deepEqual(
buildMpvBackendArgs(makeArgs({ backend: 'auto' }), {
@@ -245,7 +245,7 @@ test('buildMpvBackendArgs forces the same X11 renderer stack for unsupported Way
XDG_CURRENT_DESKTOP: 'KDE',
XDG_SESSION_DESKTOP: 'plasma',
}),
['--vo=gpu', '--gpu-api=opengl', '--gpu-context=x11egl,x11'],
['--gpu-context=x11vk,x11egl,x11'],
);
});
});
@@ -292,9 +292,7 @@ test('buildConfiguredMpvDefaultArgs appends maximized launch mode to configured
'--secondary-sub-visibility=no',
'--alang=ja,jp,jpn,japanese,en,eng,english,enus,en-us',
'--slang=ja,jp,jpn,japanese,en,eng,english,enus,en-us',
'--vo=gpu',
'--gpu-api=opengl',
'--gpu-context=x11egl,x11',
'--gpu-context=x11vk,x11egl,x11',
'--window-maximized=yes',
],
);
+73 -1
View File
@@ -232,6 +232,75 @@ test('parseArgs maps lifetime stats cleanup flag', () => {
assert.equal(parsed.statsCleanupLifetime, true);
});
test('parseArgs maps duplicate-line stats cleanup flags', () => {
const parsed = parseArgs(
['stats', 'cleanup', '--duplicate-lines', '--dry-run', '--lookback-days', '30'],
'subminer',
{},
);
assert.equal(parsed.statsCleanup, true);
assert.equal(parsed.statsCleanupVocab, false);
assert.equal(parsed.statsCleanupDuplicateLines, true);
assert.equal(parsed.statsCleanupDryRun, true);
assert.equal(parsed.statsCleanupLookbackDays, 30);
const fractional = parseArgs(
['stats', 'cleanup', '--duplicate-lines', '--lookback-days', '1.5'],
'subminer',
{},
);
assert.equal(fractional.statsCleanupLookbackDays, 1);
});
test('parseArgs rejects duplicate-line flags without the duplicate-lines mode', () => {
const error = withProcessExitIntercept(() => {
parseArgs(['stats', 'cleanup', '--dry-run'], 'subminer', {});
});
assert.equal(error.code, 1);
assert.match(error.stderr, /--dry-run and --lookback-days require --duplicate-lines/);
});
test('parseArgs rejects an empty lookback value outside duplicate-line cleanup', () => {
const error = withProcessExitIntercept(() => {
parseArgs(['stats', '--lookback-days', ''], 'subminer', {});
});
assert.equal(error.code, 1);
assert.match(error.stderr, /--dry-run and --lookback-days require --duplicate-lines/);
});
test('parseArgs rejects combining explicit cleanup modes', () => {
for (const modes of [
['--lifetime', '--duplicate-lines'],
['--vocab', '--duplicate-lines'],
['--vocab', '--lifetime'],
]) {
const error = withProcessExitIntercept(() => {
parseArgs(['stats', 'cleanup', ...modes], 'subminer', {});
});
assert.equal(error.code, 1);
assert.match(error.stderr, /Stats cleanup runs one mode at a time/);
}
});
test('parseArgs rejects unusable lookback windows', () => {
for (const value of ['0', '0.5', '-5', 'soon']) {
const error = withProcessExitIntercept(() => {
parseArgs(
['stats', 'cleanup', '--duplicate-lines', '--lookback-days', value],
'subminer',
{},
);
});
assert.equal(error.code, 1);
assert.match(error.stderr, /--lookback-days must be at least one day/);
}
});
test('parseArgs rejects cleanup-only stats flags without cleanup action', () => {
const error = withProcessExitIntercept(() => {
parseArgs(['stats', '--vocab'], 'subminer', {});
@@ -239,7 +308,10 @@ test('parseArgs rejects cleanup-only stats flags without cleanup action', () =>
assert.equal(error.code, 1);
assert.match(error.message, /exit:1/);
assert.match(error.stderr, /Stats --vocab and --lifetime flags require the cleanup action/);
assert.match(
error.stderr,
/Stats --vocab, --lifetime and --duplicate-lines flags require the cleanup action/,
);
});
test('parseArgs maps stats rebuild action to cleanup lifetime mode', () => {
+3
View File
@@ -142,6 +142,9 @@ export interface Args {
statsCleanup?: boolean;
statsCleanupVocab?: boolean;
statsCleanupLifetime?: boolean;
statsCleanupDuplicateLines?: boolean;
statsCleanupDryRun?: boolean;
statsCleanupLookbackDays?: number;
dictionaryTarget?: string;
doctor: boolean;
doctorRefreshKnownWords: boolean;
+10 -4
View File
@@ -2,7 +2,7 @@
"name": "subminer",
"productName": "SubMiner",
"desktopName": "SubMiner.desktop",
"version": "0.19.1",
"version": "0.19.2",
"description": "All-in-one sentence mining overlay with AnkiConnect and dictionary integration",
"packageManager": "bun@1.3.5",
"main": "dist/main-entry.js",
@@ -84,8 +84,9 @@
"overrides": {
"@xmldom/xmldom": "0.8.13",
"app-builder-lib": "26.15.3",
"brace-expansion": "5.0.8",
"brace-expansion": "5.0.9",
"electron-builder-squirrel-windows": "26.15.3",
"fast-uri": "3.1.5",
"form-data": "4.0.6",
"ip-address": "10.2.0",
"js-yaml": "4.3.0",
@@ -93,7 +94,8 @@
"minimatch": "10.2.5",
"picomatch": "4.0.4",
"tar": "7.5.21",
"tmp": "0.2.7"
"tmp": "0.2.7",
"undici": "7.29.0"
},
"keywords": [
"anki",
@@ -125,7 +127,7 @@
"@types/ws": "^8.18.1",
"electron": "42.6.0",
"electron-builder": "26.15.3",
"undici": "7.28.0",
"undici": "7.29.0",
"esbuild": "^0.25.12",
"eslint": "^10.8.0",
"prettier": "^3.8.1",
@@ -258,6 +260,10 @@
{
"from": "dist/launcher/subminer",
"to": "launcher/subminer"
},
{
"from": "CHANGELOG.md",
"to": "CHANGELOG.md"
}
]
},
-32
View File
@@ -1,32 +0,0 @@
## Highlights
### Added
- Kiku/Lapis Word Card Type Setting
- A new setting (Mining/Anki > Kiku/Lapis Features > "Word Card Type") lets you choose which card-type flag gets marked on Kiku/Lapis word cards, including a click-card option that SubMiner couldn't set before.
- Handy if you only want click cards flagged instead of the default word-and-sentence marking.
- Choosing a card type now clears any other flags automatically, so a note can't end up marked as two types at once.
### Fixed
- Yomitan Popup on macOS
- Fixed the popup going unresponsive after mining a card — clicks outside it no longer leak through to mpv, and the overlay no longer flickers hidden and shown.
- Scrolling over the popup now scrolls its definitions instead of seeking the video.
- YouTube Playlist Links
- Opening a video from a playlist URL (like a Watch Later link with `list=`/`index=`) no longer times out while loading subtitles, metadata, or playback info.
## What's Changed
- feat(anki): add configurable word card type for Kiku/Lapis by @ksyasuda in #175
- fix(overlay): keep Yomitan popup interactive on macOS/Windows by @ksyasuda in #177
- fix(youtube): prevent playlist URLs from stalling yt-dlp probes by @ksyasuda in #180
## Installation
See the README and docs/installation guide for full setup steps.
## Assets
- Linux: `SubMiner.AppImage`
- macOS: `SubMiner-*.dmg` and `SubMiner-*.zip`
- Windows: `SubMiner-*.exe` and `SubMiner-*-win.zip`
- Optional extras: `subminer-assets.tar.gz` and the `subminer` launcher
Note: the `subminer` wrapper script uses Bun (`#!/usr/bin/env bun`), so `bun` must be installed and on `PATH`.
+24
View File
@@ -399,6 +399,30 @@ test('hasExplicitCommand and shouldStartApp preserve command intent', () => {
assert.equal(statsLifetimeRebuild.statsCleanupLifetime, true);
assert.equal(statsLifetimeRebuild.statsCleanupVocab, false);
assert.throws(
() =>
parseArgs([
'--stats',
'--stats-cleanup',
'--stats-cleanup-duplicate-lines',
'--stats-cleanup-lookback-days',
'0.5',
]),
/at least one day/,
);
assert.equal(
parseArgs([
'--stats',
'--stats-cleanup',
'--stats-cleanup-duplicate-lines',
'--stats-cleanup-lookback-days',
'1.5',
]).statsCleanupLookbackDays,
1,
);
assert.equal(parseArgs(['--stats-cleanup-lookback-days=30']).statsCleanupLookbackDays, 30);
assert.throws(() => parseArgs(['--stats-cleanup-lookback-days=30=oops']), /at least one day/);
const jellyfinLibraries = parseArgs(['--jellyfin-libraries']);
assert.equal(jellyfinLibraries.jellyfinLibraries, true);
assert.equal(hasExplicitCommand(jellyfinLibraries), true);
+22 -1
View File
@@ -64,6 +64,9 @@ export interface CliArgs {
statsCleanup?: boolean;
statsCleanupVocab?: boolean;
statsCleanupLifetime?: boolean;
statsCleanupDuplicateLines?: boolean;
statsCleanupDryRun?: boolean;
statsCleanupLookbackDays?: number;
statsResponsePath?: string;
jellyfin: boolean;
jellyfinLogin: boolean;
@@ -109,6 +112,14 @@ export interface CliArgs {
export type CliCommandSource = 'initial' | 'second-instance';
function parseStatsCleanupLookbackDays(value: string | undefined): number {
const days = Number(value);
if (!Number.isFinite(days) || days < 1) {
throw new Error('Stats --lookback-days must be at least one day.');
}
return Math.floor(days);
}
export function parseArgs(argv: string[]): CliArgs {
const args: CliArgs = {
background: false,
@@ -167,6 +178,8 @@ export function parseArgs(argv: string[]): CliArgs {
statsCleanup: false,
statsCleanupVocab: false,
statsCleanupLifetime: false,
statsCleanupDuplicateLines: false,
statsCleanupDryRun: false,
jellyfin: false,
jellyfinLogin: false,
jellyfinLogout: false,
@@ -368,7 +381,15 @@ export function parseArgs(argv: string[]): CliArgs {
} else if (arg === '--stats-cleanup') args.statsCleanup = true;
else if (arg === '--stats-cleanup-vocab') args.statsCleanupVocab = true;
else if (arg === '--stats-cleanup-lifetime') args.statsCleanupLifetime = true;
else if (arg.startsWith('--stats-response-path=')) {
else if (arg === '--stats-cleanup-duplicate-lines') args.statsCleanupDuplicateLines = true;
else if (arg === '--stats-cleanup-dry-run') args.statsCleanupDryRun = true;
else if (arg.startsWith('--stats-cleanup-lookback-days=')) {
args.statsCleanupLookbackDays = parseStatsCleanupLookbackDays(
arg.slice('--stats-cleanup-lookback-days='.length),
);
} else if (arg === '--stats-cleanup-lookback-days') {
args.statsCleanupLookbackDays = parseStatsCleanupLookbackDays(readValue(argv[i + 1]));
} else if (arg.startsWith('--stats-response-path=')) {
const value = arg.split('=', 2)[1];
if (value) args.statsResponsePath = value;
} else if (arg === '--stats-response-path') {
@@ -1032,6 +1032,180 @@ describe('stats server API routes', () => {
]);
});
it('POST /api/stats/maintenance/duplicate-lines forwards the window and dry-run flag', async () => {
let seenOptions: unknown = null;
const summary = {
dryRun: true,
lookbackDays: 30,
scannedLines: 900,
burstGroups: 2,
removedLines: 180,
removedWordOccurrences: 540,
removedKanjiOccurrences: 120,
samples: [],
};
const app = createStatsApp(
createMockTracker({
cleanupDuplicateSubtitleLines: async (options: unknown) => {
seenOptions = options;
return summary;
},
}),
);
const res = await app.request('/api/stats/maintenance/duplicate-lines', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ dryRun: true, lookbackDays: 30 }),
});
assert.equal(res.status, 200);
assert.deepEqual(await res.json(), summary);
assert.deepEqual(seenOptions, { dryRun: true, lookbackDays: 30 });
});
it('POST /api/stats/maintenance/duplicate-lines rejects cross-origin simple requests', async () => {
let cleanupCalls = 0;
const app = createStatsApp(
createMockTracker({
cleanupDuplicateSubtitleLines: async () => {
cleanupCalls += 1;
throw new Error('cleanup must not run');
},
}),
);
const res = await app.request('/api/stats/maintenance/duplicate-lines', {
method: 'POST',
headers: {
'Content-Type': 'text/plain',
Origin: 'https://attacker.example',
},
body: JSON.stringify({ dryRun: false, lookbackDays: null }),
});
assert.equal(res.status, 415);
assert.equal(cleanupCalls, 0);
});
it('POST /api/stats/maintenance/duplicate-lines rejects a window shorter than a day', async () => {
let cleanupCalls = 0;
const app = createStatsApp(
createMockTracker({
cleanupDuplicateSubtitleLines: async () => {
cleanupCalls += 1;
return {
dryRun: true,
lookbackDays: null,
scannedLines: 0,
burstGroups: 0,
removedLines: 0,
removedWordOccurrences: 0,
removedKanjiOccurrences: 0,
samples: [],
};
},
}),
);
const res = await app.request('/api/stats/maintenance/duplicate-lines', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ dryRun: true, lookbackDays: 0.5 }),
});
assert.equal(res.status, 400);
assert.equal(cleanupCalls, 0);
});
it('POST /api/stats/maintenance/duplicate-lines floors a fractional multi-day window', async () => {
let seenOptions: unknown = null;
const app = createStatsApp(
createMockTracker({
cleanupDuplicateSubtitleLines: async (options: unknown) => {
seenOptions = options;
return {
dryRun: true,
lookbackDays: 1,
scannedLines: 0,
burstGroups: 0,
removedLines: 0,
removedWordOccurrences: 0,
removedKanjiOccurrences: 0,
samples: [],
};
},
}),
);
const res = await app.request('/api/stats/maintenance/duplicate-lines', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ dryRun: true, lookbackDays: 1.5 }),
});
assert.equal(res.status, 200);
assert.deepEqual(seenOptions, { dryRun: true, lookbackDays: 1 });
});
it('POST /api/stats/maintenance/duplicate-lines accepts an explicit empty object for all history', async () => {
let seenOptions: unknown = null;
const app = createStatsApp(
createMockTracker({
cleanupDuplicateSubtitleLines: async (options: unknown) => {
seenOptions = options;
return {
dryRun: false,
lookbackDays: null,
scannedLines: 0,
burstGroups: 0,
removedLines: 0,
removedWordOccurrences: 0,
removedKanjiOccurrences: 0,
samples: [],
};
},
}),
);
const res = await app.request('/api/stats/maintenance/duplicate-lines', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: '{}',
});
assert.equal(res.status, 200);
assert.deepEqual(seenOptions, { dryRun: false, lookbackDays: null });
});
for (const malformed of [
{ name: 'a missing body', body: undefined },
{ name: 'malformed JSON', body: '{' },
{ name: 'JSON null', body: 'null' },
{ name: 'a JSON array', body: '[]' },
]) {
it(`POST /api/stats/maintenance/duplicate-lines rejects ${malformed.name}`, async () => {
let cleanupCalls = 0;
const app = createStatsApp(
createMockTracker({
cleanupDuplicateSubtitleLines: async () => {
cleanupCalls += 1;
throw new Error('cleanup must not run');
},
}),
);
const res = await app.request('/api/stats/maintenance/duplicate-lines', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: malformed.body,
});
assert.equal(res.status, 400);
assert.equal(cleanupCalls, 0);
});
}
it('PUT /api/stats/excluded-words rejects malformed rows', async () => {
const app = createStatsApp(createMockTracker());
+195
View File
@@ -0,0 +1,195 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
assOverrideSignature,
assToPlainText,
collectAssOverrideCommands,
extractAssOverrideBlocks,
hasAssTemporalOverride,
isAnimatedAssEffectKind,
isAssTemporalCommand,
normalizePlainSubtitleText,
parseAssEffectField,
} from './ass-text';
test('assToPlainText drops vector drawing runs', () => {
assert.equal(
assToPlainText(
'{\\an5\\pos(730,1042)\\p1\\blur1}m 20 0 b 10 0 0 10 0 20 b 0 31 10 40 20 40 {\\p0}',
),
'',
);
});
test('assToPlainText keeps text around drawing runs on the same event', () => {
assert.equal(
assToPlainText('{\\p1}m 0 0 l 10 10{\\p0}本文{\\p1}m 5 5 l 6 6{\\p0}続き'),
'本文続き',
);
});
test('assToPlainText leaves \\pos alone when no drawing mode is active', () => {
assert.equal(assToPlainText('{\\pos(960,1068)\\bord3}位置指定'), '位置指定');
});
test('assToPlainText does not read \\pos as a drawing tag', () => {
assert.equal(assToPlainText('{\\p1\\pos(1,2)}m 0 0 l 5 5'), '');
});
test('assToPlainText resolves line-break and space escapes', () => {
assert.equal(assToPlainText('一行目\\N二行目'), '一行目\n二行目');
assert.equal(assToPlainText('一行目\\n二行目'), '一行目\n二行目');
assert.equal(assToPlainText('一行目\\N二行目', ' '), '一行目 二行目');
assert.equal(assToPlainText('間\\h隔'), '間 隔');
});
test('assToPlainText matches mpv on brace and backslash sequences', () => {
// mpv has no `\{` / `\}` / `\\` escapes: the backslashes are literal text and the
// braces still open and close an override block.
assert.equal(assToPlainText('\\{注\\}'), '\\');
assert.equal(assToPlainText('\\\\N'), '\\\n');
});
test('assToPlainText renders an unclosed override block verbatim', () => {
// mpv shows the stray brace; guessing where the block ended can eat a whole line.
assert.equal(assToPlainText('本文{\\pos(1,2)'), '本文{\\pos(1,2)');
});
test('assToPlainText is idempotent', () => {
const samples = [
'{\\an5\\p1}m 0 0 l 5 5{\\p0}本文',
'\\{注\\}',
'\\\\N',
'本文{\\pos(1,2)',
'一行目\\N二行目\\h終わり',
];
for (const sample of samples) {
const once = assToPlainText(sample);
assert.equal(assToPlainText(once), once, sample);
}
});
test('assToPlainText normalizes CRLF before converting', () => {
assert.equal(assToPlainText('一行目\r\n二行目'), '一行目\n二行目');
});
test('normalizePlainSubtitleText settles whitespace without decoding ASS', () => {
// A brace reaching this layer is literal text mpv chose to show, not markup.
assert.equal(normalizePlainSubtitleText('本文{\\pos(1,2)'), '本文{\\pos(1,2)');
assert.equal(normalizePlainSubtitleText('一行目\\N二行目'), '一行目\n二行目');
assert.equal(
normalizePlainSubtitleText('一行目\\N二行目', { collapseLineBreaks: true }),
'一行目 二行目',
);
assert.equal(normalizePlainSubtitleText(' 余白 ', { trim: false }), ' 余白 ');
});
test('normalizePlainSubtitleText is idempotent', () => {
for (const sample of ['一行目\\N二行目', '間\\h隔', '本文{\\pos(1,2)', ' 余白 ']) {
const once = normalizePlainSubtitleText(sample);
assert.equal(normalizePlainSubtitleText(once), once, sample);
}
});
test('extractAssOverrideBlocks returns block contents', () => {
assert.deepEqual(extractAssOverrideBlocks('{\\an8}上{\\fad(200,200)}下'), [
'\\an8',
'\\fad(200,200)',
]);
assert.deepEqual(extractAssOverrideBlocks('括弧なし'), []);
});
test('collectAssOverrideCommands captures names and arguments from blocks only', () => {
const commands = collectAssOverrideCommands('{\\pos(1,2)\\1c&HFFFFFF&\\kf30}歌詞');
assert.deepEqual(commands, [
{ name: 'pos', args: '1,2', animated: false },
{ name: '1c', args: '&HFFFFFF&', animated: false },
{ name: 'kf', args: '30', animated: false },
]);
// A `\pos(...)` sitting in visible text is not typesetting markup.
assert.deepEqual(collectAssOverrideCommands('\\pos(730,1042) と書いてある'), []);
});
test('collectAssOverrideCommands marks tags animated by a wrapping \\t', () => {
const commands = collectAssOverrideCommands('{\\clip(0,0,10,10)\\t(0,500,\\frz30)}文字');
assert.deepEqual(
commands.map((command) => [command.name, command.animated]),
[
['clip', false],
['t', false],
['frz', true],
],
);
assert.equal(hasAssTemporalOverride(commands), true);
});
test('collectAssOverrideCommands stops descending into deeply nested \\t tags', () => {
// Nested far past the recursion cap. Uncapped, this recurses once per level, and a
// pathological line (real files reach one or two levels) overflows the stack.
const nesting = 32;
const block = `{${'\\t(0,500,'.repeat(nesting)}\\frz30${')'.repeat(nesting)}}文字`;
const commands = collectAssOverrideCommands(block);
// The outer `\t` plus one per allowed recursion level, and nothing from below the cap.
assert.equal(commands.length, 9);
assert.deepEqual(new Set(commands.map((command) => command.name)), new Set(['t']));
assert.equal(hasAssTemporalOverride(commands), true);
});
test('hasAssTemporalOverride ignores static placement and shape tags', () => {
assert.equal(
hasAssTemporalOverride(collectAssOverrideCommands('{\\pos(1,2)\\clip(m 1 1)\\blur2}文字')),
false,
);
assert.equal(hasAssTemporalOverride(collectAssOverrideCommands('{\\move(1,2,3,4)}文字')), true);
});
test('isAssTemporalCommand covers only intrinsically animated tags', () => {
for (const command of ['t', 'move', 'k', 'kf', 'ko', 'K']) {
assert.equal(isAssTemporalCommand(command), true, command);
}
for (const command of ['clip', 'iclip', 'frz', 'fscx', 'blur', 'be', 'pos', 'fad']) {
assert.equal(isAssTemporalCommand(command), false, command);
}
});
test('assOverrideSignature distinguishes events by their override values', () => {
const first = assOverrideSignature(collectAssOverrideCommands('{\\clip(m 1 1)}歌詞'));
const second = assOverrideSignature(collectAssOverrideCommands('{\\clip(m 2 2)}歌詞'));
const repeat = assOverrideSignature(collectAssOverrideCommands('{\\clip(m 1 1)}別の行'));
assert.notEqual(first, second);
assert.equal(first, repeat);
});
test('parseAssEffectField classifies the event-level Effect column', () => {
assert.equal(parseAssEffectField(''), 'none');
assert.equal(parseAssEffectField(' '), 'none');
assert.equal(parseAssEffectField('Banner;20;1;0'), 'banner');
assert.equal(parseAssEffectField('Scroll up;0;0;30;10'), 'scroll');
assert.equal(parseAssEffectField('Scroll down;0;0;30;10'), 'scroll');
assert.equal(parseAssEffectField('Karaoke'), 'karaoke');
assert.equal(parseAssEffectField('fx-template'), 'other');
});
test('parseAssEffectField matches stock effect names exactly', () => {
// Custom effect names that merely start with a stock name are not stock effects.
assert.equal(parseAssEffectField('scrolling-credit'), 'other');
assert.equal(parseAssEffectField('bannerfx;1'), 'other');
assert.equal(parseAssEffectField('karaoke-template'), 'other');
assert.equal(parseAssEffectField('Scroll'), 'other');
});
test('isAnimatedAssEffectKind covers the stock animated effects only', () => {
assert.equal(isAnimatedAssEffectKind('karaoke'), true);
assert.equal(isAnimatedAssEffectKind('banner'), true);
assert.equal(isAnimatedAssEffectKind('scroll'), true);
// Typesetting groups put static template names in this column too.
assert.equal(isAnimatedAssEffectKind('other'), false);
assert.equal(isAnimatedAssEffectKind('none'), false);
});
+280
View File
@@ -0,0 +1,280 @@
/*
* ASS/SSA text handling, split into two deliberately distinct contracts:
*
* assToPlainText() raw ASS event text -> plain text. Ingestion only.
* normalizePlainSubtitleText() already-decoded text -> display/lookup form.
*
* Subtitle text is decoded from ASS exactly once, at the point it enters the app: the
* file cue parser does it for sidecar/embedded scripts, and mpv does it for live text
* (`sub-text` is already run through mpv's own `ass_to_plaintext`). Everything
* downstream -- renderer, timing tracker, tokenizer, tokenization cache keys -- gets
* plain text and only normalizes whitespace, so no layer decodes the same string twice.
*
* assToPlainText mirrors mpv's `ass_to_plaintext` rather than inventing its own rules,
* so a cue parsed from a file reads the same as the same line arriving live:
* - `{...}` override blocks are markup
* - `\pN ... \p0` runs are vector paths, not dialogue
* - `\N`, `\n` and `\h` are the only escapes; `\{`, `\}` and `\\` are NOT escapes,
* so `\{注\}` decodes to a lone backslash exactly as mpv renders it
* - an unclosed `{` is rendered verbatim instead of swallowing the rest of the line
* Because the decoder never emits an escape or a closed brace, running it twice is a
* no-op -- but downstream code should still use normalizePlainSubtitleText.
*/
/** What `\N` and `\n` become. */
export type AssLineBreak = '\n' | ' ';
// `\p<n>` with n > 0 switches libass into vector-drawing mode: everything until the
// next `\p0` is a path (`m 20 0 b 10 0 ...`), not dialogue. The negative lookahead keeps
// `\pos(...)` from being read as a drawing tag.
const ASS_DRAWING_SCALE_PATTERN = /\\p(?![a-zA-Z])(\d*)/g;
function readDrawingScale(block: string): number | null {
ASS_DRAWING_SCALE_PATTERN.lastIndex = 0;
let scale: number | null = null;
let match: RegExpExecArray | null;
// Drawing mode is whatever the last `\p` tag in this block set it to.
while ((match = ASS_DRAWING_SCALE_PATTERN.exec(block)) !== null) {
scale = match[1] ? Number(match[1]) : 0;
}
return scale;
}
/** Resolve `\N`, `\n` and `\h`. The only text-level escapes libass recognises. */
function resolveWhitespaceEscapes(text: string, lineBreak: AssLineBreak): string {
return text.replace(/\\([Nnh])/g, (_match, escaped: string) =>
escaped === 'h' ? ' ' : lineBreak,
);
}
/** Strip `{...}` override blocks and the drawing runs they enable. */
function stripAssMarkup(raw: string): string {
let out = '';
let cursor = 0;
let drawing = false;
while (cursor < raw.length) {
if (raw[cursor] !== '{') {
if (!drawing) {
out += raw[cursor];
}
cursor += 1;
continue;
}
const close = raw.indexOf('}', cursor + 1);
if (close === -1) {
// mpv shows an unclosed `{` and everything after it. Guessing where the block was
// meant to end can eat a whole line of dialogue.
if (!drawing) {
out += raw.slice(cursor);
}
break;
}
const scale = readDrawingScale(raw.slice(cursor, close + 1));
if (scale !== null) {
drawing = scale > 0;
}
cursor = close + 1;
}
return out;
}
/**
* Decode a raw ASS/SSA event text field. Call this once, where the text enters the app;
* downstream layers take the result as plain text.
*/
export function assToPlainText(text: string, lineBreak: AssLineBreak = '\n'): string {
if (!text) return '';
return resolveWhitespaceEscapes(stripAssMarkup(text.replace(/\r\n/g, '\n')), lineBreak);
}
export interface NormalizePlainSubtitleTextOptions {
/** Fold every line break into a single space. */
collapseLineBreaks?: boolean;
trim?: boolean;
}
/**
* Whitespace normalization for text that has already been decoded -- by mpv for live
* subtitles, by the cue parser for files. Override blocks and drawing runs are none of
* this function's business; a `{` that reaches here is literal text mpv chose to show.
*
* `\N`/`\n`/`\h` are still folded, because subtitle sources outside the ASS path (asbplayer
* and other websocket clients) forward them raw and the display layer has to cope.
*/
export function normalizePlainSubtitleText(
text: string,
options: NormalizePlainSubtitleTextOptions = {},
): string {
if (!text) return '';
const { collapseLineBreaks = false, trim = true } = options;
let normalized = resolveWhitespaceEscapes(
text.replace(/\r\n/g, '\n'),
collapseLineBreaks ? ' ' : '\n',
);
if (collapseLineBreaks) {
normalized = normalized.replace(/\n/g, ' ').replace(/\s+/g, ' ');
}
return trim ? normalized.trim() : normalized;
}
/** The contents of each `{...}` block, without the braces. */
export function extractAssOverrideBlocks(text: string): string[] {
const blocks: string[] = [];
let cursor = 0;
while (cursor < text.length) {
const open = text.indexOf('{', cursor);
if (open === -1) {
break;
}
const close = text.indexOf('}', open + 1);
if (close === -1) {
break;
}
blocks.push(text.slice(open + 1, close));
cursor = close + 1;
}
return blocks;
}
export interface AssOverrideCommand {
/** Tag name without the backslash, e.g. `pos`, `kf`, `1c`. */
name: string;
/** Everything the tag was given, e.g. `960,1068` for `\pos(960,1068)`. */
args: string;
/** Nested inside a `\t(...)` argument, so its value is animated over the event. */
animated: boolean;
}
const ASS_OVERRIDE_NAME_PATTERN = /[1-4]?[a-zA-Z]+/y;
function readCommandArgs(block: string, start: number): { args: string; next: number } {
if (block[start] === '(') {
let depth = 0;
for (let i = start; i < block.length; i += 1) {
if (block[i] === '(') depth += 1;
else if (block[i] === ')') {
depth -= 1;
if (depth === 0) {
return { args: block.slice(start + 1, i), next: i + 1 };
}
}
}
return { args: block.slice(start + 1), next: block.length };
}
const nextTag = block.indexOf('\\', start);
const end = nextTag === -1 ? block.length : nextTag;
return { args: block.slice(start, end), next: end };
}
// `\t(...)` can wrap another `\t(...)`, and nothing in the format stops an author (or a
// malformed file) from nesting them thousands deep. Real typesetting never goes past one
// or two levels, so stop recursing well before the call stack is at risk.
const MAX_ANIMATION_NESTING_DEPTH = 8;
function parseOverrideBlock(
block: string,
animated: boolean,
into: AssOverrideCommand[],
depth = 0,
): void {
let cursor = 0;
while (cursor < block.length) {
if (block[cursor] !== '\\') {
cursor += 1;
continue;
}
ASS_OVERRIDE_NAME_PATTERN.lastIndex = cursor + 1;
const nameMatch = ASS_OVERRIDE_NAME_PATTERN.exec(block);
if (!nameMatch) {
cursor += 1;
continue;
}
const name = nameMatch[0];
const { args, next } = readCommandArgs(block, cursor + 1 + name.length);
into.push({ name, args: args.trim(), animated });
// `\t(0,500,\frz30)` animates whatever it wraps, so record the inner tags too.
if (name === 't' && args.includes('\\') && depth < MAX_ANIMATION_NESTING_DEPTH) {
parseOverrideBlock(args, true, into, depth + 1);
}
cursor = next;
}
}
/**
* Override commands with their arguments, in source order. Only `{...}` blocks are
* inspected, so a `\pos(...)` sitting in visible text is never mistaken for markup.
*/
export function collectAssOverrideCommands(text: string): AssOverrideCommand[] {
const commands: AssOverrideCommand[] = [];
for (const block of extractAssOverrideBlocks(text)) {
parseOverrideBlock(block, false, commands);
}
return commands;
}
// Tags that are animated by definition: `\t` interpolates, `\move` travels, and the
// karaoke tags advance a highlight across the event's own duration. Everything else --
// `\pos`, `\clip`, `\frz`, `\blur`, `\fad` -- is a static value for the event, so its
// presence says nothing about whether neighbouring events form one animation.
const ASS_TEMPORAL_COMMANDS = new Set(['t', 'move', 'k', 'kf', 'ko', 'K']);
export function isAssTemporalCommand(name: string): boolean {
return ASS_TEMPORAL_COMMANDS.has(name);
}
/** True when the event animates on its own, or animates a static tag through `\t(...)`. */
export function hasAssTemporalOverride(commands: readonly AssOverrideCommand[]): boolean {
return commands.some((command) => command.animated || isAssTemporalCommand(command.name));
}
/**
* Canonical form of an event's override values, for comparing consecutive events. Two
* events with the same signature were typeset identically, so neither is a frame of an
* animation the other belongs to.
*/
export function assOverrideSignature(commands: readonly AssOverrideCommand[]): string {
return commands.map((command) => `${command.name}(${command.args})`).join('|');
}
export type AssEffectKind = 'none' | 'banner' | 'scroll' | 'karaoke' | 'other';
// The stock effects, matched exactly. Typesetting groups put their own template names in
// this column -- `scrolling-credit` is a static sign, not libass's `Scroll up` -- so a
// prefix match would hand out animation evidence to arbitrary custom effects.
const STOCK_ASS_EFFECTS = new Map<string, AssEffectKind>([
['banner', 'banner'],
['scroll up', 'scroll'],
['scroll down', 'scroll'],
['karaoke', 'karaoke'],
]);
/**
* The event-level `Effect` column. The stock values (`Banner;...`, `Scroll up;...`,
* `Scroll down;...`, `Karaoke`) all animate; anything else is a custom name and lands in
* `other`.
*/
export function parseAssEffectField(raw: string): AssEffectKind {
const value = raw.trim().toLowerCase();
if (!value) return 'none';
const name = value.split(';', 1)[0]!.trim();
return STOCK_ASS_EFFECTS.get(name) ?? 'other';
}
const ANIMATED_ASS_EFFECT_KINDS = new Set<AssEffectKind>(['banner', 'scroll', 'karaoke']);
export function isAnimatedAssEffectKind(kind: AssEffectKind): boolean {
return ANIMATED_ASS_EFFECT_KINDS.has(kind);
}
@@ -91,6 +91,11 @@ import {
markVideoWatched,
upsertCoverArt,
} from './immersion-tracker/query-maintenance';
import {
cleanupDuplicateSubtitleLines,
type DuplicateSubtitleLineCleanupOptions,
type DuplicateSubtitleLineCleanupSummary,
} from './immersion-tracker/duplicate-line-cleanup';
import { repairJellyfinStreamVideoLinks } from './immersion-tracker/jellyfin-link-repair';
import {
repairLegacySeasonlessAnimeRows,
@@ -595,6 +600,18 @@ export class ImmersionTrackerService {
});
}
/**
* Collapse animation bursts that earlier versions recorded frame by frame. The whole
* queue is drained first so a burst still waiting to be written is scanned as stored
* rows rather than surviving the cleanup and landing a moment after it.
*/
async cleanupDuplicateSubtitleLines(
options: DuplicateSubtitleLineCleanupOptions = {},
): Promise<DuplicateSubtitleLineCleanupSummary> {
this.drainQueue();
return cleanupDuplicateSubtitleLines(this.db, options);
}
async rebuildLifetimeSummaries(): Promise<LifetimeRebuildSummary> {
this.flushTelemetry(true);
this.flushNow();
@@ -1799,6 +1816,24 @@ export class ImmersionTrackerService {
}
}
/**
* Write out everything queued, not just the next batch.
*
* `flushNow` writes at most `batchSize` entries and does nothing at all while the write
* lock is held, so a maintenance pass that runs straight after it can still be reading
* a database that is missing rows. Each pass has to shrink the queue to continue: a
* failed flush puts its batch back, and looping on that would never finish.
*/
private drainQueue(): void {
while (this.queue.length > 0) {
const pendingBefore = this.queue.length;
this.flushNow();
if (this.queue.length >= pendingBefore) {
return;
}
}
}
private flushSingle(write: QueuedWrite): void {
executeQueuedWrite(write, this.preparedStatements);
}
@@ -0,0 +1,349 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { Database } from '../sqlite.js';
import type { DatabaseSync } from '../sqlite.js';
import { ensureSchema } from '../storage.js';
import { cleanupDuplicateSubtitleLines } from '../duplicate-line-cleanup.js';
const DAY_MS = 86_400_000;
const BASE_MS = 1_700_000_000_000;
const WORD_ID = 1;
interface SeedLine {
session: number;
text: string;
startMs: number;
endMs: number;
/** Recording wall-clock, i.e. what the lookback window filters on. */
createdMs?: number;
}
function makeDbPath(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-duplicate-line-test-'));
return path.join(dir, 'immersion.sqlite');
}
function cleanupDbPath(dbPath: string): void {
const dir = path.dirname(dbPath);
if (!fs.existsSync(dir)) return;
fs.rmSync(dir, { recursive: true, force: true });
}
/** One episode, two sessions of it, and one word occurrence per seeded line. */
function seed(db: DatabaseSync, lines: SeedLine[]): void {
db.exec(`
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (1, 'show', 'Show', ${BASE_MS}, ${BASE_MS});
INSERT INTO imm_videos(video_id, video_key, anime_id, canonical_title, source_type, watched, duration_ms, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (1, 'v1', 1, 'Ep 1', 1, 1, 1440000, ${BASE_MS}, ${BASE_MS});
INSERT INTO imm_sessions(session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (1, 's1', 1, '${BASE_MS}', '${BASE_MS + 1000}', 2, ${BASE_MS}, ${BASE_MS}),
(2, 's2', 1, '${BASE_MS + DAY_MS}', '${BASE_MS + DAY_MS + 1000}', 2, ${BASE_MS}, ${BASE_MS});
INSERT INTO imm_words(id, headword, word, reading, part_of_speech, pos1, first_seen, last_seen, frequency)
VALUES (${WORD_ID}, '飛び上がる', '飛び上がる', '', 'verb', '動詞', ${Math.floor(BASE_MS / 1000)}, ${Math.floor(BASE_MS / 1000)}, 0);
`);
const insertLine = db.prepare(
`INSERT INTO imm_subtitle_lines(
line_id, session_id, video_id, anime_id, line_index,
segment_start_ms, segment_end_ms, text, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (?, ?, 1, 1, ?, ?, ?, ?, ?, ?)`,
);
const insertOccurrence = db.prepare(
`INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms)
VALUES (?, ?, 1, ?)`,
);
lines.forEach((line, index) => {
const lineId = index + 1;
const lineIndex = index + 1;
const createdMs = line.createdMs ?? BASE_MS;
insertLine.run(
lineId,
line.session,
lineIndex,
line.startMs,
line.endMs,
line.text,
createdMs,
createdMs,
);
insertOccurrence.run(lineId, WORD_ID, createdMs);
});
db.exec(`
UPDATE imm_words SET frequency = (
SELECT COALESCE(SUM(o.occurrence_count), 0)
FROM imm_word_line_occurrences o WHERE o.word_id = imm_words.id
)
`);
}
function createDb(lines: SeedLine[]): { db: DatabaseSync; dbPath: string } {
const dbPath = makeDbPath();
const db = new Database(dbPath);
ensureSchema(db);
seed(db, lines);
return { db, dbPath };
}
/** A typeset line mpv reported once per animation frame. */
function karaokeFrames(
session: number,
text: string,
startMs: number,
frames: number,
frameMs: number,
): SeedLine[] {
return Array.from({ length: frames }, (_, index) => ({
session,
text,
startMs: startMs + index * frameMs,
endMs: startMs + (index + 1) * frameMs,
}));
}
function countLines(db: DatabaseSync): number {
return (db.prepare('SELECT COUNT(*) AS total FROM imm_subtitle_lines').get() as { total: number })
.total;
}
function wordFrequency(db: DatabaseSync): number {
const row = db.prepare('SELECT frequency FROM imm_words WHERE id = ?').get(WORD_ID) as {
frequency: number;
} | null;
return row?.frequency ?? 0;
}
test('a karaoke burst collapses to one line and gives back its word counts', () => {
const { db, dbPath } = createDb([
...karaokeFrames(1, '飛び上がる', 10_000, 40, 40),
{ session: 1, text: 'おはよう', startMs: 20_000, endMs: 22_000 },
]);
try {
const summary = cleanupDuplicateSubtitleLines(db);
assert.equal(summary.burstGroups, 1);
assert.equal(summary.removedLines, 39);
assert.equal(summary.removedWordOccurrences, 39);
assert.equal(countLines(db), 2);
assert.equal(wordFrequency(db), 2);
// The surviving line covers the whole run, the way the parsed cue would.
const kept = db
.prepare(
'SELECT segment_start_ms AS startMs, segment_end_ms AS endMs FROM imm_subtitle_lines WHERE line_id = 1',
)
.get() as { startMs: number; endMs: number };
assert.equal(kept.startMs, 10_000);
assert.equal(kept.endMs, 10_000 + 40 * 40);
assert.equal(summary.samples.length, 1);
assert.equal(summary.samples[0]!.text, '飛び上がる');
assert.equal(summary.samples[0]!.frames, 40);
assert.equal(summary.samples[0]!.videoTitle, 'Ep 1');
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('ordinary repeated dialogue survives', () => {
// Six contiguous `飛び上がる`, each held for a normal beat rather than a frame.
const lines = Array.from({ length: 6 }, (_, index) => ({
session: 1,
text: '飛び上がる',
startMs: 5_000 + index * 800,
endMs: 5_000 + (index + 1) * 800,
}));
const { db, dbPath } = createDb(lines);
try {
const summary = cleanupDuplicateSubtitleLines(db);
assert.equal(summary.burstGroups, 0);
assert.equal(summary.removedLines, 0);
assert.equal(countLines(db), 6);
assert.equal(wordFrequency(db), 6);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a long run of quarter-second frames is still a burst', () => {
// Between the timing-only bound (0.1s) and the animation-frame bound (0.3s): heavier
// typesetting lands here, and the run length is what makes it conclusive.
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
try {
const summary = cleanupDuplicateSubtitleLines(db);
assert.equal(summary.burstGroups, 1);
assert.equal(summary.removedLines, 5);
assert.equal(countLines(db), 1);
assert.equal(wordFrequency(db), 1);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a qualifying short-frame burst may end with one long hold frame', () => {
const { db, dbPath } = createDb([
...karaokeFrames(1, '飛び上がる', 10_000, 8, 40),
{ session: 1, text: '飛び上がる', startMs: 10_320, endMs: 12_320 },
]);
try {
const summary = cleanupDuplicateSubtitleLines(db);
assert.equal(summary.burstGroups, 1);
assert.equal(summary.removedLines, 8);
assert.equal(countLines(db), 1);
assert.equal(wordFrequency(db), 1);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a long event before the final frame prevents burst cleanup', () => {
const { db, dbPath } = createDb([
...karaokeFrames(1, '飛び上がる', 10_000, 5, 40),
{ session: 1, text: '飛び上がる', startMs: 10_200, endMs: 12_200 },
{ session: 1, text: '飛び上がる', startMs: 12_200, endMs: 12_240 },
]);
try {
const summary = cleanupDuplicateSubtitleLines(db);
assert.equal(summary.burstGroups, 0);
assert.equal(countLines(db), 7);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a run of frames longer than the animation bound survives', () => {
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 400));
try {
const summary = cleanupDuplicateSubtitleLines(db);
assert.equal(summary.burstGroups, 0);
assert.equal(countLines(db), 6);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a short run below the threshold survives', () => {
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 40));
try {
const summary = cleanupDuplicateSubtitleLines(db);
assert.equal(summary.burstGroups, 0);
assert.equal(countLines(db), 4);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('the same line in a rewatch session is never merged into the first watch', () => {
const { db, dbPath } = createDb([
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
...karaokeFrames(2, '飛び上がる', 10_000, 6, 40),
]);
try {
const summary = cleanupDuplicateSubtitleLines(db);
assert.equal(summary.burstGroups, 2);
assert.equal(summary.removedLines, 10);
// One surviving line per session, not one across both.
assert.equal(countLines(db), 2);
assert.equal(wordFrequency(db), 2);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a gap between runs splits them', () => {
const { db, dbPath } = createDb([
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
...karaokeFrames(1, '飛び上がる', 60_000, 6, 40),
]);
try {
const summary = cleanupDuplicateSubtitleLines(db);
assert.equal(summary.burstGroups, 2);
assert.equal(countLines(db), 2);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a dry run reports what an apply would do and writes nothing', () => {
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
try {
const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
assert.equal(preview.dryRun, true);
assert.equal(preview.removedLines, 39);
assert.equal(countLines(db), 40);
assert.equal(wordFrequency(db), 40);
const applied = cleanupDuplicateSubtitleLines(db);
assert.equal(applied.removedLines, preview.removedLines);
assert.equal(applied.removedWordOccurrences, preview.removedWordOccurrences);
assert.equal(countLines(db), 1);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('the lookback window leaves older bursts alone', () => {
const recentMs = BASE_MS;
const oldMs = BASE_MS - 40 * DAY_MS;
const { db, dbPath } = createDb([
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40).map((line) => ({
...line,
createdMs: oldMs,
})),
...karaokeFrames(2, '飛び上がる', 10_000, 6, 40).map((line) => ({
...line,
createdMs: recentMs,
})),
]);
globalThis.__subminerTestNowMs = BASE_MS;
try {
const summary = cleanupDuplicateSubtitleLines(db, { lookbackDays: 30 });
assert.equal(summary.lookbackDays, 30);
assert.equal(summary.scannedLines, 6);
assert.equal(summary.burstGroups, 1);
assert.equal(summary.removedLines, 5);
// Six untouched old frames plus the one surviving recent line.
assert.equal(countLines(db), 7);
assert.equal(wordFrequency(db), 7);
} finally {
globalThis.__subminerTestNowMs = undefined;
db.close();
cleanupDbPath(dbPath);
}
});
@@ -0,0 +1,386 @@
/*
* Retroactive removal of animation-burst subtitle lines from the stats database.
*
* Before the live ingest gate existed, a karaoke OP recorded one line -- and one count
* for every word in it -- per animation frame, which is enough to put an OP lyric at the
* top of "Top Repeated Words" for good. This module finds those runs in what is already
* stored and takes them back down to one line.
*
* Only timing is available here: the stored text has been stripped of ASS markup, so the
* authoring evidence the file-level parser uses (`\t`, `\move`, karaoke timing, a
* changing override signature) is long gone. What is left is a run of identical,
* contiguous, short-lived lines inside a single session.
*
* The run has to be as long as the timing-only rule in `subtitle-cue-dedup` demands, but
* its short frames may be as long as the animation-frame bound rather than the much
* tighter timing-only one. A qualifying run may end with one longer hold, which is a
* common karaoke shape. Five or more repeats of the same text, each ending where the next
* begins, is already conclusive on its own -- no dialogue does that -- and the tighter
* bound would walk straight past the heavier typesetting that motivated this, where
* frames sit nearer a quarter of a second. Both bounds are options, so a cautious run can
* ask for more, and a dry run always reports before anything is removed.
*
* Scope: subtitle lines, their word/kanji occurrences, and the `imm_words`/`imm_kanji`
* aggregates those occurrences feed. Session telemetry (`lines_seen`, `tokens_seen`) and
* the rollups derived from it are left alone; they are cumulative samples taken at record
* time, and for sessions whose raw rows have since been pruned they cannot be recomputed.
*/
import type { DatabaseSync } from './sqlite';
import {
ANIMATION_FRAME_MAX_SECONDS,
DUPLICATE_CUE_GAP_TOLERANCE_SECONDS,
MIN_TIMING_ONLY_FRAMES,
} from '../subtitle-burst-constants';
import {
applyLexicalRemovals,
makePlaceholders,
planLexicalRemovalsForLines,
toDbTimestamp,
} from './query-shared';
import { nowMs } from './time';
const MS_PER_DAY = 86_400_000;
/** SQLite caps bound parameters per statement; stay well under it. */
const LINE_ID_BATCH_SIZE = 400;
const DEFAULT_SAMPLE_LIMIT = 20;
export interface DuplicateSubtitleLineCleanupOptions {
/** Only consider lines recorded within this many days. Null or omitted = all history. */
lookbackDays?: number | null;
/** Measure without writing. */
dryRun?: boolean;
/** Identical contiguous lines needed before a run counts as an animation. */
minRunLength?: number;
/** Longest a single event may last and still look like an animation frame. */
maxFrameSeconds?: number;
/** How many of the largest runs to describe in the summary. */
sampleLimit?: number;
}
export interface DuplicateSubtitleLineBurst {
sessionId: number;
videoId: number;
text: string;
/** Kept line, extended to cover the whole run. */
keptLineId: number;
removedLineIds: number[];
startMs: number;
endMs: number;
}
export interface DuplicateSubtitleLineSample {
videoId: number;
videoTitle: string | null;
text: string;
frames: number;
removedLines: number;
startMs: number;
endMs: number;
}
export interface DuplicateSubtitleLineCleanupSummary {
dryRun: boolean;
lookbackDays: number | null;
scannedLines: number;
burstGroups: number;
removedLines: number;
removedWordOccurrences: number;
removedKanjiOccurrences: number;
samples: DuplicateSubtitleLineSample[];
}
export interface StoredSubtitleLineRow {
lineId: number;
sessionId: number;
videoId: number;
text: string;
startMs: number;
endMs: number;
}
interface ResolvedBounds {
lookbackDays: number | null;
minRunLength: number;
maxFrameMs: number;
gapToleranceMs: number;
sampleLimit: number;
}
function resolveBounds(options: DuplicateSubtitleLineCleanupOptions): ResolvedBounds {
const lookbackDays =
typeof options.lookbackDays === 'number' && Number.isFinite(options.lookbackDays)
? Math.max(1, Math.floor(options.lookbackDays))
: null;
const minRunLength =
typeof options.minRunLength === 'number' && Number.isFinite(options.minRunLength)
? Math.max(2, Math.floor(options.minRunLength))
: MIN_TIMING_ONLY_FRAMES;
const maxFrameSeconds =
typeof options.maxFrameSeconds === 'number' && options.maxFrameSeconds > 0
? options.maxFrameSeconds
: ANIMATION_FRAME_MAX_SECONDS;
const sampleLimit =
typeof options.sampleLimit === 'number' && options.sampleLimit >= 0
? Math.floor(options.sampleLimit)
: DEFAULT_SAMPLE_LIMIT;
return {
lookbackDays,
minRunLength,
maxFrameMs: Math.round(maxFrameSeconds * 1000),
gapToleranceMs: Math.round(DUPLICATE_CUE_GAP_TOLERANCE_SECONDS * 1000),
sampleLimit,
};
}
/**
* `CREATED_DATE` holds epoch milliseconds on rows this app wrote, but older and synced
* rows can carry seconds, so normalize before comparing against the cutoff.
*/
const CREATED_MS_SQL = `
CASE
WHEN sl.CREATED_DATE < 10000000000 THEN sl.CREATED_DATE * 1000
ELSE sl.CREATED_DATE
END`;
function readCandidateLines(db: DatabaseSync, bounds: ResolvedBounds): StoredSubtitleLineRow[] {
const scope =
bounds.lookbackDays === null
? ''
: `AND sl.CREATED_DATE IS NOT NULL AND ${CREATED_MS_SQL} >= ?`;
const params = bounds.lookbackDays === null ? [] : [nowMs() - bounds.lookbackDays * MS_PER_DAY];
return db
.prepare(
`SELECT
sl.line_id AS lineId,
sl.session_id AS sessionId,
sl.video_id AS videoId,
sl.text AS text,
sl.segment_start_ms AS startMs,
sl.segment_end_ms AS endMs
FROM imm_subtitle_lines sl
WHERE sl.segment_start_ms IS NOT NULL
AND sl.segment_end_ms IS NOT NULL
${scope}
ORDER BY sl.session_id, sl.video_id, sl.segment_start_ms, sl.line_id`,
)
.all(...params) as StoredSubtitleLineRow[];
}
function isBurst(run: StoredSubtitleLineRow[], bounds: ResolvedBounds): boolean {
if (run.length < bounds.minRunLength) {
return false;
}
const isShortFrame = (row: StoredSubtitleLineRow): boolean =>
row.endMs - row.startMs <= bounds.maxFrameMs;
if (run.every(isShortFrame)) {
return true;
}
// Karaoke commonly finishes its short animation frames with one long hold. Only the
// final event may exceed the frame bound, and the strict short-frame threshold must
// already have been met before it.
return (
run.length - 1 >= bounds.minRunLength &&
run.slice(0, -1).every(isShortFrame) &&
!isShortFrame(run[run.length - 1]!)
);
}
function toBurst(run: StoredSubtitleLineRow[]): DuplicateSubtitleLineBurst {
const [first] = run;
return {
sessionId: first!.sessionId,
videoId: first!.videoId,
text: first!.text,
keptLineId: first!.lineId,
removedLineIds: run.slice(1).map((row) => row.lineId),
startMs: first!.startMs,
endMs: run.reduce((latest, row) => Math.max(latest, row.endMs), first!.endMs),
};
}
/**
* Group stored lines into animation runs.
*
* Runs never cross a session, which is what keeps a rewatch intact: the same episode
* watched twice stores the same line twice, and those two belong to different sessions.
*/
export function findDuplicateSubtitleLineBursts(
rows: readonly StoredSubtitleLineRow[],
options: DuplicateSubtitleLineCleanupOptions = {},
): DuplicateSubtitleLineBurst[] {
const bounds = resolveBounds(options);
const bursts: DuplicateSubtitleLineBurst[] = [];
let run: StoredSubtitleLineRow[] = [];
let chainEndMs = 0;
const closeRun = (): void => {
if (run.length > 1 && isBurst(run, bounds)) {
bursts.push(toBurst(run));
}
run = [];
};
for (const row of rows) {
const previous = run[run.length - 1];
const continuesRun =
previous !== undefined &&
previous.sessionId === row.sessionId &&
previous.videoId === row.videoId &&
previous.text === row.text &&
row.startMs <= chainEndMs + bounds.gapToleranceMs;
if (continuesRun) {
run.push(row);
chainEndMs = Math.max(chainEndMs, row.endMs);
continue;
}
closeRun();
run = [row];
chainEndMs = row.endMs;
}
closeRun();
return bursts;
}
function chunk<T>(values: T[], size: number): T[][] {
const chunks: T[][] = [];
for (let i = 0; i < values.length; i += size) {
chunks.push(values.slice(i, i + size));
}
return chunks;
}
function buildSamples(
db: DatabaseSync,
bursts: DuplicateSubtitleLineBurst[],
sampleLimit: number,
): DuplicateSubtitleLineSample[] {
if (sampleLimit === 0 || bursts.length === 0) {
return [];
}
const largest = [...bursts]
.sort((a, b) => b.removedLineIds.length - a.removedLineIds.length)
.slice(0, sampleLimit);
const videoIds = [...new Set(largest.map((burst) => burst.videoId))];
const titles = new Map<number, string>();
for (const batch of chunk(videoIds, LINE_ID_BATCH_SIZE)) {
const rows = db
.prepare(
`SELECT video_id AS videoId, canonical_title AS title
FROM imm_videos
WHERE video_id IN (${makePlaceholders(batch)})`,
)
.all(...batch) as Array<{ videoId: number; title: string | null }>;
for (const row of rows) {
if (row.title) titles.set(row.videoId, row.title);
}
}
return largest.map((burst) => ({
videoId: burst.videoId,
videoTitle: titles.get(burst.videoId) ?? null,
text: burst.text,
frames: burst.removedLineIds.length + 1,
removedLines: burst.removedLineIds.length,
startMs: burst.startMs,
endMs: burst.endMs,
}));
}
function sumRemovedOccurrences(
db: DatabaseSync,
table: 'imm_word_line_occurrences' | 'imm_kanji_line_occurrences',
lineIds: number[],
): number {
let total = 0;
for (const batch of chunk(lineIds, LINE_ID_BATCH_SIZE)) {
const row = db
.prepare(
`SELECT COALESCE(SUM(occurrence_count), 0) AS total
FROM ${table}
WHERE line_id IN (${makePlaceholders(batch)})`,
)
.get(...batch) as { total: number } | null;
total += row?.total ?? 0;
}
return total;
}
function applyBursts(db: DatabaseSync, bursts: DuplicateSubtitleLineBurst[]): void {
const removedLineIds = bursts.flatMap((burst) => burst.removedLineIds);
const currentMs = toDbTimestamp(nowMs());
db.exec('BEGIN IMMEDIATE');
try {
for (const batch of chunk(removedLineIds, LINE_ID_BATCH_SIZE)) {
const placeholders = makePlaceholders(batch);
// Measured before the delete, applied after it: `applyLexicalRemovals` checks the
// surviving occurrences to decide whether a zeroed count really means the word is
// gone, so the rows it inspects have to be the post-delete ones.
const plan = planLexicalRemovalsForLines(db, batch);
db.prepare(`DELETE FROM imm_word_line_occurrences WHERE line_id IN (${placeholders})`).run(
...batch,
);
db.prepare(`DELETE FROM imm_kanji_line_occurrences WHERE line_id IN (${placeholders})`).run(
...batch,
);
db.prepare(`DELETE FROM imm_subtitle_lines WHERE line_id IN (${placeholders})`).run(...batch);
applyLexicalRemovals(db, plan);
}
const extendStmt = db.prepare(
`UPDATE imm_subtitle_lines
SET segment_end_ms = ?, LAST_UPDATE_DATE = ?
WHERE line_id = ? AND (segment_end_ms IS NULL OR segment_end_ms < ?)`,
);
for (const burst of bursts) {
extendStmt.run(burst.endMs, currentMs, burst.keptLineId, burst.endMs);
}
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
}
/**
* Collapse stored animation bursts down to one line each.
*
* A dry run measures exactly what an apply would remove, using the same scan, so the
* numbers shown in a confirmation prompt are the numbers that will happen.
*/
export function cleanupDuplicateSubtitleLines(
db: DatabaseSync,
options: DuplicateSubtitleLineCleanupOptions = {},
): DuplicateSubtitleLineCleanupSummary {
const bounds = resolveBounds(options);
const dryRun = options.dryRun === true;
const rows = readCandidateLines(db, bounds);
const bursts = findDuplicateSubtitleLineBursts(rows, options);
const removedLineIds = bursts.flatMap((burst) => burst.removedLineIds);
const summary: DuplicateSubtitleLineCleanupSummary = {
dryRun,
lookbackDays: bounds.lookbackDays,
scannedLines: rows.length,
burstGroups: bursts.length,
removedLines: removedLineIds.length,
removedWordOccurrences: sumRemovedOccurrences(db, 'imm_word_line_occurrences', removedLineIds),
removedKanjiOccurrences: sumRemovedOccurrences(
db,
'imm_kanji_line_occurrences',
removedLineIds,
),
samples: buildSamples(db, bursts, bounds.sampleLimit),
};
if (dryRun || removedLineIds.length === 0) {
return summary;
}
applyBursts(db, bursts);
return summary;
}
@@ -268,6 +268,19 @@ export function planLexicalRemovalsForSessions(
return planLexicalRemovals(db, `sl.session_id IN (${makePlaceholders(sessionIds)})`, sessionIds);
}
/**
* Measure what deleting these individual subtitle lines removes from the vocabulary
* tables. Used by the duplicate-line cleanup, which drops animation frames out of the
* middle of sessions that otherwise stay intact.
*/
export function planLexicalRemovalsForLines(
db: DatabaseSync,
lineIds: number[],
): LexicalRemovalPlan {
if (lineIds.length === 0) return EMPTY_LEXICAL_REMOVAL_PLAN;
return planLexicalRemovals(db, `sl.line_id IN (${makePlaceholders(lineIds)})`, lineIds);
}
/** Measure what deleting these videos removes from the vocabulary tables. */
export function planLexicalRemovalsForVideos(
db: DatabaseSync,
+5 -1
View File
@@ -1,5 +1,9 @@
export { Texthooker } from './texthooker';
export { hasMpvWebsocketPlugin, SubtitleWebSocket } from './subtitle-ws';
export {
hasMpvWebsocketPlugin,
isSubtitleAnnotationUpgrade,
SubtitleWebSocket,
} from './subtitle-ws';
export { registerGlobalShortcuts } from './shortcut';
export { createIpcDepsRuntime, registerIpcHandlers } from './ipc';
export { shortcutMatchesInputForLocalFallback } from './shortcut-fallback';
+15
View File
@@ -1,6 +1,7 @@
import electron from 'electron';
import type { BrowserWindow as ElectronBrowserWindow, IpcMainEvent } from 'electron';
import type {
ChangelogSnapshot,
CompiledSessionBinding,
ControllerConfigUpdate,
PlaylistBrowserMutationResult,
@@ -122,6 +123,7 @@ export interface IpcServiceDeps {
removeCharacterDictionaryManagedEntry?: (mediaId: number) => Promise<unknown>;
moveCharacterDictionaryManagedEntry?: (mediaId: number, direction: 1 | -1) => Promise<unknown>;
appendClipboardVideoToQueue: () => { ok: boolean; message: string };
getChangelogSnapshot?: (options?: { refresh?: boolean }) => Promise<ChangelogSnapshot>;
getPlaylistBrowserSnapshot: () => Promise<PlaylistBrowserSnapshot>;
appendPlaylistBrowserFile: (filePath: string) => Promise<PlaylistBrowserMutationResult>;
playPlaylistBrowserIndex: (index: number) => Promise<PlaylistBrowserMutationResult>;
@@ -297,6 +299,7 @@ export interface IpcDepsRuntimeOptions {
removeCharacterDictionaryManagedEntry?: (mediaId: number) => Promise<unknown>;
moveCharacterDictionaryManagedEntry?: (mediaId: number, direction: 1 | -1) => Promise<unknown>;
appendClipboardVideoToQueue: () => { ok: boolean; message: string };
getChangelogSnapshot?: (options?: { refresh?: boolean }) => Promise<ChangelogSnapshot>;
getPlaylistBrowserSnapshot: () => Promise<PlaylistBrowserSnapshot>;
appendPlaylistBrowserFile: (filePath: string) => Promise<PlaylistBrowserMutationResult>;
playPlaylistBrowserIndex: (index: number) => Promise<PlaylistBrowserMutationResult>;
@@ -418,6 +421,7 @@ export function createIpcDepsRuntime(options: IpcDepsRuntimeOptions): IpcService
entries: [],
})),
appendClipboardVideoToQueue: options.appendClipboardVideoToQueue,
getChangelogSnapshot: options.getChangelogSnapshot,
getPlaylistBrowserSnapshot: options.getPlaylistBrowserSnapshot,
appendPlaylistBrowserFile: options.appendPlaylistBrowserFile,
playPlaylistBrowserIndex: options.playPlaylistBrowserIndex,
@@ -820,6 +824,17 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar
return deps.appendClipboardVideoToQueue();
});
ipc.handle(IPC_CHANNELS.request.getChangelogSnapshot, async (_event, payload: unknown) => {
const refresh =
typeof payload === 'object' && payload !== null && 'refresh' in payload
? (payload as { refresh?: unknown }).refresh === true
: false;
if (!deps.getChangelogSnapshot) {
throw new Error('Changelog service is unavailable.');
}
return await deps.getChangelogSnapshot({ refresh });
});
ipc.handle(IPC_CHANNELS.request.getPlaylistBrowserSnapshot, async () => {
return await deps.getPlaylistBrowserSnapshot();
});
+2 -2
View File
@@ -205,7 +205,7 @@ test('runStartupBootstrapRuntime skips lifecycle when generate-config flow handl
assert.deepEqual(calls, ['setLog:warn:cli', 'forceX11', 'enforceWayland']);
});
test('runStartupBootstrapRuntime enables quiet background mode by default', () => {
test('runStartupBootstrapRuntime lets config govern background log level by default', () => {
const calls: string[] = [];
const args = makeArgs({ background: true });
@@ -222,7 +222,7 @@ test('runStartupBootstrapRuntime enables quiet background mode by default', () =
});
assert.equal(result.backgroundMode, true);
assert.deepEqual(calls, ['setLog:warn:cli', 'forceX11', 'enforceWayland', 'startLifecycle']);
assert.deepEqual(calls, ['forceX11', 'enforceWayland', 'startLifecycle']);
});
test('runStartupBootstrapRuntime enables quiet update mode by default', () => {
+1 -1
View File
@@ -45,7 +45,7 @@ export function runStartupBootstrapRuntime(
if (initialArgs.logLevel) {
deps.setLogLevel(initialArgs.logLevel, 'cli');
} else if (initialArgs.background || initialArgs.update) {
} else if (initialArgs.update) {
deps.setLogLevel('warn', 'cli');
}
@@ -5,6 +5,7 @@ import {
buildSentenceSearchOptions,
enrichSessionsWithKnownWordMetrics,
parseBooleanQuery,
parseDuplicateLineCleanupBody,
parseExcludedWordsBody,
parseIntQuery,
} from './route-support.js';
@@ -40,6 +41,19 @@ export function registerStatsLibraryRoutes(
return c.json(statsJson('setExcludedWords', { ok: true }));
});
// Collapse animation bursts older versions recorded frame by frame. `dryRun` measures
// the same scan without writing, so the confirmation the user sees is the real cost.
app.post('/api/stats/maintenance/duplicate-lines', async (c) => {
const contentType = c.req.header('content-type')?.split(';', 1)[0]?.trim().toLowerCase();
if (contentType !== 'application/json') return c.body(null, 415);
const body = await c.req.json().catch(() => null);
const options = parseDuplicateLineCleanupBody(body);
if (!options) return c.body(null, 400);
const { dryRun, lookbackDays } = options;
const result = await tracker.cleanupDuplicateSubtitleLines({ dryRun, lookbackDays });
return c.json(statsJson('duplicateLineCleanup', result));
});
app.get('/api/stats/vocabulary/occurrences', async (c) => {
const headword = (c.req.query('headword') ?? '').trim();
const word = (c.req.query('word') ?? '').trim();
@@ -88,6 +88,35 @@ export function parseExcludedWordsBody(body: unknown): StatsExcludedWord[] | nul
return words;
}
/**
* Read a duplicate-line cleanup request. An explicit object with no lookback scans all
* history. Invalid bodies and invalid windows are rejected instead of broadening scope.
*/
export function parseDuplicateLineCleanupBody(body: unknown): {
dryRun: boolean;
lookbackDays: number | null;
} | null {
if (!body || typeof body !== 'object' || Array.isArray(body)) {
return null;
}
const source = body as Record<string, unknown>;
if (source.dryRun !== undefined && typeof source.dryRun !== 'boolean') {
return null;
}
const rawLookback = source.lookbackDays;
if (
rawLookback !== undefined &&
rawLookback !== null &&
(typeof rawLookback !== 'number' || !Number.isFinite(rawLookback) || rawLookback < 1)
) {
return null;
}
return {
dryRun: source.dryRun === true,
lookbackDays: typeof rawLookback === 'number' ? Math.floor(rawLookback) : null,
};
}
export function loadKnownWordsSet(cachePath: string | undefined): Set<string> | null {
if (!cachePath || !existsSync(cachePath)) return null;
try {
+397 -22
View File
@@ -8,6 +8,7 @@ import {
runSubsyncManual,
triggerSubsyncFromConfig,
} from './subsync';
import type { SubsyncManualPayload } from '../../types';
function makeDeps(
overrides: Partial<TriggerSubsyncFromConfigDeps> = {},
@@ -76,7 +77,7 @@ test('triggerSubsyncFromConfig opens manual picker', async () => {
await triggerSubsyncFromConfig(
makeDeps({
openManualPicker: (payload) => {
payloadTrackCount = payload.sourceTracks.length;
payloadTrackCount = payload.subtitleTracks.length;
ffsubsyncAvailable = payload.ffsubsyncAvailable;
},
showMpvOsd: (text) => {
@@ -88,9 +89,9 @@ test('triggerSubsyncFromConfig opens manual picker', async () => {
}),
);
assert.equal(payloadTrackCount, 1);
assert.equal(payloadTrackCount, 2);
assert.equal(ffsubsyncAvailable, true);
assert.ok(osd.includes('Subsync: choose engine and source'));
assert.ok(osd.includes('Subsync: choose engine and subtitles'));
assert.equal(inProgressState, false);
});
@@ -140,7 +141,7 @@ test('triggerSubsyncFromConfig does not run automatic sync', async () => {
await triggerSubsyncFromConfig(
makeDeps({
openManualPicker: (payload) => {
payloadTrackCount = payload.sourceTracks.length;
payloadTrackCount = payload.subtitleTracks.length;
},
showMpvOsd: (text) => {
osd.push(text);
@@ -152,9 +153,9 @@ test('triggerSubsyncFromConfig does not run automatic sync', async () => {
}),
);
assert.equal(payloadTrackCount, 1);
assert.equal(payloadTrackCount, 2);
assert.equal(spinnerRan, false);
assert.deepEqual(osd, ['Subsync: choose engine and source']);
assert.deepEqual(osd, ['Subsync: choose engine and subtitles']);
});
test('triggerSubsyncFromConfig dedupes repeated subtitle source tracks', async () => {
@@ -195,12 +196,71 @@ test('triggerSubsyncFromConfig dedupes repeated subtitle source tracks', async (
},
}),
openManualPicker: (payload) => {
payloadTrackCount = payload.sourceTracks.length;
payloadTrackCount = payload.subtitleTracks.length;
},
}),
);
assert.equal(payloadTrackCount, 1);
assert.equal(payloadTrackCount, 2);
});
test('triggerSubsyncFromConfig keeps both active tracks when they share a file', async () => {
let payload: SubsyncManualPayload | null = null;
await triggerSubsyncFromConfig(
makeDeps({
getMpvClient: () => ({
connected: true,
currentAudioStreamIndex: null,
send: () => {},
requestProperty: async (name: string) => {
if (name === 'path') return '/tmp/video.mkv';
if (name === 'sid') return 1;
if (name === 'secondary-sid') return 2;
if (name === 'track-list') {
// mpv appends a duplicate entry when the same file is re-added, so
// the primary and secondary slots can point at one path.
return [
{
id: 1,
type: 'sub',
selected: true,
external: true,
'external-filename': '/tmp/ref.srt',
},
{
id: 2,
type: 'sub',
selected: true,
external: true,
'external-filename': '/tmp/ref.srt',
},
{
id: 3,
type: 'sub',
selected: false,
external: true,
'external-filename': '/tmp/ref.srt',
},
];
}
return null;
},
}),
openManualPicker: (nextPayload) => {
payload = nextPayload;
},
}),
);
assert.ok(payload);
const resolved = payload as SubsyncManualPayload;
assert.deepEqual(
resolved.subtitleTracks.map((track) => track.id),
[1, 2],
);
assert.equal(resolved.defaultReferenceTrackId, 2);
assert.equal(resolved.defaultTargetTrackId, 1);
});
test('triggerSubsyncFromConfig reports failures to OSD', async () => {
@@ -217,15 +277,157 @@ test('triggerSubsyncFromConfig reports failures to OSD', async () => {
assert.ok(osd.some((line) => line.startsWith('Subsync failed: MPV not connected')));
});
test('runSubsyncManual requires a source track for alass', async () => {
const result = await runSubsyncManual({ engine: 'alass', sourceTrackId: null }, makeDeps());
test('runSubsyncManual requires a reference track for alass', async () => {
const result = await runSubsyncManual({ engine: 'alass', referenceTrackId: null }, makeDeps());
assert.deepEqual(result, {
ok: false,
message: 'Select a subtitle source track for alass',
message: 'Select a reference subtitle track for alass',
});
});
test('runSubsyncManual rejects alass when reference and target are the same track', async () => {
const result = await runSubsyncManual(
{ engine: 'alass', referenceTrackId: 2, targetTrackId: 2 },
makeDeps(),
);
assert.deepEqual(result, {
ok: false,
message: 'Reference and out-of-sync subtitles must be different tracks',
});
});
test('runSubsyncManual rejects an unknown target track', async () => {
const result = await runSubsyncManual(
{ engine: 'alass', referenceTrackId: 2, targetTrackId: 99 },
makeDeps(),
);
assert.deepEqual(result, {
ok: false,
message: 'Select the out-of-sync subtitle track to retime',
});
});
test('runSubsyncManual rejects the video reference for remote media', async () => {
const result = await runSubsyncManual(
{ engine: 'alass', referenceMode: 'video' },
makeDeps({
getMpvClient: () => ({
connected: true,
currentAudioStreamIndex: null,
send: () => {},
requestProperty: async (name: string) => {
if (name === 'path') return 'https://jellyfin.example/Videos/movie/stream.mkv';
if (name === 'sid') return 1;
if (name === 'secondary-sid') return null;
if (name === 'track-list') {
return [{ id: 1, type: 'sub', selected: true, lang: 'jpn' }];
}
return null;
},
}),
}),
);
assert.equal(result.ok, false);
assert.match(result.message, /cannot use a stream URL as reference/);
});
test('openSubsyncManualPicker defaults the reference to the secondary subtitle track', async () => {
let payload: SubsyncManualPayload | null = null;
await triggerSubsyncFromConfig(
makeDeps({
getMpvClient: () => ({
connected: true,
currentAudioStreamIndex: null,
send: () => {},
requestProperty: async (name: string) => {
if (name === 'path') return '/tmp/video.mkv';
if (name === 'sid') return 1;
if (name === 'secondary-sid') return 3;
if (name === 'track-list') {
return [
{ id: 1, type: 'sub', selected: true, lang: 'jpn' },
{
id: 2,
type: 'sub',
selected: false,
external: true,
lang: 'eng',
'external-filename': '/tmp/other.srt',
},
{
id: 3,
type: 'sub',
selected: true,
external: true,
lang: 'eng',
'external-filename': '/tmp/secondary.srt',
},
];
}
return null;
},
}),
openManualPicker: (nextPayload) => {
payload = nextPayload;
},
}),
);
assert.ok(payload);
const resolved = payload as SubsyncManualPayload;
assert.deepEqual(
resolved.subtitleTracks.map((track) => track.id),
[1, 2, 3],
);
assert.equal(resolved.defaultReferenceTrackId, 3);
assert.equal(resolved.defaultTargetTrackId, 1);
assert.equal(resolved.videoReferenceAvailable, true);
});
test('openSubsyncManualPicker never defaults to a reference missing from the track list', async () => {
let payload: SubsyncManualPayload | null = null;
await triggerSubsyncFromConfig(
makeDeps({
getMpvClient: () => ({
connected: true,
currentAudioStreamIndex: null,
send: () => {},
requestProperty: async (name: string) => {
if (name === 'path') return '/tmp/video.mkv';
if (name === 'sid') return 1;
if (name === 'secondary-sid') return 2;
if (name === 'track-list') {
return [
{ id: 1, type: 'sub', selected: true, lang: 'jpn' },
// Secondary track with no usable file path: filtered out of the picker.
{ id: 2, type: 'sub', selected: true, external: true, 'external-filename': '' },
{ id: 3, type: 'sub', selected: false, lang: 'eng' },
];
}
return null;
},
}),
openManualPicker: (nextPayload) => {
payload = nextPayload;
},
}),
);
assert.ok(payload);
const resolved = payload as SubsyncManualPayload;
assert.deepEqual(
resolved.subtitleTracks.map((track) => track.id),
[1, 3],
);
assert.equal(resolved.defaultReferenceTrackId, 3);
});
test('triggerSubsyncFromConfig does not validate sync tool paths before manual selection', async () => {
const osd: string[] = [];
const inProgress: boolean[] = [];
@@ -242,7 +444,7 @@ test('triggerSubsyncFromConfig does not validate sync tool paths before manual s
inProgress.push(value);
},
openManualPicker: (payload) => {
payloadTrackCount = payload.sourceTracks.length;
payloadTrackCount = payload.subtitleTracks.length;
},
showMpvOsd: (text) => {
osd.push(text);
@@ -251,8 +453,8 @@ test('triggerSubsyncFromConfig does not validate sync tool paths before manual s
);
assert.deepEqual(inProgress, [false]);
assert.equal(payloadTrackCount, 1);
assert.deepEqual(osd, ['Subsync: choose engine and source']);
assert.equal(payloadTrackCount, 2);
assert.deepEqual(osd, ['Subsync: choose engine and subtitles']);
});
function writeExecutableScript(filePath: string, content: string): void {
@@ -333,7 +535,7 @@ test('runSubsyncManual constructs ffsubsync command and returns success', async
}),
});
const result = await runSubsyncManual({ engine: 'ffsubsync', sourceTrackId: null }, deps);
const result = await runSubsyncManual({ engine: 'ffsubsync' }, deps);
assert.equal(result.ok, true);
assert.equal(result.message, 'Subtitle synchronized with ffsubsync');
@@ -346,7 +548,7 @@ test('runSubsyncManual constructs ffsubsync command and returns success', async
const ffOutputFlagIndex = ffArgs.indexOf('-o');
assert.equal(ffOutputFlagIndex >= 0, true);
assert.equal(ffArgs[ffOutputFlagIndex + 1], toShellPath(primaryPath));
assert.equal(sentCommands[0]?.[0], 'sub_add');
assert.equal(sentCommands[0]?.[0], 'sub-add');
assert.deepEqual(sentCommands[1], ['set_property', 'sub-delay', 0]);
});
@@ -399,7 +601,7 @@ test('runSubsyncManual writes deterministic _retimed filename when replace is fa
}),
});
const result = await runSubsyncManual({ engine: 'ffsubsync', sourceTrackId: null }, deps);
const result = await runSubsyncManual({ engine: 'ffsubsync' }, deps);
assert.equal(result.ok, true);
const ffArgs = fs.readFileSync(ffsubsyncLogPath, 'utf8').trim().split('\n');
@@ -453,7 +655,7 @@ test('runSubsyncManual reports ffsubsync command failures with details', async (
}),
});
const result = await runSubsyncManual({ engine: 'ffsubsync', sourceTrackId: null }, deps);
const result = await runSubsyncManual({ engine: 'ffsubsync' }, deps);
assert.equal(result.ok, false);
assert.equal(result.message.startsWith('ffsubsync synchronization failed'), true);
@@ -518,7 +720,7 @@ test('runSubsyncManual constructs alass command and returns failure on non-zero
}),
});
const result = await runSubsyncManual({ engine: 'alass', sourceTrackId: 2 }, deps);
const result = await runSubsyncManual({ engine: 'alass', referenceTrackId: 2 }, deps);
assert.equal(result.ok, false);
assert.equal(typeof result.message, 'string');
@@ -528,6 +730,179 @@ test('runSubsyncManual constructs alass command and returns failure on non-zero
assert.equal(alassArgs[1], toShellPath(primaryPath));
});
function makeAlassSelectionDeps(tmpDir: string): {
deps: TriggerSubsyncFromConfigDeps;
alassLogPath: string;
videoPath: string;
primaryPath: string;
sourcePath: string;
sentCommands: Array<Array<string | number>>;
} {
const alassLogPath = path.join(tmpDir, 'alass-args.log');
const alassPath = path.join(tmpDir, 'alass.sh');
const ffmpegPath = path.join(tmpDir, 'ffmpeg.sh');
const ffsubsyncPath = path.join(tmpDir, 'ffsubsync.sh');
const videoPath = path.join(tmpDir, 'video.mkv');
const primaryPath = path.join(tmpDir, 'primary.srt');
const sourcePath = path.join(tmpDir, 'source.srt');
fs.writeFileSync(videoPath, 'video');
fs.writeFileSync(primaryPath, 'sub');
fs.writeFileSync(sourcePath, 'sub2');
writeExecutableScript(ffmpegPath, '#!/bin/sh\nexit 0\n');
writeExecutableScript(ffsubsyncPath, '#!/bin/sh\nexit 0\n');
writeExecutableScript(
alassPath,
`#!/bin/sh\n: > "${toShellPath(alassLogPath)}"\nfor arg in "$@"; do printf '%s\\n' "$arg" >> "${toShellPath(alassLogPath)}"; done\n: > "$3"\nexit 0\n`,
);
const trackList: Array<Record<string, unknown>> = [
{ id: 1, type: 'sub', selected: true, external: true, 'external-filename': primaryPath },
{ id: 2, type: 'sub', selected: true, external: true, 'external-filename': sourcePath },
];
const sentCommands: Array<Array<string | number>> = [];
const deps = makeDeps({
getMpvClient: () => ({
connected: true,
currentAudioStreamIndex: null,
send: (payload) => {
sentCommands.push(payload.command);
if (payload.command[0] === 'sub-add' || payload.command[0] === 'sub_add') {
trackList.push({
id: trackList.length + 1,
type: 'sub',
selected: false,
external: true,
'external-filename': payload.command[1],
});
}
},
requestProperty: async (name: string) => {
if (name === 'path') return videoPath;
if (name === 'sid') return 1;
if (name === 'secondary-sid') return 2;
if (name === 'track-list') return trackList;
return null;
},
}),
getResolvedConfig: () => ({
alassPath,
ffsubsyncPath,
ffmpegPath,
replace: false,
}),
});
return { deps, alassLogPath, videoPath, primaryPath, sourcePath, sentCommands };
}
test('runSubsyncManual uses the video file as alass reference when requested', async () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subsync-alass-video-ref-'));
const { deps, alassLogPath, videoPath, primaryPath } = makeAlassSelectionDeps(tmpDir);
const result = await runSubsyncManual({ engine: 'alass', referenceMode: 'video' }, deps);
assert.equal(result.ok, true);
const alassArgs = fs.readFileSync(alassLogPath, 'utf8').trim().split('\n');
assert.equal(alassArgs[0], toShellPath(videoPath));
assert.equal(alassArgs[1], toShellPath(primaryPath));
});
test('runSubsyncManual retimes the selected target track instead of the primary', async () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subsync-alass-target-'));
const { deps, alassLogPath, primaryPath, sourcePath, sentCommands } =
makeAlassSelectionDeps(tmpDir);
const result = await runSubsyncManual(
{ engine: 'alass', referenceTrackId: 1, targetTrackId: 2 },
deps,
);
assert.equal(result.ok, true);
const alassArgs = fs.readFileSync(alassLogPath, 'utf8').trim().split('\n');
assert.equal(alassArgs[0], toShellPath(primaryPath));
assert.equal(alassArgs[1], toShellPath(sourcePath));
assert.equal(sentCommands[0]?.[0], 'sub-add');
assert.equal(sentCommands[0]?.[2], 'auto');
});
test('runSubsyncManual keeps a retimed secondary track in the secondary slot', async () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subsync-alass-secondary-slot-'));
const alassPath = path.join(tmpDir, 'alass.sh');
const ffmpegPath = path.join(tmpDir, 'ffmpeg.sh');
const ffsubsyncPath = path.join(tmpDir, 'ffsubsync.sh');
const videoPath = path.join(tmpDir, 'video.mkv');
const primaryPath = path.join(tmpDir, 'ja.srt');
const secondaryPath = path.join(tmpDir, 'en.srt');
const retimedPath = path.join(tmpDir, 'en_retimed.srt');
fs.writeFileSync(videoPath, 'video');
fs.writeFileSync(primaryPath, 'ja');
fs.writeFileSync(secondaryPath, 'en');
writeExecutableScript(ffmpegPath, '#!/bin/sh\nexit 0\n');
writeExecutableScript(ffsubsyncPath, '#!/bin/sh\nexit 0\n');
writeExecutableScript(alassPath, '#!/bin/sh\n: > "$3"\nexit 0\n');
const trackList: Array<Record<string, unknown>> = [
{ id: 1, type: 'sub', selected: true, external: true, 'external-filename': primaryPath },
{ id: 2, type: 'sub', selected: true, external: true, 'external-filename': secondaryPath },
];
const sentCommands: Array<Array<string | number>> = [];
const deps = makeDeps({
getMpvClient: () => ({
connected: true,
currentAudioStreamIndex: null,
send: (payload) => {
sentCommands.push(payload.command);
if (payload.command[0] === 'sub-add') {
trackList.push({
id: 3,
type: 'sub',
selected: false,
external: true,
'external-filename': payload.command[1],
});
}
},
requestProperty: async (name: string) => {
if (name === 'path') return videoPath;
if (name === 'sid') return 1;
if (name === 'secondary-sid') return 2;
if (name === 'track-list') return trackList;
return null;
},
}),
getResolvedConfig: () => ({
alassPath,
ffsubsyncPath,
ffmpegPath,
replace: false,
}),
});
const result = await runSubsyncManual(
{ engine: 'alass', referenceTrackId: 1, targetTrackId: 2 },
deps,
);
assert.equal(result.ok, true);
assert.deepEqual(sentCommands[0], ['sub-add', retimedPath, 'auto']);
assert.deepEqual(sentCommands[1], ['set_property', 'secondary-sub-delay', 0]);
assert.deepEqual(sentCommands[2], ['set_property', 'secondary-sid', 3]);
assert.equal(
sentCommands.some((command) => command[1] === 'sub-delay'),
false,
);
assert.equal(
sentCommands.some((command) => command[1] === 'sid'),
false,
);
assert.equal(
sentCommands.some((command) => command[1] === 'sid'),
false,
);
});
test('runSubsyncManual keeps internal alass source file alive until sync finishes', async () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subsync-alass-internal-source-'));
const alassPath = path.join(tmpDir, 'alass.sh');
@@ -589,11 +964,11 @@ test('runSubsyncManual keeps internal alass source file alive until sync finishe
}),
});
const result = await runSubsyncManual({ engine: 'alass', sourceTrackId: 2 }, deps);
const result = await runSubsyncManual({ engine: 'alass', referenceTrackId: 2 }, deps);
assert.equal(result.ok, true);
assert.equal(result.message, 'Subtitle synchronized with alass');
assert.equal(sentCommands[0]?.[0], 'sub_add');
assert.equal(sentCommands[0]?.[0], 'sub-add');
assert.deepEqual(sentCommands[1], ['set_property', 'sub-delay', 0]);
});
@@ -645,7 +1020,7 @@ test('runSubsyncManual resolves string sid values from mpv stream properties', a
}),
});
const result = await runSubsyncManual({ engine: 'ffsubsync', sourceTrackId: null }, deps);
const result = await runSubsyncManual({ engine: 'ffsubsync' }, deps);
assert.equal(result.ok, true);
assert.equal(result.message, 'Subtitle synchronized with ffsubsync');
+201 -40
View File
@@ -21,6 +21,11 @@ interface FileExtractionResult {
temporary: boolean;
}
type SubtitleSlot = 'primary' | 'secondary';
const SYNCED_TRACK_LOOKUP_ATTEMPTS = 5;
const SYNCED_TRACK_LOOKUP_RETRY_MS = 100;
function summarizeCommandFailure(command: string, result: CommandResult): string {
const parts = [
`code=${result.code ?? 'n/a'}`,
@@ -90,16 +95,28 @@ function getSourceTrackIdentity(track: MpvTrack): string {
return 'unknown';
}
function dedupeSourceTracks(tracks: MpvTrack[]): MpvTrack[] {
const deduped = new Map<string, MpvTrack>();
function isPinned(track: MpvTrack, pinnedIds: Set<number>): boolean {
return typeof track.id === 'number' && pinnedIds.has(track.id);
}
// Pinned tracks (the active primary/secondary) always survive, even when two of
// them point at the same file; only unpinned duplicates are collapsed.
function dedupeSubtitleTracks(tracks: MpvTrack[], pinnedIds: Set<number>): MpvTrack[] {
const pinnedIdentities = new Set(
tracks.filter((track) => isPinned(track, pinnedIds)).map(getSourceTrackIdentity),
);
const winners = new Map<string, MpvTrack>();
for (const track of tracks) {
if (isPinned(track, pinnedIds)) continue;
const identity = getSourceTrackIdentity(track);
const existing = deduped.get(identity);
if (pinnedIdentities.has(identity)) continue;
const existing = winners.get(identity);
if (!existing || (track.selected && !existing.selected)) {
deduped.set(identity, track);
winners.set(identity, track);
}
}
return [...deduped.values()];
const kept = new Set(winners.values());
return tracks.filter((track) => isPinned(track, pinnedIds) || kept.has(track));
}
export interface TriggerSubsyncFromConfigDeps extends SubsyncCoreDeps {
@@ -142,20 +159,21 @@ async function gatherSubsyncContext(client: MpvClientLike): Promise<SubsyncConte
}
const secondaryTrack = subtitleTracks.find((track) => track.id === secondarySid) ?? null;
const sourceTracks = subtitleTracks
.filter((track) => track.id !== sid)
.filter((track) => {
if (!track.external) return true;
const filename = track['external-filename'];
return typeof filename === 'string' && filename.length > 0;
});
const uniqueSourceTracks = dedupeSourceTracks(sourceTracks);
const usableTracks = subtitleTracks.filter((track) => {
if (typeof track.id !== 'number') return false;
if (!track.external) return true;
const filename = track['external-filename'];
return typeof filename === 'string' && filename.length > 0;
});
return {
videoPath,
primaryTrack,
secondaryTrack,
sourceTracks: uniqueSourceTracks,
subtitleTracks: dedupeSubtitleTracks(
usableTracks,
new Set([sid, secondarySid].filter((id): id is number => typeof id === 'number')),
),
audioStreamIndex: client.currentAudioStreamIndex,
};
}
@@ -271,41 +289,104 @@ async function runFfsubsyncSync(
return runCommand(ffsubsyncPath, args);
}
function loadSyncedSubtitle(client: MpvClientLike, pathToLoad: string): void {
function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
// mpv may echo the path back with different separators, and Windows paths are
// case-insensitive, so compare normalized forms instead of raw strings.
function normalizeSubtitlePathForCompare(value: string): string {
const normalized = value.replace(/\\/g, '/');
return process.platform === 'win32' ? normalized.toLowerCase() : normalized;
}
async function findAddedSubtitleTrackId(
client: MpvClientLike,
pathToLoad: string,
): Promise<number | null> {
const wanted = normalizeSubtitlePathForCompare(pathToLoad);
// sub-add is queued, so the track may not appear in the first track-list reply.
for (let attempt = 0; attempt < SYNCED_TRACK_LOOKUP_ATTEMPTS; attempt += 1) {
let tracks: MpvTrack[] = [];
try {
const trackListRaw = await client.requestProperty('track-list');
tracks = Array.isArray(trackListRaw) ? normalizeTrackIds(trackListRaw as MpvTrack[]) : [];
} catch {
return null;
}
// Re-adding a file mpv already knows appends a duplicate entry; the newest
// one holds the retimed content, so prefer the last match.
const matches = tracks.filter((track) => {
if (track.type !== 'sub') return false;
const filename = track['external-filename'];
return typeof filename === 'string' && normalizeSubtitlePathForCompare(filename) === wanted;
});
const added = matches[matches.length - 1];
if (added && typeof added.id === 'number') {
return added.id;
}
if (attempt < SYNCED_TRACK_LOOKUP_ATTEMPTS - 1) {
await delay(SYNCED_TRACK_LOOKUP_RETRY_MS);
}
}
return null;
}
async function loadSyncedSubtitle(
client: MpvClientLike,
pathToLoad: string,
slot: SubtitleSlot,
): Promise<void> {
if (!client.connected) {
throw new Error('MPV disconnected while loading subtitle');
}
client.send({ command: ['sub_add', pathToLoad] });
if (slot === 'secondary') {
// Keep the primary track untouched: load without selecting, then point
// secondary-sid at the freshly added track.
client.send({ command: ['sub-add', pathToLoad, 'auto'] });
client.send({ command: ['set_property', 'secondary-sub-delay', 0] });
const addedTrackId = await findAddedSubtitleTrackId(client, pathToLoad);
if (addedTrackId === null) {
throw new Error('Synchronized subtitle did not appear in the mpv track list');
}
client.send({ command: ['set_property', 'secondary-sid', addedTrackId] });
return;
}
client.send({ command: ['sub-add', pathToLoad] });
client.send({ command: ['set_property', 'sub-delay', 0] });
}
async function subsyncToReference(
engine: 'alass' | 'ffsubsync',
referenceFilePath: string,
targetTrack: MpvTrack,
context: SubsyncContext,
resolved: SubsyncResolvedConfig,
client: MpvClientLike,
slot: SubtitleSlot,
): Promise<SubsyncResult> {
const ffmpegPath = ensureExecutablePath(resolved.ffmpegPath, 'ffmpeg');
const primaryExtraction = await extractSubtitleTrackToFile(
const targetExtraction = await extractSubtitleTrackToFile(
ffmpegPath,
context.videoPath,
context.primaryTrack,
targetTrack,
);
const replacePrimary = resolved.replace !== false && !primaryExtraction.temporary;
const outputPath = buildRetimedPath(primaryExtraction.path, replacePrimary);
const replaceTarget = resolved.replace !== false && !targetExtraction.temporary;
const outputPath = buildRetimedPath(targetExtraction.path, replaceTarget);
try {
let result: CommandResult;
if (engine === 'alass') {
const alassPath = ensureExecutablePath(resolved.alassPath, 'alass');
result = await runAlassSync(alassPath, referenceFilePath, primaryExtraction.path, outputPath);
result = await runAlassSync(alassPath, referenceFilePath, targetExtraction.path, outputPath);
} else {
const ffsubsyncPath = ensureExecutablePath(resolved.ffsubsyncPath, 'ffsubsync');
result = await runFfsubsyncSync(
ffsubsyncPath,
context.videoPath,
primaryExtraction.path,
targetExtraction.path,
outputPath,
context.audioStreamIndex,
);
@@ -319,13 +400,13 @@ async function subsyncToReference(
};
}
loadSyncedSubtitle(client, outputPath);
await loadSyncedSubtitle(client, outputPath, slot);
return {
ok: true,
message: `Subtitle synchronized with ${engine}`,
};
} finally {
cleanupTemporaryFile(primaryExtraction);
cleanupTemporaryFile(targetExtraction);
}
}
@@ -337,6 +418,25 @@ function validateFfsubsyncReference(videoPath: string): void {
}
}
function resolveTargetTrack(
request: SubsyncManualRunRequest,
context: SubsyncContext,
): MpvTrack | null {
if (request.targetTrackId === undefined || request.targetTrackId === null) {
return context.primaryTrack;
}
return getTrackById(context.subtitleTracks, request.targetTrackId);
}
// Retiming the secondary track must not steal the primary slot: the synced file
// goes back where the out-of-sync one was.
function resolveTargetSlot(targetTrack: MpvTrack, context: SubsyncContext): SubtitleSlot {
if (typeof targetTrack.id !== 'number') return 'primary';
if (targetTrack.id === context.primaryTrack.id) return 'primary';
if (context.secondaryTrack && targetTrack.id === context.secondaryTrack.id) return 'secondary';
return 'primary';
}
export async function runSubsyncManual(
request: SubsyncManualRunRequest,
deps: SubsyncCoreDeps,
@@ -345,6 +445,12 @@ export async function runSubsyncManual(
const context = await gatherSubsyncContext(client);
const resolved = deps.getResolvedConfig();
const targetTrack = resolveTargetTrack(request, context);
if (!targetTrack) {
return { ok: false, message: 'Select the out-of-sync subtitle track to retime' };
}
const targetSlot = resolveTargetSlot(targetTrack, context);
if (request.engine === 'ffsubsync') {
try {
validateFfsubsyncReference(context.videoPath);
@@ -354,22 +460,64 @@ export async function runSubsyncManual(
message: `ffsubsync synchronization failed: ${(error as Error).message}`,
};
}
return subsyncToReference('ffsubsync', context.videoPath, context, resolved, client);
return subsyncToReference(
'ffsubsync',
context.videoPath,
targetTrack,
context,
resolved,
client,
targetSlot,
);
}
const sourceTrack = getTrackById(context.sourceTracks, request.sourceTrackId ?? null);
if (!sourceTrack) {
return { ok: false, message: 'Select a subtitle source track for alass' };
if (request.referenceMode === 'video') {
if (isRemoteMediaPath(context.videoPath)) {
return {
ok: false,
message:
'alass cannot use a stream URL as reference. Pick a reference subtitle track instead.',
};
}
return subsyncToReference(
'alass',
context.videoPath,
targetTrack,
context,
resolved,
client,
targetSlot,
);
}
const referenceTrack = getTrackById(context.subtitleTracks, request.referenceTrackId ?? null);
if (!referenceTrack) {
return { ok: false, message: 'Select a reference subtitle track for alass' };
}
if (referenceTrack.id === targetTrack.id) {
return { ok: false, message: 'Reference and out-of-sync subtitles must be different tracks' };
}
const ffmpegPath = ensureExecutablePath(resolved.ffmpegPath, 'ffmpeg');
let sourceExtraction: FileExtractionResult | null = null;
let referenceExtraction: FileExtractionResult | null = null;
try {
sourceExtraction = await extractSubtitleTrackToFile(ffmpegPath, context.videoPath, sourceTrack);
return await subsyncToReference('alass', sourceExtraction.path, context, resolved, client);
referenceExtraction = await extractSubtitleTrackToFile(
ffmpegPath,
context.videoPath,
referenceTrack,
);
return await subsyncToReference(
'alass',
referenceExtraction.path,
targetTrack,
context,
resolved,
client,
targetSlot,
);
} finally {
if (sourceExtraction) {
cleanupTemporaryFile(sourceExtraction);
if (referenceExtraction) {
cleanupTemporaryFile(referenceExtraction);
}
}
}
@@ -377,14 +525,27 @@ export async function runSubsyncManual(
export async function openSubsyncManualPicker(deps: TriggerSubsyncFromConfigDeps): Promise<void> {
const client = getMpvClientForSubsync(deps);
const context = await gatherSubsyncContext(client);
const subtitleTracks = context.subtitleTracks
.filter((track) => typeof track.id === 'number')
.map((track) => ({
id: track.id as number,
label: formatTrackLabel(track),
}));
const primaryTrackId =
typeof context.primaryTrack.id === 'number' ? context.primaryTrack.id : null;
const secondaryTrackId =
typeof context.secondaryTrack?.id === 'number' ? context.secondaryTrack.id : null;
const payload: SubsyncManualPayload = {
subtitleTracks,
// The secondary track can be filtered or deduped out of the emitted list,
// so only default to it when the picker actually offers it.
defaultReferenceTrackId:
subtitleTracks.find((track) => track.id === secondaryTrackId)?.id ??
subtitleTracks.find((track) => track.id !== primaryTrackId)?.id ??
null,
defaultTargetTrackId: primaryTrackId,
videoReferenceAvailable: !isRemoteMediaPath(context.videoPath),
ffsubsyncAvailable: !isRemoteMediaPath(context.videoPath),
sourceTracks: context.sourceTracks
.filter((track) => typeof track.id === 'number')
.map((track) => ({
id: track.id as number,
label: formatTrackLabel(track),
})),
};
deps.openManualPicker(payload);
}
@@ -397,7 +558,7 @@ export async function triggerSubsyncFromConfig(deps: TriggerSubsyncFromConfigDep
try {
await openSubsyncManualPicker(deps);
deps.showMpvOsd('Subsync: choose engine and source');
deps.showMpvOsd('Subsync: choose engine and subtitles');
} catch (error) {
deps.showMpvOsd(`Subsync failed: ${(error as Error).message}`);
} finally {
@@ -0,0 +1,40 @@
/*
* Thresholds that decide when a run of repeated subtitle events is one animation.
*
* Three consumers have to agree on these numbers or the same karaoke line is one cue in
* the sidebar and two hundred in the stats: the file-level cue dedup
* (`subtitle-cue-dedup`), the live gate that decides what immersion stats record
* (`subtitle-line-dedup-gate`), and the retroactive database cleanup
* (`immersion-tracker/duplicate-line-cleanup`).
*/
/**
* Back-to-back frames of the same animation are authored flush against each other; a
* tiny tolerance absorbs the centisecond rounding of the ASS timestamp format.
*/
export const DUPLICATE_CUE_GAP_TOLERANCE_SECONDS = 0.05;
/**
* A burst is a *sequence*. Two adjacent events are two events, not an animation --
* characters do repeat each other, and a repeated line can legitimately be short.
*/
export const MIN_BURST_EVENTS = 3;
/**
* Real dialogue holds on screen for about a second, so a run with a couple of much
* shorter events among them looks like frames. Used only alongside authoring evidence.
*/
export const ANIMATION_FRAME_MAX_SECONDS = 0.3;
/** A karaoke run usually ends on a long "hold" frame, so not every event is short. */
export const MIN_TAGGED_BURST_FRAMES = 2;
/**
* SRT and VTT carry no authoring metadata at all, so timing is the only signal available
* -- which makes it the easiest one to get wrong. ASS->SRT conversion leaves frames at
* ~0.04s, well under any real utterance, and a burst leaves many of them behind. Both
* bounds are deliberately far stricter than the ASS path: a run of ordinary short lines
* (`えっ` traded between characters) must not clear them.
*/
export const TIMING_ONLY_FRAME_MAX_SECONDS = 0.1;
export const MIN_TIMING_ONLY_FRAMES = 5;
+180
View File
@@ -0,0 +1,180 @@
/*
* Duplicate/animation-burst collapsing for parsed subtitle cues.
*
* Split out of the cue parser so the parsing rules and the "is this run one animation?"
* heuristics can be read -- and tested -- on their own. The parser owns the cue shape;
* this module only decides which cues survive.
*/
import { hasAssTemporalOverride, isAnimatedAssEffectKind } from './ass-text';
import {
ANIMATION_FRAME_MAX_SECONDS,
DUPLICATE_CUE_GAP_TOLERANCE_SECONDS,
MIN_BURST_EVENTS,
MIN_TAGGED_BURST_FRAMES,
MIN_TIMING_ONLY_FRAMES,
TIMING_ONLY_FRAME_MAX_SECONDS,
} from './subtitle-burst-constants';
import type {
AnnotatedSubtitleCue,
SubtitleCue,
SubtitleSourceFormat,
} from './subtitle-cue-parser';
function cueKey(cue: SubtitleCue): string {
return `${cue.startTime}|${cue.endTime}|${cue.text}`;
}
/**
* Identical text over an identical span is redundant however it was authored -- most
* often a layered ASS event stacking a shadow copy under the visible one.
*/
function collapseExactDuplicates(cues: AnnotatedSubtitleCue[]): AnnotatedSubtitleCue[] {
const seen = new Set<string>();
return cues.filter((cue) => {
const key = cueKey(cue);
if (seen.has(key)) {
return false;
}
seen.add(key);
return true;
});
}
function countFramesShorterThan(run: AnnotatedSubtitleCue[], maxSeconds: number): number {
return run.filter((cue) => cue.endTime - cue.startTime < maxSeconds).length;
}
/**
* Evidence that a run of ASS events is one animation rather than several authored lines.
* A static tag says nothing on its own -- three events sharing one `\clip(...)` are three
* signs -- so the tag has to be temporal by nature (`\t`, `\move`, karaoke timing, or
* anything wrapped in `\t(...)`), an animated `Effect` column, or a value that actually
* changes from event to event, which is how per-frame typesetting is authored.
*/
export function hasAssAnimationEvidence(run: AnnotatedSubtitleCue[]): boolean {
if (run.every((cue) => hasAssTemporalOverride(cue.overrides))) {
return true;
}
if (run.every((cue) => isAnimatedAssEffectKind(cue.effectKind))) {
return true;
}
const [first] = run;
const everyEventTypeset = run.every((cue) => cue.overrides.length > 0);
const signatureChanges = run.some((cue) => cue.overrideSignature !== first!.overrideSignature);
return everyEventTypeset && signatureChanges;
}
export function isAnimationBurst(
run: AnnotatedSubtitleCue[],
format: SubtitleSourceFormat,
): boolean {
if (run.length < MIN_BURST_EVENTS) {
return false;
}
if (format === 'srt') {
return (
run.length >= MIN_TIMING_ONLY_FRAMES &&
countFramesShorterThan(run, TIMING_ONLY_FRAME_MAX_SECONDS) === run.length
);
}
if (countFramesShorterThan(run, ANIMATION_FRAME_MAX_SECONDS) < MIN_TAGGED_BURST_FRAMES) {
return false;
}
// One animation belongs to one styled, one named source line. Two characters trading
// the same short word are two styles or two actors, and never merge.
const [first] = run;
if (run.some((cue) => cue.style !== first!.style || cue.name !== first!.name)) {
return false;
}
return hasAssAnimationEvidence(run);
}
/**
* Karaoke and sign typesetting emits one Dialogue event per animation frame, all carrying
* the same visible text over a contiguous span. Collapse each such run into a single cue.
*
* Only runs that look like animation collapse. Two ordinary lines that happen to repeat
* -- several characters each saying `おはよう` in turn, a positioned sign redrawn with a
* different fade -- stay separate, because merging them would destroy real mineable lines.
*/
function collapseAnimationBursts(
cues: AnnotatedSubtitleCue[],
format: SubtitleSourceFormat,
): AnnotatedSubtitleCue[] {
const indicesByText = new Map<string, number[]>();
cues.forEach((cue, index) => {
const bucket = indicesByText.get(cue.text);
if (bucket) {
bucket.push(index);
} else {
indicesByText.set(cue.text, [index]);
}
});
const dropped = new Set<number>();
const extendedEnd = new Map<number, number>();
for (const indices of indicesByText.values()) {
if (indices.length < MIN_BURST_EVENTS) {
continue;
}
let runStart = 0;
while (runStart < indices.length) {
let runEnd = runStart;
let chainEnd = cues[indices[runStart]!]!.endTime;
while (runEnd + 1 < indices.length) {
const next = cues[indices[runEnd + 1]!]!;
if (next.startTime > chainEnd + DUPLICATE_CUE_GAP_TOLERANCE_SECONDS) {
break;
}
chainEnd = Math.max(chainEnd, next.endTime);
runEnd += 1;
}
const run = indices.slice(runStart, runEnd + 1).map((index) => cues[index]!);
if (isAnimationBurst(run, format)) {
for (let i = runStart + 1; i <= runEnd; i += 1) {
dropped.add(indices[i]!);
}
extendedEnd.set(indices[runStart]!, chainEnd);
}
runStart = runEnd + 1;
}
}
if (dropped.size === 0) {
return cues;
}
const merged: AnnotatedSubtitleCue[] = [];
cues.forEach((cue, index) => {
if (dropped.has(index)) {
return;
}
const end = extendedEnd.get(index);
merged.push(end !== undefined && end > cue.endTime ? { ...cue, endTime: end } : cue);
});
return merged;
}
/**
* Collapse redundant cues. Input must already be sorted by non-decreasing `startTime`,
* ties broken by `endTime` then source `order` -- burst detection chains events by
* comparing each one against the running end of the events before it, so an unsorted
* list breaks runs apart and leaves the frames behind.
*/
export function mergeDuplicateCues(
cues: AnnotatedSubtitleCue[],
format: SubtitleSourceFormat,
): AnnotatedSubtitleCue[] {
return collapseAnimationBursts(collapseExactDuplicates(cues), format);
}
+393 -3
View File
@@ -91,6 +91,17 @@ test('parseSrtCues skips malformed timing lines gracefully', () => {
assert.equal(cues[0]!.text, '有効');
});
test('parseSubtitleCues strips complete brace blocks from SRT and VTT text', () => {
const content = ['1', '00:00:01,000 --> 00:00:02,000', '彼は{謎}と言った', ''].join('\n');
for (const filename of ['test.srt', 'test.vtt']) {
const cues = parseSubtitleCues(content, filename);
assert.equal(cues.length, 1, filename);
assert.equal(cues[0]!.text, '彼はと言った', filename);
}
});
test('parseAssCues parses basic ASS dialogue lines', () => {
const content = [
'[Script Info]',
@@ -137,7 +148,9 @@ test('parseAssCues handles text containing commas', () => {
assert.equal(cues[0]!.text, 'はい、そうです、ね');
});
test('parseAssCues handles \\N line breaks', () => {
test('parseAssCues decodes \\N line breaks into real newlines', () => {
// ASS is decoded once, here at ingestion, so cue text matches what mpv hands over for
// the same line played live.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
@@ -146,7 +159,7 @@ test('parseAssCues handles \\N line breaks', () => {
const cues = parseAssCues(content);
assert.equal(cues[0]!.text, '一行目\\N二行目');
assert.equal(cues[0]!.text, '一行目\n二行目');
});
test('parseAssCues strips HTML-like markup while preserving ASS line breaks', () => {
@@ -158,7 +171,46 @@ test('parseAssCues strips HTML-like markup while preserving ASS line breaks', ()
const cues = parseAssCues(content);
assert.equal(cues[0]!.text, '一行目\\N二行目');
assert.equal(cues[0]!.text, '一行目\n二行目');
});
test('parseAssCues drops vector drawing runs enabled by \\p', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 1,0:00:01.00,0:00:04.00,Default,,0,0,0,,{\\an5\\pos(730,1042)\\p1\\blur1}m 20 0 b 10 0 0 10 0 20 b 0 31 10 40 20 40 {\\p0}',
'Dialogue: 0,0:00:05.00,0:00:08.00,Default,,0,0,0,,これは字幕',
].join('\n');
const cues = parseAssCues(content);
assert.equal(cues.length, 1);
assert.equal(cues[0]!.text, 'これは字幕');
});
test('parseAssCues keeps text that follows a \\p0 reset on the same line', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:04.00,Default,,0,0,0,,{\\p1}m 0 0 l 10 10{\\p0}本文{\\p1}m 5 5 l 6 6{\\p0}続き',
].join('\n');
const cues = parseAssCues(content);
assert.equal(cues.length, 1);
assert.equal(cues[0]!.text, '本文続き');
});
test('parseAssCues leaves \\pos untouched when no drawing mode is active', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:04.00,Default,,0,0,0,,{\\pos(960,1068)\\bord3}位置指定',
].join('\n');
const cues = parseAssCues(content);
assert.equal(cues[0]!.text, '位置指定');
});
test('parseAssCues returns empty for content without Events section', () => {
@@ -258,6 +310,344 @@ test('parseSubtitleCues returns cues sorted by start time', () => {
assert.equal(cues[1]!.text, '二番目');
});
test('parseSubtitleCues collapses per-frame karaoke duplicates into one cue', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.05,OP_JP,,0,0,0,,{\\clip(m 1 1)}過ぎ去ってしまう瞬間を',
'Dialogue: 0,0:00:01.05,0:00:01.09,OP_JP,,0,0,0,,{\\clip(m 2 2)}過ぎ去ってしまう瞬間を',
'Dialogue: 0,0:00:01.09,0:00:03.55,OP_JP,,0,0,0,,{\\clip(m 3 3)}過ぎ去ってしまう瞬間を',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 1);
assert.equal(cues[0]!.startTime, 1.0);
assert.equal(cues[0]!.endTime, 3.55);
assert.equal(cues[0]!.text, '過ぎ去ってしまう瞬間を');
});
test('parseSubtitleCues keeps back-to-back plain dialogue repeats separate', () => {
// Several characters greeting in turn: distinct utterances that happen to abut.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:04:05.67,0:04:06.82,Dial_JP,,0,0,0,,おはよう',
'Dialogue: 0,0:04:06.82,0:04:07.56,Dial_JP,,0,0,0,,おはよう',
'Dialogue: 0,0:04:07.56,0:04:08.78,Dial_JP,,0,0,0,,おはよう',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
assert.equal(cues[0]!.endTime, 246.82);
assert.equal(cues[2]!.startTime, 247.56);
});
test('parseSubtitleCues collapses exact duplicate cues even without effect tags', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:04.00,Default,,0,0,0,,重なった行',
'Dialogue: 1,0:00:01.00,0:00:04.00,Default,,0,0,0,,重なった行',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 1);
});
test('parseSubtitleCues collapses tag-less animation frames in converted SRT', () => {
// ASS -> SRT conversion drops override tags, so only the ~0.04s frame timing remains.
const lines = ['1', '00:00:07,870 --> 00:00:07,910', 'Kaguya Wants to be Confessed to', ''];
for (let i = 1; i < 8; i++) {
const start = 7910 + (i - 1) * 40;
const end = start + 40;
const at = (ms: number) =>
`00:00:0${Math.floor(ms / 1000)},${String(ms % 1000).padStart(3, '0')}`;
lines.push(String(i + 1), `${at(start)} --> ${at(end)}`, 'Kaguya Wants to be Confessed to', '');
}
const cues = parseSubtitleCues(lines.join('\n'), 'test.srt');
assert.equal(cues.length, 1);
assert.equal(cues[0]!.startTime, 7.87);
});
test('parseSubtitleCues keeps identical lines that recur far apart', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:02.00,Default,,0,0,0,,なんで',
'Dialogue: 0,0:05:00.00,0:05:01.00,Default,,0,0,0,,なんで',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 2);
assert.equal(cues[0]!.startTime, 1.0);
assert.equal(cues[1]!.startTime, 300.0);
});
test('parseSubtitleCues keeps two positioned signs that repeat the same text', () => {
// Both carry override tags, but `\pos` and `\fad` are static placement, not animation.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:01:00.00,0:01:03.00,Sign,,0,0,0,,{\\pos(960,120)\\fad(200,200)}第一話',
'Dialogue: 0,0:01:03.00,0:01:06.00,Sign,,0,0,0,,{\\pos(960,900)\\fad(200,200)}第一話',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 2);
assert.equal(cues[1]!.startTime, 63.0);
});
test('parseSubtitleCues keeps a run of ordinary positioned lines separate', () => {
// Three events is a sequence, but none of them runs at animation-frame speed.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:01:00.00,0:01:02.00,Sign,,0,0,0,,{\\pos(960,120)\\fad(100,100)}止まれ',
'Dialogue: 0,0:01:02.00,0:01:04.00,Sign,,0,0,0,,{\\pos(960,120)\\fad(100,100)}止まれ',
'Dialogue: 0,0:01:04.00,0:01:06.00,Sign,,0,0,0,,{\\pos(960,120)\\fad(100,100)}止まれ',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
});
test('parseSubtitleCues keeps a short repeated SRT pair without burst evidence', () => {
const content = [
'1',
'00:00:01,000 --> 00:00:01,200',
'えっ',
'',
'2',
'00:00:01,200 --> 00:00:01,400',
'えっ',
'',
].join('\n');
const cues = parseSubtitleCues(content, 'test.srt');
assert.equal(cues.length, 2);
});
test('parseSubtitleCues collapses a burst marked only by the Effect column', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.05,OP_JP,,0,0,0,Karaoke,歌詞',
'Dialogue: 0,0:00:01.05,0:00:01.09,OP_JP,,0,0,0,Karaoke,歌詞',
'Dialogue: 0,0:00:01.09,0:00:03.55,OP_JP,,0,0,0,Karaoke,歌詞',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 1);
assert.equal(cues[0]!.endTime, 3.55);
});
test('parseSubtitleCues keeps a second karaoke burst that starts after a gap', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.05,OP_JP,,0,0,0,,{\\clip(m 1 1)}リフレイン',
'Dialogue: 0,0:00:01.05,0:00:01.09,OP_JP,,0,0,0,,{\\clip(m 2 2)}リフレイン',
'Dialogue: 0,0:00:01.09,0:00:03.00,OP_JP,,0,0,0,,{\\clip(m 3 3)}リフレイン',
'Dialogue: 0,0:00:20.00,0:00:20.05,OP_JP,,0,0,0,,{\\clip(m 1 1)}リフレイン',
'Dialogue: 0,0:00:20.05,0:00:20.09,OP_JP,,0,0,0,,{\\clip(m 2 2)}リフレイン',
'Dialogue: 0,0:00:20.09,0:00:22.00,OP_JP,,0,0,0,,{\\clip(m 3 3)}リフレイン',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 2);
assert.equal(cues[0]!.endTime, 3.0);
assert.equal(cues[1]!.startTime, 20.0);
assert.equal(cues[1]!.endTime, 22.0);
});
test('parseSubtitleCues does not merge a burst into unrelated dialogue between frames', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.05,OP_JP,,0,0,0,,{\\clip(m 1 1)}歌詞',
'Dialogue: 0,0:00:01.02,0:00:03.00,Dial_JP,,0,0,0,,別のセリフ',
'Dialogue: 0,0:00:01.05,0:00:01.09,OP_JP,,0,0,0,,{\\clip(m 2 2)}歌詞',
'Dialogue: 0,0:00:01.09,0:00:03.55,OP_JP,,0,0,0,,{\\clip(m 3 3)}歌詞',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 2);
assert.deepEqual(
cues.map((cue) => cue.text),
['歌詞', '別のセリフ'],
);
assert.equal(cues[0]!.endTime, 3.55);
});
test('parseSubtitleCues keeps rapid ASS lines from different actors separate', () => {
// Three 200ms `えっ` reactions traded between characters. Fast, adjacent and identical,
// but authored as three lines: different styles and different actors.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Dial_A,アリス,0,0,0,,えっ',
'Dialogue: 0,0:00:01.20,0:00:01.40,Dial_B,ボブ,0,0,0,,えっ',
'Dialogue: 0,0:00:01.40,0:00:01.60,Dial_C,キャロル,0,0,0,,えっ',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
});
test('parseSubtitleCues reads the speaker column when it is spelled Actor', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Actor, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Dial_JP,アリス,0,0,0,,えっ',
'Dialogue: 0,0:00:01.20,0:00:01.40,Dial_JP,ボブ,0,0,0,,えっ',
'Dialogue: 0,0:00:01.40,0:00:01.60,Dial_JP,キャロル,0,0,0,,えっ',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
});
test('parseSubtitleCues does not treat a custom Effect name as animation', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Sign,,0,0,0,scrolling-credit,制作',
'Dialogue: 0,0:00:01.20,0:00:01.40,Sign,,0,0,0,scrolling-credit,制作',
'Dialogue: 0,0:00:01.40,0:00:01.60,Sign,,0,0,0,scrolling-credit,制作',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
});
test('parseSubtitleCues keeps rapid ASS lines that share a style but not an actor', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Dial_JP,アリス,0,0,0,,えっ',
'Dialogue: 0,0:00:01.20,0:00:01.40,Dial_JP,ボブ,0,0,0,,えっ',
'Dialogue: 0,0:00:01.40,0:00:01.60,Dial_JP,キャロル,0,0,0,,えっ',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
});
test('parseSubtitleCues keeps untagged rapid ASS repeats separate', () => {
// No overrides at all: timing-only evidence is an SRT/VTT fallback and must not apply
// to ASS, where the absence of typesetting is itself evidence of plain dialogue.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.05,Dial_JP,,0,0,0,,えっ',
'Dialogue: 0,0:00:01.05,0:00:01.10,Dial_JP,,0,0,0,,えっ',
'Dialogue: 0,0:00:01.10,0:00:01.15,Dial_JP,,0,0,0,,えっ',
'Dialogue: 0,0:00:01.15,0:00:01.20,Dial_JP,,0,0,0,,えっ',
'Dialogue: 0,0:00:01.20,0:00:01.25,Dial_JP,,0,0,0,,えっ',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 5);
});
test('parseSubtitleCues keeps repeated signs sharing one static clip', () => {
// `\clip` is a static shape for the event. Three events with the identical clip were
// typeset the same way, so none of them is a frame of the others.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Sign,,0,0,0,,{\\clip(0,0,100,100)}注意',
'Dialogue: 0,0:00:01.20,0:00:01.40,Sign,,0,0,0,,{\\clip(0,0,100,100)}注意',
'Dialogue: 0,0:00:01.40,0:00:01.60,Sign,,0,0,0,,{\\clip(0,0,100,100)}注意',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 3);
});
test('parseSubtitleCues collapses a sign animated through \\t', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Sign,,0,0,0,,{\\pos(10,10)\\t(0,200,\\frz30)}回る',
'Dialogue: 0,0:00:01.20,0:00:01.40,Sign,,0,0,0,,{\\pos(10,10)\\t(0,200,\\frz30)}回る',
'Dialogue: 0,0:00:01.40,0:00:03.00,Sign,,0,0,0,,{\\pos(10,10)\\t(0,200,\\frz30)}回る',
].join('\n');
const cues = parseSubtitleCues(content, 'test.ass');
assert.equal(cues.length, 1);
assert.equal(cues[0]!.endTime, 3.0);
});
test('parseSubtitleCues keeps a short repeated SRT run above the frame threshold', () => {
// Five contiguous 200ms cues: a sequence, but nowhere near animation-frame speed.
const lines: string[] = [];
for (let i = 0; i < 5; i++) {
const start = 1000 + i * 200;
const at = (ms: number) =>
`00:00:0${Math.floor(ms / 1000)},${String(ms % 1000).padStart(3, '0')}`;
lines.push(String(i + 1), `${at(start)} --> ${at(start + 200)}`, 'えっ', '');
}
const cues = parseSubtitleCues(lines.join('\n'), 'test.srt');
assert.equal(cues.length, 5);
});
test('parseSubtitleCues keeps a short SRT frame run below the minimum length', () => {
// Four 40ms frames: frame-speed, but too few to tell an animation from an artefact.
const lines: string[] = [];
for (let i = 0; i < 4; i++) {
const start = 7870 + i * 40;
const at = (ms: number) =>
`00:00:0${Math.floor(ms / 1000)},${String(ms % 1000).padStart(3, '0')}`;
lines.push(String(i + 1), `${at(start)} --> ${at(start + 40)}`, 'タイトル', '');
}
const cues = parseSubtitleCues(lines.join('\n'), 'test.srt');
assert.equal(cues.length, 4);
});
test('parseSubtitleCues applies ASS burst rules to ASS content behind an .srt filename', () => {
// The extension lies, so the SRT parser finds nothing and the content-sniffing fallback
// takes over -- which has to carry the `ass` source format with it, or the far stricter
// timing-only thresholds would let this karaoke burst through as three cues.
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:01.20,Karaoke,,0,0,0,,{\\k20}歌詞',
'Dialogue: 0,0:00:01.20,0:00:01.40,Karaoke,,0,0,0,,{\\k20}歌詞',
'Dialogue: 0,0:00:01.40,0:00:03.00,Karaoke,,0,0,0,,{\\k20}歌詞',
].join('\n');
const cues = parseSubtitleCues(content, 'test.srt');
assert.equal(cues.length, 1);
assert.equal(cues[0]!.startTime, 1.0);
assert.equal(cues[0]!.endTime, 3.0);
assert.equal(cues[0]!.text, '歌詞');
});
test('parseSubtitleCues detects subtitle formats from remote URLs', () => {
const assContent = [
'[Events]',
+160 -34
View File
@@ -1,9 +1,46 @@
import {
assOverrideSignature,
assToPlainText,
collectAssOverrideCommands,
parseAssEffectField,
type AssEffectKind,
type AssOverrideCommand,
} from './ass-text';
import { mergeDuplicateCues } from './subtitle-cue-dedup';
export interface SubtitleCue {
startTime: number;
endTime: number;
text: string;
}
/**
* Everything the parser knows about a source event, shared only with the dedup engine.
* Deduplication needs the authoring context -- which style the line belongs to, which
* override commands it carries, whether the `Effect` column was set -- to tell a karaoke
* burst apart from two characters saying the same word in turn. None of it is meaningful
* outside the parser, so the public API stays `{startTime, endTime, text}`.
*/
export interface AnnotatedSubtitleCue extends SubtitleCue {
/** Text exactly as authored, override blocks and all. */
rawText: string;
style: string;
layer: number;
/** ASS `Name`/`Actor` column. */
name: string;
/** ASS `Effect` column, verbatim. */
effect: string;
effectKind: AssEffectKind;
/** Override commands found in `{...}` blocks, with their arguments. */
overrides: readonly AssOverrideCommand[];
/** Canonical form of `overrides`, for spotting values that change across a run. */
overrideSignature: string;
/** Position in the source file, so sorting by time stays deterministic across layers. */
order: number;
}
export type SubtitleSourceFormat = 'ass' | 'srt';
const HTML_SUBTITLE_TAG_PATTERN = /<\/?[A-Za-z][^>\n]*>/g;
const SRT_TIMING_PATTERN =
@@ -23,12 +60,21 @@ function parseTimestamp(
);
}
/**
* The single ASS decode for the file path: cues leave the parser as plain text with real
* line breaks, matching what mpv hands over for the same line played live. No layer
* downstream decodes ASS again.
*/
function sanitizeSubtitleCueText(text: string): string {
return text.replace(ASS_OVERRIDE_TAG_PATTERN, '').replace(HTML_SUBTITLE_TAG_PATTERN, '').trim();
return assToPlainText(text, '\n').replace(HTML_SUBTITLE_TAG_PATTERN, '').trim();
}
export function parseSrtCues(content: string): SubtitleCue[] {
const cues: SubtitleCue[] = [];
function toPublicCues(cues: AnnotatedSubtitleCue[]): SubtitleCue[] {
return cues.map(({ startTime, endTime, text }) => ({ startTime, endTime, text }));
}
function parseAnnotatedSrtCues(content: string): AnnotatedSubtitleCue[] {
const cues: AnnotatedSubtitleCue[] = [];
const lines = content.split(/\r?\n/);
let i = 0;
@@ -60,20 +106,39 @@ export function parseSrtCues(content: string): SubtitleCue[] {
i += 1;
}
const text = sanitizeSubtitleCueText(textLines.join('\n'));
const rawText = textLines.join('\n');
const text = sanitizeSubtitleCueText(rawText);
if (text) {
cues.push({ startTime, endTime, text });
cues.push({
startTime,
endTime,
text,
rawText,
style: '',
layer: 0,
name: '',
effect: '',
effectKind: 'none',
// SRT and VTT carry no authoring metadata, and the dedup engine never reads
// overrides for those formats -- collecting them would be parsing for nobody.
overrides: [],
overrideSignature: '',
order: cues.length,
});
}
}
return cues;
}
const ASS_OVERRIDE_TAG_PATTERN = /\{[^}]*\}/g;
export function parseSrtCues(content: string): SubtitleCue[] {
return toPublicCues(parseAnnotatedSrtCues(content));
}
const ASS_TIMING_PATTERN = /^(\d+):(\d{2}):(\d{2})\.(\d{1,2})$/;
const ASS_FORMAT_PREFIX = 'Format:';
const ASS_DIALOGUE_PREFIX = 'Dialogue:';
const ASS_NAME_FIELD_ALIASES = ['name', 'actor'];
function parseAssTimestamp(raw: string): number | null {
const match = ASS_TIMING_PATTERN.exec(raw.trim());
@@ -87,13 +152,43 @@ function parseAssTimestamp(raw: string): number | null {
return hours * 3600 + minutes * 60 + seconds + centiseconds / 100;
}
export function parseAssCues(content: string): SubtitleCue[] {
const cues: SubtitleCue[] = [];
function readField(fields: string[], index: number): string {
return index >= 0 && index < fields.length ? fields[index]!.trim() : '';
}
function findFieldIndex(formatFields: string[], aliases: string[]): number {
for (const alias of aliases) {
const index = formatFields.indexOf(alias);
if (index >= 0) {
return index;
}
}
return -1;
}
function parseAnnotatedAssCues(content: string): AnnotatedSubtitleCue[] {
const cues: AnnotatedSubtitleCue[] = [];
const lines = content.split(/\r?\n/);
let inEventsSection = false;
let startFieldIndex = -1;
let endFieldIndex = -1;
let textFieldIndex = -1;
const fieldIndex = {
start: -1,
end: -1,
text: -1,
style: -1,
layer: -1,
name: -1,
effect: -1,
};
const resetFieldIndex = () => {
fieldIndex.start = -1;
fieldIndex.end = -1;
fieldIndex.text = -1;
fieldIndex.style = -1;
fieldIndex.layer = -1;
fieldIndex.name = -1;
fieldIndex.effect = -1;
};
for (const line of lines) {
const trimmed = line.trim();
@@ -101,9 +196,7 @@ export function parseAssCues(content: string): SubtitleCue[] {
if (trimmed.startsWith('[') && trimmed.endsWith(']')) {
inEventsSection = trimmed.toLowerCase() === '[events]';
if (!inEventsSection) {
startFieldIndex = -1;
endFieldIndex = -1;
textFieldIndex = -1;
resetFieldIndex();
}
continue;
}
@@ -117,9 +210,15 @@ export function parseAssCues(content: string): SubtitleCue[] {
.slice(ASS_FORMAT_PREFIX.length)
.split(',')
.map((field) => field.trim().toLowerCase());
startFieldIndex = formatFields.indexOf('start');
endFieldIndex = formatFields.indexOf('end');
textFieldIndex = formatFields.indexOf('text');
fieldIndex.start = formatFields.indexOf('start');
fieldIndex.end = formatFields.indexOf('end');
fieldIndex.text = formatFields.indexOf('text');
fieldIndex.style = formatFields.indexOf('style');
fieldIndex.layer = formatFields.indexOf('layer');
// Aegisub writes the speaker column as `Actor`; the v4+ spec calls it `Name`.
// Missing it costs the burst check its speaker guard, so both spellings count.
fieldIndex.name = findFieldIndex(formatFields, ASS_NAME_FIELD_ALIASES);
fieldIndex.effect = formatFields.indexOf('effect');
continue;
}
@@ -127,34 +226,57 @@ export function parseAssCues(content: string): SubtitleCue[] {
continue;
}
if (startFieldIndex < 0 || endFieldIndex < 0 || textFieldIndex < 0) {
if (fieldIndex.start < 0 || fieldIndex.end < 0 || fieldIndex.text < 0) {
continue;
}
const fields = trimmed.slice(ASS_DIALOGUE_PREFIX.length).split(',');
if (
startFieldIndex >= fields.length ||
endFieldIndex >= fields.length ||
textFieldIndex >= fields.length
fieldIndex.start >= fields.length ||
fieldIndex.end >= fields.length ||
fieldIndex.text >= fields.length
) {
continue;
}
const startTime = parseAssTimestamp(fields[startFieldIndex]!);
const endTime = parseAssTimestamp(fields[endFieldIndex]!);
const startTime = parseAssTimestamp(fields[fieldIndex.start]!);
const endTime = parseAssTimestamp(fields[fieldIndex.end]!);
if (startTime === null || endTime === null) {
continue;
}
const text = sanitizeSubtitleCueText(fields.slice(textFieldIndex).join(','));
if (text) {
cues.push({ startTime, endTime, text });
const rawText = fields.slice(fieldIndex.text).join(',');
const text = sanitizeSubtitleCueText(rawText);
if (!text) {
continue;
}
const effect = readField(fields, fieldIndex.effect);
const layer = Number(readField(fields, fieldIndex.layer));
const overrides = collectAssOverrideCommands(rawText);
cues.push({
startTime,
endTime,
text,
rawText,
style: readField(fields, fieldIndex.style),
layer: Number.isFinite(layer) ? layer : 0,
name: readField(fields, fieldIndex.name),
effect,
effectKind: parseAssEffectField(effect),
overrides,
overrideSignature: assOverrideSignature(overrides),
order: cues.length,
});
}
return cues;
}
export function parseAssCues(content: string): SubtitleCue[] {
return toPublicCues(parseAnnotatedAssCues(content));
}
function detectSubtitleFormat(source: string): 'srt' | 'vtt' | 'ass' | 'ssa' | null {
const [normalizedSource = source] =
(() => {
@@ -173,27 +295,31 @@ function detectSubtitleFormat(source: string): 'srt' | 'vtt' | 'ass' | 'ssa' | n
export function parseSubtitleCues(content: string, filename: string): SubtitleCue[] {
const format = detectSubtitleFormat(filename);
let cues: SubtitleCue[];
let cues: AnnotatedSubtitleCue[];
let sourceFormat: SubtitleSourceFormat = 'srt';
switch (format) {
case 'srt':
case 'vtt':
cues = parseSrtCues(content);
cues = parseAnnotatedSrtCues(content);
break;
case 'ass':
case 'ssa':
cues = parseAssCues(content);
cues = parseAnnotatedAssCues(content);
sourceFormat = 'ass';
break;
default:
cues = [];
}
if (cues.length === 0) {
const assCues = parseAssCues(content);
const srtCues = parseSrtCues(content);
cues = assCues.length >= srtCues.length ? assCues : srtCues;
const assCues = parseAnnotatedAssCues(content);
const srtCues = parseAnnotatedSrtCues(content);
const preferAss = assCues.length >= srtCues.length;
cues = preferAss ? assCues : srtCues;
sourceFormat = preferAss && assCues.length > 0 ? 'ass' : 'srt';
}
cues.sort((a, b) => a.startTime - b.startTime);
return cues;
cues.sort((a, b) => a.startTime - b.startTime || a.endTime - b.endTime || a.order - b.order);
return toPublicCues(mergeDuplicateCues(cues, sourceFormat));
}
@@ -0,0 +1,169 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { createSubtitleLineDedupGate } from './subtitle-line-dedup-gate';
import type { SubtitleCue } from '../../types';
function karaokeFrames(text: string, start: number, frames: number, frameSeconds: number) {
return Array.from({ length: frames }, (_, index) => ({
text,
startSec: start + index * frameSeconds,
endSec: start + (index + 1) * frameSeconds,
}));
}
test('parsed cues drop the frames the sidebar already collapsed', () => {
// What `mergeDuplicateCues` leaves behind for a karaoke run: one cue over the run.
const cues: SubtitleCue[] = [
{ startTime: 10, endTime: 14, text: '飛び上がる' },
{ startTime: 14, endTime: 16, text: 'もしも' },
];
const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
const recorded = karaokeFrames('飛び上がる', 10, 40, 0.04).filter((sample) =>
gate.shouldRecord(sample),
);
assert.equal(recorded.length, 1);
assert.equal(recorded[0]!.startSec, 10);
assert.equal(gate.shouldRecord({ text: 'もしも', startSec: 14, endSec: 16 }), true);
});
test('parsed cues keep separate lines that merely repeat', () => {
const cues: SubtitleCue[] = [
{ startTime: 3, endTime: 3.4, text: 'えっ' },
{ startTime: 3.4, endTime: 3.9, text: 'えっ' },
{ startTime: 3.9, endTime: 4.5, text: 'えっ' },
];
const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
const recorded = cues.filter((cue) =>
gate.shouldRecord({ text: cue.text, startSec: cue.startTime, endSec: cue.endTime }),
);
assert.equal(recorded.length, 3);
});
test('parsed cues outrank the streaming heuristic for short repeated cues', () => {
// Long enough to trip the timing-only rule, but the parser saw these with full
// lookahead and kept them, so every one of them is a line the sidebar shows.
const cues: SubtitleCue[] = Array.from({ length: 8 }, (_, index) => ({
startTime: 3 + index * 0.08,
endTime: 3 + (index + 1) * 0.08,
text: 'えっ',
}));
const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
const recorded = cues.filter((cue) =>
gate.shouldRecord({ text: cue.text, startSec: cue.startTime, endSec: cue.endTime }),
);
assert.equal(recorded.length, 8);
});
test('parsed cues preserve legitimately separate cues only 40ms apart', () => {
const cues: SubtitleCue[] = Array.from({ length: 8 }, (_, index) => ({
startTime: 3 + index * 0.04,
endTime: 3 + (index + 1) * 0.04,
text: 'えっ',
}));
const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
const recorded = cues.filter((cue) =>
gate.shouldRecord({ text: cue.text, startSec: cue.startTime, endSec: cue.endTime }),
);
assert.equal(recorded.length, 8);
});
test('a line whose timing does not match any cue still records', () => {
// A shifted track, an embedded sub nobody parsed: no match, no drop.
const cues: SubtitleCue[] = [{ startTime: 10, endTime: 14, text: '飛び上がる' }];
const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
assert.equal(gate.shouldRecord({ text: '飛び上がる', startSec: 42, endSec: 44 }), true);
});
test('shifted parsed text falls back to streaming burst detection', () => {
const cues: SubtitleCue[] = [{ startTime: 10, endTime: 14, text: '飛び上がる' }];
const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
const recorded = karaokeFrames('飛び上がる', 42, 40, 0.04).filter((sample) =>
gate.shouldRecord(sample),
);
assert.equal(recorded.length, 4);
});
test('replacing the parsed cue source forgets a streaming run', () => {
let cues: SubtitleCue[] = [];
const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
karaokeFrames('飛び上がる', 42, 20, 0.04).forEach((sample) => gate.shouldRecord(sample));
cues = [];
assert.equal(gate.shouldRecord({ text: '飛び上がる', startSec: 42.8, endSec: 42.84 }), true);
});
test('without parsed cues a long run of identical short frames stops recording', () => {
const gate = createSubtitleLineDedupGate({ getParsedCues: () => null });
const recorded = karaokeFrames('ひとしずく', 0, 200, 0.04).filter((sample) =>
gate.shouldRecord(sample),
);
assert.equal(recorded.length, 4);
});
test('without parsed cues ordinary repeated dialogue keeps recording', () => {
const gate = createSubtitleLineDedupGate({ getParsedCues: () => null });
// Six contiguous `えっ`, each held for a normal beat rather than an animation frame.
const recorded = karaokeFrames('えっ', 0, 6, 0.6).filter((sample) => gate.shouldRecord(sample));
assert.equal(recorded.length, 6);
});
test('the same event offered twice does not advance the run', () => {
const gate = createSubtitleLineDedupGate({ getParsedCues: () => null });
// mpv fires the timing handler once for `sub-start` and once for `sub-end`.
for (let i = 0; i < 8; i += 1) {
assert.equal(gate.shouldRecord({ text: '待って', startSec: 5, endSec: 5.05 }), true);
}
});
test('a gap between frames starts a new run', () => {
const gate = createSubtitleLineDedupGate({ getParsedCues: () => null });
const first = karaokeFrames('もし', 0, 6, 0.04).filter((sample) => gate.shouldRecord(sample));
const second = karaokeFrames('もし', 30, 6, 0.04).filter((sample) => gate.shouldRecord(sample));
assert.equal(first.length, 4);
assert.equal(second.length, 4);
});
test('reset forgets the streaming run', () => {
const gate = createSubtitleLineDedupGate({ getParsedCues: () => null });
karaokeFrames('もし', 0, 20, 0.04).forEach((sample) => gate.shouldRecord(sample));
gate.reset();
assert.equal(gate.shouldRecord({ text: 'もし', startSec: 0.8, endSec: 0.84 }), true);
});
test('reset ignores stale parsed cues until the source publishes a new cue list', () => {
let cues: SubtitleCue[] = [{ startTime: 10, endTime: 14, text: '飛び上がる' }];
const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
assert.equal(gate.shouldRecord({ text: '飛び上がる', startSec: 10, endSec: 10.04 }), true);
gate.reset();
const recordedWithStaleCues = karaokeFrames('飛び上がる', 10.04, 8, 0.04).filter((sample) =>
gate.shouldRecord(sample),
);
assert.equal(recordedWithStaleCues.length, 4);
cues = [{ startTime: 20, endTime: 24, text: '飛び上がる' }];
assert.equal(gate.shouldRecord({ text: '飛び上がる', startSec: 20, endSec: 20.04 }), true);
assert.equal(gate.shouldRecord({ text: '飛び上がる', startSec: 20.04, endSec: 20.08 }), false);
});
@@ -0,0 +1,208 @@
/*
* Decides which live mpv subtitle lines reach the immersion stats.
*
* The sidebar reads a parsed subtitle file, so it can collapse an animation burst with
* full lookahead (`subtitle-cue-dedup`). Stats are fed from mpv's `sub-start`/`sub-end`
* properties instead -- one event per animation frame, each with its own start time --
* so without a gate a karaoke OP counts its lyrics once per frame and buries every real
* word in the vocabulary charts.
*
* Two layers, in order:
*
* 1. When the active source has been parsed, its cue list has *already* been collapsed.
* A live line that lands inside a surviving cue of the same text, but after that
* cue's start, is a frame the sidebar merged away, so stats drop it too. This is the
* layer that keeps the two views consistent by construction.
* 2. Otherwise (embedded track nobody parsed, a source whose timings mpv has shifted)
* fall back to timing alone. No authoring metadata is available live -- mpv delivers
* `sub-text-ass` after `sub-start`/`sub-end`, so any ASS text read here belongs to the
* previous event -- which puts this layer in the same position as the SRT path in
* `subtitle-cue-dedup`, and it uses that path's deliberately strict bounds.
*/
import { normalizePlainSubtitleText } from './ass-text';
import {
DUPLICATE_CUE_GAP_TOLERANCE_SECONDS,
MIN_TIMING_ONLY_FRAMES,
TIMING_ONLY_FRAME_MAX_SECONDS,
} from './subtitle-burst-constants';
import type { SubtitleCue } from './subtitle-cue-parser';
export interface SubtitleLineSample {
text: string;
startSec: number;
endSec: number;
}
export interface SubtitleLineDedupGateDeps {
/** Cues for the active source, already collapsed by the parser. */
getParsedCues: () => readonly SubtitleCue[] | null | undefined;
}
export interface SubtitleLineDedupGate {
/** False when this line is an animation frame of a line already recorded. */
shouldRecord: (sample: SubtitleLineSample) => boolean;
/** Forget run state and ignore the current cue list until its source is replaced. */
reset: () => void;
}
interface CueSpan {
startTime: number;
endTime: number;
}
interface StreamingRunState {
text: string;
startMs: number;
chainEndSec: number;
/** Contiguous identical short frames seen so far, including the recorded first one. */
frames: number;
}
/** Exact cue identity, separate from the looser tolerance used to chain adjacent frames. */
const CUE_START_IDENTITY_TOLERANCE_SECONDS = 0.005;
function normalizeLineText(text: string): string {
return normalizePlainSubtitleText(text, { collapseLineBreaks: true });
}
function buildSpansByText(cues: readonly SubtitleCue[]): Map<string, CueSpan[]> {
const spansByText = new Map<string, CueSpan[]>();
for (const cue of cues) {
const key = normalizeLineText(cue.text);
if (!key) continue;
const span = { startTime: cue.startTime, endTime: cue.endTime };
const existing = spansByText.get(key);
if (existing) {
existing.push(span);
} else {
spansByText.set(key, [span]);
}
}
return spansByText;
}
/**
* A frame the parser merged away: the same text, starting inside a surviving cue but
* after it began.
*
* Starting a cue always wins over falling inside one. The first frame of a collapsed run
* starts *at* the merged cue, and a line the parser deliberately kept separate -- three
* characters trading `えっ` back to back -- begins exactly where the one before it ends.
*/
function isMergedAwayFrame(spans: readonly CueSpan[], startSec: number): boolean | null {
const coveringSpans = spans.filter(
(span) =>
startSec >= span.startTime - CUE_START_IDENTITY_TOLERANCE_SECONDS &&
startSec <= span.endTime + CUE_START_IDENTITY_TOLERANCE_SECONDS,
);
if (coveringSpans.length === 0) {
return null;
}
const startsOwnCue = spans.some(
(span) => Math.abs(startSec - span.startTime) <= CUE_START_IDENTITY_TOLERANCE_SECONDS,
);
if (startsOwnCue) {
return false;
}
return coveringSpans.some(
(span) =>
startSec > span.startTime + CUE_START_IDENTITY_TOLERANCE_SECONDS &&
startSec <= span.endTime + CUE_START_IDENTITY_TOLERANCE_SECONDS,
);
}
export function createSubtitleLineDedupGate(
deps: SubtitleLineDedupGateDeps,
): SubtitleLineDedupGate {
let indexedCues: readonly SubtitleCue[] | null | undefined;
let ignoredCuesAfterReset: readonly SubtitleCue[] | null | undefined;
let spansByText: Map<string, CueSpan[]> = new Map();
let run: StreamingRunState | null = null;
const lookupSpans = (text: string): CueSpan[] | null => {
const cues = deps.getParsedCues() ?? null;
if (ignoredCuesAfterReset !== undefined) {
if (cues === ignoredCuesAfterReset) {
return null;
}
ignoredCuesAfterReset = undefined;
}
if (cues !== indexedCues) {
indexedCues = cues;
spansByText = cues?.length ? buildSpansByText(cues) : new Map();
run = null;
}
return spansByText.get(text) ?? null;
};
/**
* Timing-only burst detection over a stream. Without lookahead the run can only be
* recognised from the inside, so the first frames of a burst are recorded and the rest
* dropped -- an OP costs a handful of counted lines instead of several hundred.
*/
const advanceStreamingRun = (text: string, sample: SubtitleLineSample): boolean => {
const startMs = Math.round(sample.startSec * 1000);
// mpv reports `sub-start` and `sub-end` separately, so one event can be offered
// twice. The same start is the same frame, never the next one in a run.
if (run && run.text === text && run.startMs === startMs) {
run.chainEndSec = Math.max(run.chainEndSec, sample.endSec);
return run.frames < MIN_TIMING_ONLY_FRAMES;
}
const isShortFrame = sample.endSec - sample.startSec < TIMING_ONLY_FRAME_MAX_SECONDS;
// Frames are authored flush against each other, but typesetters do overlap them, so
// the chain only requires forward progress that stays inside the running end.
const continuesRun =
run !== null &&
run.text === text &&
isShortFrame &&
startMs > run.startMs &&
sample.startSec <= run.chainEndSec + DUPLICATE_CUE_GAP_TOLERANCE_SECONDS;
if (continuesRun && run) {
run.startMs = startMs;
run.chainEndSec = Math.max(run.chainEndSec, sample.endSec);
run.frames += 1;
} else {
run = {
text,
startMs,
chainEndSec: sample.endSec,
frames: isShortFrame ? 1 : 0,
};
}
return run.frames < MIN_TIMING_ONLY_FRAMES;
};
return {
shouldRecord: (sample) => {
const text = normalizeLineText(sample.text);
if (!text) {
return true;
}
// The parsed cue list has the final say wherever it covers this line. Falling
// through to the streaming heuristic would let it drop cues the parser looked at
// with full lookahead and deliberately kept apart, which is the disagreement
// between sidebar and stats this gate exists to prevent.
const spans = lookupSpans(text);
if (spans) {
const mergedAway = isMergedAwayFrame(spans, sample.startSec);
if (mergedAway !== null) {
run = null;
return !mergedAway;
}
}
return advanceStreamingRun(text, sample);
},
reset: () => {
run = null;
ignoredCuesAfterReset = deps.getParsedCues() ?? null;
indexedCues = undefined;
spansByText = new Map();
},
};
}
+17 -18
View File
@@ -74,7 +74,6 @@ test('prefetch service tokenizes priority window cues and caches them', async ()
preCacheTokenization: (text, data) => {
cached.set(text, data);
},
isCacheFull: () => false,
priorityWindowSize: 3,
});
@@ -91,32 +90,38 @@ test('prefetch service tokenizes priority window cues and caches them', async ()
assert.ok(cached.has('line-2'));
});
test('prefetch service stops when cache is full', async () => {
test('prefetch service warms every cue even when the cache evicts along the way', async () => {
const cues = makeCues(20);
let tokenizeCalls = 0;
let cacheSize = 0;
const tokenized: string[] = [];
// Stand-in for the LRU: only the last 5 entries survive, so later cues evict earlier ones.
const cache = new Set<string>();
const service = createSubtitlePrefetchService({
cues,
tokenizeSubtitle: async (text) => {
tokenizeCalls += 1;
tokenized.push(text);
return { text, tokens: [] };
},
preCacheTokenization: () => {
cacheSize += 1;
preCacheTokenization: (text) => {
cache.add(text);
while (cache.size > 5) {
const oldest = cache.values().next().value;
if (oldest === undefined) break;
cache.delete(oldest);
}
},
isCacheFull: () => cacheSize >= 5,
hasCachedTokenization: (text) => cache.has(text),
priorityWindowSize: 3,
});
service.start(0);
for (let i = 0; i < 30; i += 1) {
for (let i = 0; i < 60; i += 1) {
await flushMicrotasks();
}
service.stop();
// Should have stopped at 5 (cache full), not tokenized all 20
assert.ok(tokenizeCalls <= 6, `Expected <= 6 tokenize calls, got ${tokenizeCalls}`);
assert.equal(tokenized.length, 20, `Expected all 20 cues warmed, got ${tokenized.length}`);
assert.equal(new Set(tokenized).size, 20, 'Each cue is tokenized at most once per run');
});
test('prefetch service can be stopped mid-flight', async () => {
@@ -130,7 +135,6 @@ test('prefetch service can be stopped mid-flight', async () => {
return { text, tokens: [] };
},
preCacheTokenization: () => {},
isCacheFull: () => false,
priorityWindowSize: 3,
});
@@ -159,7 +163,6 @@ test('prefetch service onSeek re-prioritizes from new position', async () => {
preCacheTokenization: (text) => {
cachedTexts.push(text);
},
isCacheFull: () => false,
priorityWindowSize: 3,
});
@@ -183,7 +186,7 @@ test('prefetch service onSeek re-prioritizes from new position', async () => {
assert.ok(hasPostSeekCue, 'Should have cached cues after seek position');
});
test('prefetch service still warms the priority window when cache is full', async () => {
test('prefetch service warms the priority window ahead of the rest of the file', async () => {
const cues = makeCues(20);
const cachedTexts: string[] = [];
@@ -193,7 +196,6 @@ test('prefetch service still warms the priority window when cache is full', asyn
preCacheTokenization: (text) => {
cachedTexts.push(text);
},
isCacheFull: () => true,
priorityWindowSize: 3,
});
@@ -217,7 +219,6 @@ test('prefetch service pause/resume halts and continues tokenization', async ()
return { text, tokens: [] };
},
preCacheTokenization: () => {},
isCacheFull: () => false,
priorityWindowSize: 3,
});
@@ -255,7 +256,6 @@ test('prefetch service skips cues already present in tokenization cache', async
},
preCacheTokenization: () => {},
hasCachedTokenization: (text) => text === 'line-0' || text === 'line-1',
isCacheFull: () => false,
priorityWindowSize: 3,
});
@@ -285,7 +285,6 @@ test('prefetch service deduplicates repeated cue text within a run', async () =>
return { text, tokens: [] };
},
preCacheTokenization: () => {},
isCacheFull: () => false,
priorityWindowSize: 3,
});
+5 -7
View File
@@ -7,7 +7,6 @@ export interface SubtitlePrefetchServiceDeps {
tokenizeSubtitle: (text: string) => Promise<SubtitleData | null>;
preCacheTokenization: (text: string, data: SubtitleData) => void;
hasCachedTokenization?: (text: string) => boolean;
isCacheFull: () => boolean;
priorityWindowSize?: number;
}
@@ -57,11 +56,14 @@ export function createSubtitlePrefetchService(
let paused = false;
let currentRunId = 0;
// A run is a single bounded pass over one file's cues, deduped by `warmedKeys` and by
// `hasCachedTokenization`, so the worst case is one tokenization per cue. The cache is
// an LRU and bounds its own memory, so a full cache is not a reason to stop warming;
// stopping there used to leave the tail of longer media permanently uncached.
async function tokenizeCueList(
cuesToProcess: SubtitleCue[],
runId: number,
warmedKeys: Set<string>,
options: { allowWhenCacheFull?: boolean } = {},
): Promise<void> {
for (const cue of cuesToProcess) {
if (stopped || runId !== currentRunId) {
@@ -77,10 +79,6 @@ export function createSubtitlePrefetchService(
return;
}
if (!options.allowWhenCacheFull && deps.isCacheFull()) {
return;
}
const cacheKey = normalizeSubtitleCacheKey(cue.text);
if (!cacheKey || warmedKeys.has(cacheKey) || deps.hasCachedTokenization?.(cue.text)) {
if (cacheKey) {
@@ -110,7 +108,7 @@ export function createSubtitlePrefetchService(
// Phase 1: Priority window
const priorityCues = computePriorityWindow(cues, currentTimeSeconds, windowSize);
await tokenizeCueList(priorityCues, runId, warmedKeys, { allowWhenCacheFull: true });
await tokenizeCueList(priorityCues, runId, warmedKeys);
if (stopped || runId !== currentRunId) {
return;
@@ -0,0 +1,159 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import type { SubtitleData } from '../../types';
import { createSubtitleProcessingController } from './subtitle-processing-controller';
function flushMicrotasks(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 0));
}
test('new subtitle emits plain immediately without parallel tokenization or a stale overwrite', async () => {
const emitted: SubtitleData[] = [];
const resolvers = new Map<string, (value: SubtitleData | null) => void>();
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) =>
await new Promise<SubtitleData | null>((resolve) => {
resolvers.set(text, resolve);
}),
emitSubtitle: (payload) => emitted.push(payload),
});
controller.onSubtitleChange('first');
controller.onSubtitleChange('second');
assert.deepEqual(emitted, [
{ text: 'first', tokens: null },
{ text: 'second', tokens: null },
]);
assert.equal(resolvers.has('second'), false);
const resolveFirst = resolvers.get('first');
assert.ok(resolveFirst);
resolveFirst({ text: 'first', tokens: [] });
await flushMicrotasks();
assert.deepEqual(emitted, [
{ text: 'first', tokens: null },
{ text: 'second', tokens: null },
]);
assert.equal(resolvers.has('second'), true);
const resolveSecond = resolvers.get('second');
assert.ok(resolveSecond);
resolveSecond({ text: 'second', tokens: [] });
await flushMicrotasks();
assert.deepEqual(emitted, [
{ text: 'first', tokens: null },
{ text: 'second', tokens: null },
{ text: 'second', tokens: [] },
]);
});
test('subtitle clears immediately while previous tokenization remains pending', async () => {
const emitted: SubtitleData[] = [];
let resolveTokenization: ((value: SubtitleData | null) => void) | undefined;
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async () =>
await new Promise<SubtitleData | null>((resolve) => {
resolveTokenization = resolve;
}),
emitSubtitle: (payload) => emitted.push(payload),
});
controller.onSubtitleChange('first');
controller.onSubtitleChange('');
assert.deepEqual(emitted, [
{ text: 'first', tokens: null },
{ text: '', tokens: null },
]);
assert.ok(resolveTokenization);
resolveTokenization({ text: 'first', tokens: [] });
await flushMicrotasks();
assert.deepEqual(emitted, [
{ text: 'first', tokens: null },
{ text: '', tokens: null },
]);
});
test('returning to an uncached completed line emits it while another line is pending', async () => {
const emitted: SubtitleData[] = [];
let resolvePending: ((value: SubtitleData | null) => void) | undefined;
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => {
if (text === 'A') {
return { text, tokens: [] };
}
return await new Promise<SubtitleData | null>((resolve) => {
resolvePending = resolve;
});
},
emitSubtitle: (payload) => emitted.push(payload),
});
controller.onSubtitleChange('A');
await flushMicrotasks();
controller.invalidateTokenizationCache();
controller.onSubtitleChange('B');
controller.onSubtitleChange('A');
assert.deepEqual(emitted.at(-1), { text: 'A', tokens: null });
assert.ok(resolvePending);
resolvePending({ text: 'B', tokens: [] });
});
test('ABA subtitle changes reuse the matching first tokenization only after A is current again', async () => {
const emitted: SubtitleData[] = [];
const tokenizeCalls: string[] = [];
const resolvers: Array<(value: SubtitleData | null) => void> = [];
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => {
tokenizeCalls.push(text);
return await new Promise<SubtitleData | null>((resolve) => {
resolvers.push(resolve);
});
},
emitSubtitle: (payload) => emitted.push(payload),
});
controller.onSubtitleChange('A');
controller.onSubtitleChange('B');
controller.onSubtitleChange('A');
const resolveFirst = resolvers[0];
assert.ok(resolveFirst);
resolveFirst({ text: 'A', tokens: [{ value: 1 } as never] });
await flushMicrotasks();
assert.deepEqual(tokenizeCalls, ['A']);
assert.deepEqual(emitted, [
{ text: 'A', tokens: null },
{ text: 'B', tokens: null },
{ text: 'A', tokens: null },
{ text: 'A', tokens: [{ value: 1 } as never] },
]);
});
test('cached next subtitle does not downgrade to plain while processing is busy', async () => {
const emitted: SubtitleData[] = [];
let resolveTokenization: ((value: SubtitleData | null) => void) | undefined;
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) =>
await new Promise<SubtitleData | null>((resolve) => {
resolveTokenization = () => resolve({ text, tokens: [] });
}),
emitSubtitle: (payload) => emitted.push(payload),
});
controller.preCacheTokenization('cached', { text: 'cached', tokens: [] });
controller.onSubtitleChange('pending');
controller.onSubtitleChange('cached');
assert.deepEqual(emitted, [{ text: 'pending', tokens: null }]);
assert.ok(resolveTokenization);
resolveTokenization({ text: 'pending', tokens: [] });
await flushMicrotasks();
assert.deepEqual(emitted, [
{ text: 'pending', tokens: null },
{ text: 'cached', tokens: [] },
]);
});
@@ -7,18 +7,153 @@ function flushMicrotasks(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 0));
}
test('subtitle processing emits tokenized payload when tokenization succeeds', async () => {
test('subtitle processing emits plain payload immediately on cache miss, then tokenized payload', async () => {
const emitted: SubtitleData[] = [];
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
emitSubtitle: (payload) => emitted.push(payload),
});
controller.onSubtitleChange('字幕');
assert.deepEqual(emitted, [{ text: '字幕', tokens: null }]);
await flushMicrotasks();
assert.deepEqual(emitted, [
{ text: '字幕', tokens: null },
{ text: '字幕', tokens: [] },
]);
});
test('cache invalidation during pending tokenization does not re-emit the plain payload', async () => {
const emitted: SubtitleData[] = [];
const resolvers: Array<(value: SubtitleData | null) => void> = [];
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) =>
await new Promise<SubtitleData | null>((resolve) => {
resolvers.push(() => resolve({ text, tokens: [{ value: resolvers.length } as never] }));
}),
emitSubtitle: (payload) => emitted.push(payload),
});
controller.onSubtitleChange('行');
assert.deepEqual(emitted, [{ text: '行', tokens: null }]);
controller.invalidateTokenizationCache();
resolvers[0]?.({ text: '行', tokens: [] });
await flushMicrotasks();
// Retry for the new generation is now pending; still no duplicate plain emit.
assert.deepEqual(emitted, [{ text: '行', tokens: null }]);
resolvers[1]?.({ text: '行', tokens: [] });
await flushMicrotasks();
assert.deepEqual(emitted, [
{ text: '行', tokens: null },
{ text: '行', tokens: [{ value: 2 } as never] },
]);
});
test('failed refresh does not downgrade an already emitted tokenized subtitle', async () => {
const emitted: SubtitleData[] = [];
let tokenizeCalls = 0;
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => {
tokenizeCalls += 1;
if (tokenizeCalls > 1) {
throw new Error('tokenizer gone');
}
return { text, tokens: [] };
},
emitSubtitle: (payload) => emitted.push(payload),
});
controller.onSubtitleChange('行');
await flushMicrotasks();
controller.invalidateTokenizationCache();
controller.refreshCurrentSubtitle();
await flushMicrotasks();
assert.equal(tokenizeCalls, 2);
assert.deepEqual(emitted, [
{ text: '行', tokens: null },
{ text: '行', tokens: [] },
]);
});
test('null-tokenization refresh does not downgrade an already emitted tokenized subtitle', async () => {
const emitted: SubtitleData[] = [];
let tokenizeCalls = 0;
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => {
tokenizeCalls += 1;
return tokenizeCalls > 1 ? null : { text, tokens: [] };
},
emitSubtitle: (payload) => emitted.push(payload),
});
controller.onSubtitleChange('行');
await flushMicrotasks();
controller.invalidateTokenizationCache();
controller.refreshCurrentSubtitle();
await flushMicrotasks();
assert.equal(tokenizeCalls, 2);
assert.deepEqual(emitted, [
{ text: '行', tokens: null },
{ text: '行', tokens: [] },
]);
});
test('subtitle processing does not emit plain payload for cached lines', async () => {
const emitted: SubtitleData[] = [];
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
emitSubtitle: (payload) => emitted.push(payload),
});
controller.preCacheTokenization('字幕', { text: '字幕', tokens: [] });
controller.onSubtitleChange('字幕');
await flushMicrotasks();
assert.deepEqual(emitted, [{ text: '字幕', tokens: [] }]);
});
test('text that normalizes to nothing is never cached', () => {
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
emitSubtitle: () => {},
});
// Two different inputs both reduce to an empty key; sharing one entry would serve the
// first one's tokens for the second.
controller.preCacheTokenization(' ', { text: ' ', tokens: [] });
assert.equal(controller.hasCachedSubtitle(' '), false);
assert.equal(controller.hasCachedSubtitle('\\n'), false);
assert.equal(controller.consumeCachedSubtitle('\\n'), null);
});
test('subtitle processing shows plain line while tokenization is still pending', async () => {
const emitted: SubtitleData[] = [];
let resolveTokenization: ((value: SubtitleData | null) => void) | undefined;
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) =>
await new Promise<SubtitleData | null>((resolve) => {
resolveTokenization = () => resolve({ text, tokens: [] });
}),
emitSubtitle: (payload) => emitted.push(payload),
});
controller.onSubtitleChange('遅い行');
await flushMicrotasks();
assert.deepEqual(emitted, [{ text: '遅い行', tokens: null }]);
assert.ok(resolveTokenization);
resolveTokenization({ text: '遅い行', tokens: [] });
await flushMicrotasks();
assert.deepEqual(emitted, [
{ text: '遅い行', tokens: null },
{ text: '遅い行', tokens: [] },
]);
});
test('subtitle processing drops stale tokenization and delivers latest subtitle only once', async () => {
const emitted: SubtitleData[] = [];
let firstResolve: ((value: SubtitleData | null) => void) | undefined;
@@ -41,7 +176,11 @@ test('subtitle processing drops stale tokenization and delivers latest subtitle
await flushMicrotasks();
await flushMicrotasks();
assert.deepEqual(emitted, [{ text: 'second', tokens: [] }]);
assert.deepEqual(emitted, [
{ text: 'first', tokens: null },
{ text: 'second', tokens: null },
{ text: 'second', tokens: [] },
]);
});
test('subtitle processing skips duplicate subtitle emission', async () => {
@@ -60,7 +199,10 @@ test('subtitle processing skips duplicate subtitle emission', async () => {
controller.onSubtitleChange('same');
await flushMicrotasks();
assert.equal(emitted.length, 1);
assert.deepEqual(emitted, [
{ text: 'same', tokens: null },
{ text: 'same', tokens: [] },
]);
assert.equal(tokenizeCalls, 1);
});
@@ -84,7 +226,9 @@ test('subtitle processing reuses cached tokenization for repeated subtitle text'
assert.equal(tokenizeCalls, 2);
assert.deepEqual(emitted, [
{ text: 'first', tokens: null },
{ text: 'first', tokens: [] },
{ text: 'second', tokens: null },
{ text: 'second', tokens: [] },
{ text: 'first', tokens: [] },
]);
@@ -100,7 +244,48 @@ test('subtitle processing falls back to plain subtitle when tokenization returns
controller.onSubtitleChange('fallback');
await flushMicrotasks();
assert.deepEqual(
emitted,
[{ text: 'fallback', tokens: null }],
'plain payload should not be re-emitted when tokenization yields nothing new',
);
});
test('null tokenization is not cached and a later cue retries tokenization', async () => {
const emitted: SubtitleData[] = [];
const callsByText = new Map<string, number>();
let failNext = true;
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => {
callsByText.set(text, (callsByText.get(text) ?? 0) + 1);
if (text === 'fallback' && failNext) {
failNext = false;
return null;
}
return { text, tokens: [] };
},
emitSubtitle: (payload) => emitted.push(payload),
});
controller.onSubtitleChange('fallback');
await flushMicrotasks();
assert.equal(callsByText.get('fallback'), 1);
assert.equal(
controller.hasCachedSubtitle('fallback'),
false,
'plain fallback must not be cached when tokenization yields nothing',
);
assert.deepEqual(emitted, [{ text: 'fallback', tokens: null }]);
controller.onSubtitleChange('other');
await flushMicrotasks();
controller.onSubtitleChange('fallback');
await flushMicrotasks();
assert.equal(callsByText.get('fallback'), 2, 'later cue should retry tokenization');
assert.equal(controller.hasCachedSubtitle('fallback'), true);
assert.deepEqual(emitted.at(-1), { text: 'fallback', tokens: [] });
});
test('subtitle processing ignores duplicate current subtitle refresh without cache invalidation', async () => {
@@ -120,7 +305,10 @@ test('subtitle processing ignores duplicate current subtitle refresh without cac
await flushMicrotasks();
assert.equal(tokenizeCalls, 1);
assert.deepEqual(emitted, [{ text: 'same', tokens: [] }]);
assert.deepEqual(emitted, [
{ text: 'same', tokens: null },
{ text: 'same', tokens: [] },
]);
});
test('subtitle processing coalesces refresh requests while current subtitle is processing', async () => {
@@ -146,7 +334,10 @@ test('subtitle processing coalesces refresh requests while current subtitle is p
await flushMicrotasks();
assert.equal(tokenizeCalls, 1);
assert.deepEqual(emitted, [{ text: 'same', tokens: [] }]);
assert.deepEqual(emitted, [
{ text: 'same', tokens: null },
{ text: 'same', tokens: [] },
]);
});
test('subtitle processing refresh re-tokenizes after cache invalidation', async () => {
@@ -168,6 +359,7 @@ test('subtitle processing refresh re-tokenizes after cache invalidation', async
assert.equal(tokenizeCalls, 2);
assert.deepEqual(emitted, [
{ text: 'same', tokens: null },
{ text: 'same', tokens: [{ value: 1 } as never] },
{ text: 'same', tokens: [{ value: 2 } as never] },
]);
@@ -183,7 +375,10 @@ test('subtitle processing refresh can use explicit text override', async () => {
controller.refreshCurrentSubtitle('initial');
await flushMicrotasks();
assert.deepEqual(emitted, [{ text: 'initial', tokens: [] }]);
assert.deepEqual(emitted, [
{ text: 'initial', tokens: null },
{ text: 'initial', tokens: [] },
]);
});
test('subtitle processing cache invalidation only affects future subtitle events', async () => {
@@ -205,10 +400,10 @@ test('subtitle processing cache invalidation only affects future subtitle events
await flushMicrotasks();
assert.equal(callsByText.get('same'), 1);
assert.equal(emitted.length, 3);
assert.equal(emitted.length, 5);
controller.invalidateTokenizationCache();
assert.equal(emitted.length, 3);
assert.equal(emitted.length, 5);
controller.onSubtitleChange('different');
await flushMicrotasks();
@@ -308,25 +503,176 @@ test('hasCachedSubtitle checks prefetched entries without consuming them', async
assert.equal(controller.hasCachedSubtitle('猫\nです'), false);
});
test('isCacheFull returns false when cache is below limit', () => {
test('cache keeps every entry while below the limit', () => {
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => ({ text, tokens: null }),
tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
emitSubtitle: () => {},
cacheLimit: 8,
});
assert.equal(controller.isCacheFull(), false);
for (let i = 0; i < 8; i += 1) {
controller.preCacheTokenization(`line-${i}`, { text: `line-${i}`, tokens: [] });
}
assert.deepEqual(
Array.from({ length: 8 }, (_, i) => controller.hasCachedSubtitle(`line-${i}`)),
Array.from({ length: 8 }, () => true),
);
});
test('isCacheFull returns true when cache reaches limit', async () => {
test('cache evicts least recently used entries once the limit is reached', () => {
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
emitSubtitle: () => {},
cacheLimit: 3,
});
for (const line of ['a', 'b', 'c']) {
controller.preCacheTokenization(line, { text: line, tokens: [] });
}
// Touching 'a' makes 'b' the eviction candidate.
controller.consumeCachedSubtitle('a');
controller.preCacheTokenization('d', { text: 'd', tokens: [] });
assert.equal(controller.hasCachedSubtitle('b'), false);
assert.deepEqual(
['a', 'c', 'd'].map((line) => controller.hasCachedSubtitle(line)),
[true, true, true],
);
});
test('default cache limit covers a full-length title without evicting', () => {
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
emitSubtitle: () => {},
});
// Fill cache to the 256 limit
for (let i = 0; i < 256; i += 1) {
for (let i = 0; i < 2000; i += 1) {
controller.preCacheTokenization(`line-${i}`, { text: `line-${i}`, tokens: [] });
}
assert.equal(controller.isCacheFull(), true);
assert.equal(controller.hasCachedSubtitle('line-0'), true);
assert.equal(controller.hasCachedSubtitle('line-1999'), true);
});
test('onSubtitleChange reports whether processing was scheduled', async () => {
const emitted: SubtitleData[] = [];
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
emitSubtitle: (payload) => emitted.push(payload),
});
// New text schedules work, so an emit (and anything gated on it) will follow.
assert.equal(controller.onSubtitleChange('字幕'), true);
await flushMicrotasks();
// A repeat emits nothing, so callers must not wait on an emit that is never
// coming (subtitle prefetching would stay paused for the rest of the cue).
const emittedCount = emitted.length;
assert.equal(controller.onSubtitleChange('字幕'), false);
await flushMicrotasks();
assert.equal(emitted.length, emittedCount);
});
test('refreshCurrentSubtitle reports the empty-text emit that an in-flight run will deliver', async () => {
const emitted: SubtitleData[] = [];
let resolveFirst: ((value: SubtitleData | null) => void) | undefined;
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => {
if (text === '字幕') {
return await new Promise<SubtitleData | null>((resolve) => {
resolveFirst = resolve;
});
}
return { text, tokens: [] };
},
emitSubtitle: (payload) => emitted.push(payload),
});
controller.onSubtitleChange('字幕');
await flushMicrotasks();
// Clearing the subtitle while tokenization is in flight: the running loop
// picks the empty text up and emits it, so callers gated on that emit (the
// prefetch pause) must be told one is coming.
assert.equal(controller.refreshCurrentSubtitle(''), true);
resolveFirst?.({ text: '字幕', tokens: [] });
await flushMicrotasks();
await flushMicrotasks();
// '字幕' is the provisional plain emit the in-flight run already made before
// the refresh; '' is the emit the refresh promised.
assert.deepEqual(
emitted.map((payload) => payload.text),
['字幕', ''],
);
});
test('onProcessingSettled fires once after the queue drains, including runs that emit nothing', async () => {
const events: string[] = [];
let resolveFirst: ((value: SubtitleData | null) => void) | undefined;
let tokenizationFails = false;
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => {
if (tokenizationFails) {
return null;
}
if (text === '一行目') {
return await new Promise<SubtitleData | null>((resolve) => {
resolveFirst = resolve;
});
}
return { text, tokens: [] };
},
emitSubtitle: (payload) => events.push(`emit:${payload.text}`),
onProcessingSettled: () => events.push('settled'),
});
controller.onSubtitleChange('一行目');
await flushMicrotasks();
// A second line arrives before the first finishes: the controller still has
// work, so it must not report itself settled between the two.
controller.onSubtitleChange('二行目');
resolveFirst?.({ text: '一行目', tokens: [] });
await flushMicrotasks();
await flushMicrotasks();
assert.deepEqual(events, ['emit:一行目', 'emit:二行目', 'emit:二行目', 'settled']);
// Tokenization failure on a line already shown plain: nothing is emitted, and
// the settle signal is the only way a caller learns the work is over.
events.length = 0;
tokenizationFails = true;
controller.invalidateTokenizationCache();
assert.equal(controller.refreshCurrentSubtitle('二行目'), true);
await flushMicrotasks();
await flushMicrotasks();
assert.deepEqual(events, ['settled']);
});
test('notePlainSubtitleEmitted suppresses the controller repeat of a payload already shown', async () => {
const emitted: SubtitleData[] = [];
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
emitSubtitle: (payload) => emitted.push(payload),
});
// Autoplay priming paints the plain line itself, then asks for tokenization.
controller.notePlainSubtitleEmitted('字幕');
controller.refreshCurrentSubtitle('字幕');
await flushMicrotasks();
assert.deepEqual(emitted, [{ text: '字幕', tokens: [] }]);
});
test('refreshCurrentSubtitle reports no emit for empty text when nothing is running', async () => {
const emitted: SubtitleData[] = [];
const controller = createSubtitleProcessingController({
tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
emitSubtitle: (payload) => emitted.push(payload),
});
assert.equal(controller.refreshCurrentSubtitle(''), false);
await flushMicrotasks();
assert.deepEqual(emitted, []);
});
@@ -1,32 +1,76 @@
import type { SubtitleData } from '../../types';
import { normalizePlainSubtitleText } from './ass-text';
export interface SubtitleProcessingControllerDeps {
tokenizeSubtitle: (text: string) => Promise<SubtitleData | null>;
emitSubtitle: (payload: SubtitleData) => void;
/**
* Fires when the controller runs out of work: every scheduled line has been
* processed, whether it ended in an emit, a suppressed duplicate, or a
* tokenizer failure. Callers that hold a resource for the duration of
* processing (prefetch pausing) release it here rather than on an emit,
* which is not guaranteed to happen.
*/
onProcessingSettled?: () => void;
logDebug?: (message: string) => void;
now?: () => number;
cacheLimit?: number;
}
/**
* Pure memory bound on the LRU, not a coverage limit: prefetching runs to the end of a
* file regardless of cache pressure. Sized to hold a feature-length title (a 24-minute
* episode runs 300-400 lines, a 2-hour film ~2000) plus room for lines that repeat across
* episodes of a series, so openings and endings stay warm between titles.
*/
export const DEFAULT_SUBTITLE_TOKENIZATION_CACHE_LIMIT = 2500;
export interface SubtitleProcessingController {
onSubtitleChange: (text: string) => void;
refreshCurrentSubtitle: (textOverride?: string) => void;
/**
* Returns whether processing is now scheduled or already in flight for this
* event. A false return means the controller is idle and will do nothing, so
* onProcessingSettled will not fire; callers that pause work for the duration
* of processing (such as subtitle prefetching) must release it themselves.
*/
onSubtitleChange: (text: string) => boolean;
/** Same contract as onSubtitleChange: whether processing is pending. */
refreshCurrentSubtitle: (textOverride?: string) => boolean;
/**
* Records that this exact text has already been shown plain by someone else
* (autoplay priming paints its first frame before scheduling tokenization),
* so the controller does not repeat that payload on its way to the tokenized
* one.
*/
notePlainSubtitleEmitted: (text: string) => void;
invalidateTokenizationCache: () => void;
preCacheTokenization: (text: string, data: SubtitleData) => void;
consumeCachedSubtitle: (text: string) => SubtitleData | null;
hasCachedSubtitle: (text: string) => boolean;
isCacheFull: () => boolean;
}
/**
* Prefetched cues and live mpv text are both already decoded from ASS, so the key only
* has to settle whitespace for one authored line to resolve to one entry.
*
* An empty key is not a line: it is whatever normalization reduced to nothing. Callers
* must skip the cache for it rather than let every such input share one entry.
*/
export function normalizeSubtitleCacheKey(text: string): string {
return text.replace(/\r\n/g, '\n').replace(/\\N/g, '\n').replace(/\\n/g, '\n').trim();
return normalizePlainSubtitleText(text);
}
export function createSubtitleProcessingController(
deps: SubtitleProcessingControllerDeps,
): SubtitleProcessingController {
const SUBTITLE_TOKENIZATION_CACHE_LIMIT = 256;
const SUBTITLE_TOKENIZATION_CACHE_LIMIT =
deps.cacheLimit && deps.cacheLimit > 0
? deps.cacheLimit
: DEFAULT_SUBTITLE_TOKENIZATION_CACHE_LIMIT;
let latestText = '';
let lastEmittedText = '';
// Tracks the latest provisional plain emit across rapid changes and loop retries
// so the same line is never shown plain twice.
let lastPlainEmittedText: string | null = null;
let cacheGeneration = 0;
let lastEmittedGeneration = 0;
let processing = false;
@@ -36,6 +80,9 @@ export function createSubtitleProcessingController(
const getCachedTokenization = (text: string): SubtitleData | null => {
const cacheKey = normalizeSubtitleCacheKey(text);
if (!cacheKey) {
return null;
}
const cached = tokenizationCache.get(cacheKey);
if (!cached) {
return null;
@@ -47,7 +94,11 @@ export function createSubtitleProcessingController(
};
const setCachedTokenization = (text: string, payload: SubtitleData): void => {
tokenizationCache.set(normalizeSubtitleCacheKey(text), payload);
const cacheKey = normalizeSubtitleCacheKey(text);
if (!cacheKey) {
return;
}
tokenizationCache.set(cacheKey, payload);
while (tokenizationCache.size > SUBTITLE_TOKENIZATION_CACHE_LIMIT) {
const firstKey = tokenizationCache.keys().next().value;
if (firstKey !== undefined) {
@@ -70,9 +121,12 @@ export function createSubtitleProcessingController(
const startedAtMs = now();
if (!text.trim()) {
deps.emitSubtitle({ text, tokens: null });
if (lastPlainEmittedText !== text) {
deps.emitSubtitle({ text, tokens: null });
}
lastEmittedText = text;
lastEmittedGeneration = generation;
lastPlainEmittedText = null;
break;
}
@@ -82,11 +136,25 @@ export function createSubtitleProcessingController(
if (cachedTokenized) {
output = cachedTokenized;
} else {
// Cache miss: show the plain line on time; the tokenized payload
// upgrades it once ready. Skipped on refreshes of an already
// emitted line so downstream consumers never see a downgrade.
if (text !== lastEmittedText && text !== lastPlainEmittedText) {
deps.emitSubtitle({ text, tokens: null });
lastPlainEmittedText = text;
}
const tokenized = await deps.tokenizeSubtitle(text);
// A null result is a transient tokenizer failure, not a verdict on
// the line: caching the plain fallback would pin it untokenized for
// every later occurrence.
if (tokenized) {
output = tokenized;
// A result computed before an invalidation must not repopulate the
// fresh cache, or the retry below would serve the stale entry.
if (generation === cacheGeneration) {
setCachedTokenization(text, tokenized);
}
}
setCachedTokenization(text, output);
}
} catch (error) {
deps.logDebug?.(`Subtitle tokenization failed: ${(error as Error).message}`);
@@ -107,9 +175,16 @@ export function createSubtitleProcessingController(
continue;
}
deps.emitSubtitle(output);
// An untokenized result adds nothing when this line was already shown,
// either provisionally or as an earlier full emit (failed refresh) —
// emitting it would duplicate or downgrade what is on screen.
const plainAlreadyShown = lastPlainEmittedText === text || lastEmittedText === text;
if (!(output.tokens === null && output.text === text && plainAlreadyShown)) {
deps.emitSubtitle(output);
}
lastEmittedText = text;
lastEmittedGeneration = generation;
lastPlainEmittedText = null;
deps.logDebug?.(
`Subtitle tokenization delivered; elapsed=${now() - startedAtMs}ms, staleDrops=${staleDropCount}`,
);
@@ -126,32 +201,53 @@ export function createSubtitleProcessingController(
(latestText.trim() && cacheGeneration !== lastEmittedGeneration)
) {
processLatest();
return;
}
// Nothing left to do: signal completion even when this run emitted
// nothing (suppressed duplicate, tokenizer failure), or callers waiting
// on the controller would wait forever.
deps.onProcessingSettled?.();
});
};
return {
onSubtitleChange: (text: string) => {
if (text === latestText) {
return;
// A run already in flight for this text will still emit for it.
return processing;
}
latestText = text;
if (
processing &&
text !== lastPlainEmittedText &&
!tokenizationCache.has(normalizeSubtitleCacheKey(text))
) {
deps.emitSubtitle({ text, tokens: null });
lastPlainEmittedText = text;
}
processLatest();
return true;
},
refreshCurrentSubtitle: (textOverride?: string) => {
if (typeof textOverride === 'string') {
latestText = textOverride;
}
if (!latestText.trim()) {
return;
// A run in flight will pick this up and emit the empty subtitle, so
// the caller is still waiting on an emit.
return processing;
}
if (
processing ||
(latestText === lastEmittedText && cacheGeneration === lastEmittedGeneration)
) {
return;
if (processing) {
return true;
}
if (latestText === lastEmittedText && cacheGeneration === lastEmittedGeneration) {
return false;
}
processLatest();
return true;
},
notePlainSubtitleEmitted: (text: string) => {
lastPlainEmittedText = text;
},
invalidateTokenizationCache: () => {
tokenizationCache.clear();
@@ -169,13 +265,12 @@ export function createSubtitleProcessingController(
latestText = text;
lastEmittedText = text;
lastEmittedGeneration = cacheGeneration;
lastPlainEmittedText = null;
return cached;
},
hasCachedSubtitle: (text: string) => {
return tokenizationCache.has(normalizeSubtitleCacheKey(text));
},
isCacheFull: () => {
return tokenizationCache.size >= SUBTITLE_TOKENIZATION_CACHE_LIMIT;
const cacheKey = normalizeSubtitleCacheKey(text);
return cacheKey.length > 0 && tokenizationCache.has(cacheKey);
},
};
}
+35
View File
@@ -1,6 +1,7 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
isSubtitleAnnotationUpgrade,
serializeInitialSubtitleWebsocketMessage,
serializeSubtitleMarkup,
serializeSubtitleWebsocketMessage,
@@ -13,6 +14,40 @@ const frequencyOptions = {
mode: 'banded' as const,
};
test('annotation upgrade requires matching text and cue timing', () => {
const current: SubtitleData = {
text: '字幕',
tokens: null,
startTime: 10,
endTime: 12,
};
assert.equal(
isSubtitleAnnotationUpgrade(current, {
...current,
tokens: [],
}),
true,
);
assert.equal(
isSubtitleAnnotationUpgrade(current, {
...current,
tokens: [],
startTime: 11,
}),
false,
);
assert.equal(
isSubtitleAnnotationUpgrade(current, {
...current,
text: '次の字幕',
tokens: [],
}),
false,
);
assert.equal(isSubtitleAnnotationUpgrade(current, current), false);
});
test('serializeSubtitleMarkup escapes plain text and preserves line breaks', () => {
const payload: SubtitleData = {
text: 'a < b\nx & y',
+14
View File
@@ -20,6 +20,20 @@ export type SubtitleWebsocketFrequencyOptions = {
export type SubtitleWebsocketPayloadMode = 'plain' | 'annotated';
export function isSubtitleAnnotationUpgrade(
current: SubtitleData | null,
next: SubtitleData,
): boolean {
return (
current !== null &&
current.tokens === null &&
next.tokens !== null &&
current.text === next.text &&
current.startTime === next.startTime &&
current.endTime === next.endTime
);
}
type SubtitleWebsocketMessageOptions = {
payloadMode?: SubtitleWebsocketPayloadMode;
};
+7 -37
View File
@@ -1651,9 +1651,11 @@ test('tokenizeSubtitle clears JLPT level from standalone Yomitan particle token'
assert.equal(result.tokens?.[0]?.jlptLevel, undefined);
});
test('tokenizeSubtitle returns null tokens for empty normalized text', async () => {
test('tokenizeSubtitle returns the normalized text when it comes out empty', async () => {
// Handing back the original would push whatever normalization dropped into app state
// as if it were subtitle text.
const result = await tokenizeSubtitle(' \\n ', makeDeps());
assert.deepEqual(result, { text: ' \\n ', tokens: null });
assert.deepEqual(result, { text: '', tokens: null });
});
test('tokenizeSubtitle normalizes newlines before Yomitan parse request', async () => {
@@ -2934,44 +2936,12 @@ test('tokenizeSubtitle preserves Yomitan compound token when MeCab components ar
return [];
}
if (script.includes('parseText')) {
return [
{
source: 'scanning-parser',
index: 0,
content: [
[
{
text: '取り組んで',
reading: 'とりくんで',
headwords: [[{ term: '取り組む' }]],
},
],
[
{
text: 'もらいます',
reading: 'もらいます',
headwords: [[{ term: 'もらう' }]],
},
],
],
},
];
}
return [
{
surface: '取り',
reading: 'とり',
headword: '取',
surface: '取り組んで',
reading: 'とりくんで',
headword: '取り組む',
startPos: 0,
endPos: 2,
},
{
surface: '組んで',
reading: 'くんで',
headword: '組む',
startPos: 2,
endPos: 5,
},
{
+57 -8
View File
@@ -27,6 +27,7 @@ import {
} from './tokenizer/yomitan-parser-runtime';
import type { YomitanTermFrequency } from './tokenizer/yomitan-parser-runtime';
import { isKanaChar } from './tokenizer/token-classification';
import { normalizePlainSubtitleText } from './ass-text';
const logger = createLogger('main:tokenizer');
@@ -70,6 +71,7 @@ export interface TokenizerServiceDeps {
getNameMatchImagesEnabled?: () => boolean;
getCharacterNameImage?: (term: string) => CharacterNameImage | null;
getCurrentCharacterDictionaryMediaId?: () => number | null;
getCharacterNameCandidates?: () => { key: string; forms: string[] } | null;
getFrequencyDictionaryEnabled?: () => boolean;
getFrequencyDictionaryMatchMode?: () => FrequencyDictionaryMatchMode;
getFrequencyRank?: FrequencyDictionaryLookup;
@@ -106,6 +108,7 @@ export interface TokenizerDepsRuntimeOptions {
getNameMatchImagesEnabled?: () => boolean;
getCharacterNameImage?: (term: string) => CharacterNameImage | null;
getCurrentCharacterDictionaryMediaId?: () => number | null;
getCharacterNameCandidates?: () => { key: string; forms: string[] } | null;
getFrequencyDictionaryEnabled?: () => boolean;
getFrequencyDictionaryMatchMode?: () => FrequencyDictionaryMatchMode;
getFrequencyRank?: FrequencyDictionaryLookup;
@@ -266,6 +269,7 @@ export function createTokenizerDepsRuntime(
getNameMatchImagesEnabled: options.getNameMatchImagesEnabled,
getCharacterNameImage: options.getCharacterNameImage,
getCurrentCharacterDictionaryMediaId: options.getCurrentCharacterDictionaryMediaId,
getCharacterNameCandidates: options.getCharacterNameCandidates,
getFrequencyDictionaryEnabled: options.getFrequencyDictionaryEnabled,
getFrequencyDictionaryMatchMode: options.getFrequencyDictionaryMatchMode ?? (() => 'headword'),
getFrequencyRank: options.getFrequencyRank,
@@ -716,15 +720,30 @@ function getAnnotationOptions(deps: TokenizerServiceDeps): TokenizerAnnotationOp
};
}
// Per-line stage durations for the pipeline debug log; every field is filled in
// by the stage that awaits the corresponding work.
interface TokenizationStageTimings {
scanMs?: number;
mecabMs?: number;
frequencyMs?: number;
annotateMs?: number;
}
async function parseWithYomitanInternalParser(
text: string,
deps: TokenizerServiceDeps,
options: TokenizerAnnotationOptions,
stageTimings?: TokenizationStageTimings,
): Promise<MergedToken[] | null> {
const scanStartedAtMs = Date.now();
const selectedTokens = await requestYomitanScanTokens(text, deps, logger, {
includeNameMatchMetadata: options.nameMatchEnabled,
currentCharacterDictionaryMediaId: deps.getCurrentCharacterDictionaryMediaId?.() ?? null,
nameCandidates: deps.getCharacterNameCandidates?.() ?? null,
});
if (stageTimings) {
stageTimings.scanMs = Date.now() - scanStartedAtMs;
}
if (!selectedTokens || selectedTokens.length === 0) {
return null;
}
@@ -757,6 +776,7 @@ async function parseWithYomitanInternalParser(
const frequencyRankPromise: Promise<YomitanFrequencyIndex> = options.frequencyEnabled
? (async () => {
const frequencyStartedAtMs = Date.now();
const frequencyMatchMode = options.frequencyMatchMode;
const termReadingList = buildYomitanFrequencyTermReadingList(
normalizedSelectedTokens,
@@ -767,12 +787,17 @@ async function parseWithYomitanInternalParser(
deps,
logger,
);
return buildYomitanFrequencyIndex(yomitanFrequencies);
const frequencyIndex = buildYomitanFrequencyIndex(yomitanFrequencies);
if (stageTimings) {
stageTimings.frequencyMs = Date.now() - frequencyStartedAtMs;
}
return frequencyIndex;
})()
: Promise.resolve({ byPair: new Map(), byTerm: new Map() });
const mecabEnrichmentPromise: Promise<MergedToken[]> = needsMecabPosEnrichment(options)
? (async () => {
const mecabStartedAtMs = Date.now();
try {
const mecabTokens = await deps.tokenizeWithMecab(text);
const enrichTokensWithMecab = deps.enrichTokensWithMecab ?? enrichTokensWithMecabAsync;
@@ -786,6 +811,10 @@ async function parseWithYomitanInternalParser(
`textLength=${text.length}`,
);
return normalizedSelectedTokens;
} finally {
if (stageTimings) {
stageTimings.mecabMs = Date.now() - mecabStartedAtMs;
}
}
})()
: Promise.resolve(normalizedSelectedTokens);
@@ -858,14 +887,14 @@ export async function tokenizeSubtitle(
text: string,
deps: TokenizerServiceDeps,
): Promise<SubtitleData> {
const displayText = text
.replace(/\r\n/g, '\n')
.replace(/\\N/g, '\n')
.replace(/\\n/g, '\n')
.trim();
const displayText = normalizePlainSubtitleText(text);
// ASS decoding already happened upstream (cue parser for files, mpv for live text), so
// all this drops is whitespace -- but a whitespace-only line still normalizes to empty.
// Return the normalized form anyway: handing back the original would put a blank line
// into application state as if it were subtitle text.
if (!displayText) {
return { text, tokens: null };
return { text: displayText, tokens: null };
}
const tokenizeText = displayText
@@ -876,15 +905,35 @@ export async function tokenizeSubtitle(
const annotationOptions = getAnnotationOptions(deps);
annotationOptions.sourceText = tokenizeText;
const yomitanTokens = await parseWithYomitanInternalParser(tokenizeText, deps, annotationOptions);
const stageTimings: TokenizationStageTimings = {};
const startedAtMs = Date.now();
const logStageTimings = (tokenCount: number): void => {
logger.debug(
`Subtitle tokenization stages; textLength=${tokenizeText.length}, tokenCount=${tokenCount}, ` +
`scanMs=${stageTimings.scanMs ?? '-'}, mecabMs=${stageTimings.mecabMs ?? '-'}, ` +
`frequencyMs=${stageTimings.frequencyMs ?? '-'}, annotateMs=${stageTimings.annotateMs ?? '-'}, ` +
`totalMs=${Date.now() - startedAtMs}`,
);
};
const yomitanTokens = await parseWithYomitanInternalParser(
tokenizeText,
deps,
annotationOptions,
stageTimings,
);
if (yomitanTokens && yomitanTokens.length > 0) {
const annotateStartedAtMs = Date.now();
const annotatedTokens = await applyAnnotationStage(yomitanTokens, deps, annotationOptions);
stageTimings.annotateMs = Date.now() - annotateStartedAtMs;
const renderedTokens = applyCharacterNameImages(annotatedTokens, deps, annotationOptions);
logStageTimings(renderedTokens.length);
return {
text: displayText,
tokens: renderedTokens.length > 0 ? renderedTokens : null,
};
}
logStageTimings(0);
return { text: displayText, tokens: null };
}
@@ -0,0 +1,4 @@
// Title prefix of the dictionaries SubMiner generates per media. Lives on its
// own because both the main process and the injected scan runtime match on it,
// and the injected fragments interpolate it into their own source.
export const CHARACTER_DICTIONARY_TITLE_PREFIX = 'SubMiner Character Dictionary';
@@ -366,8 +366,11 @@ export function createReplayMessageStore(messages: GoldenRecordedMessage[]): Rep
};
}
async function runInjectedScriptInVm(script: string, store: ReplayMessageStore): Promise<unknown> {
return await vm.runInNewContext(script, {
// One persistent context per fixture, matching the real parser window: the
// scan runtime installs itself once into globalThis and later per-line call
// scripts reuse it.
function createInjectedScriptVm(store: ReplayMessageStore): (script: string) => Promise<unknown> {
const context = vm.createContext({
chrome: {
runtime: {
lastError: null,
@@ -393,6 +396,7 @@ async function runInjectedScriptInVm(script: string, store: ReplayMessageStore):
Set,
String,
});
return async (script: string) => await vm.runInContext(script, context);
}
export function createReplayTokenizerDeps(fixture: GoldenFixture): TokenizerServiceDeps {
@@ -400,13 +404,14 @@ export function createReplayTokenizerDeps(fixture: GoldenFixture): TokenizerServ
const scriptResults = new Map(
fixture.recording.scripts.map((entry) => [entry.sha256, entry] as const),
);
const runInjectedScriptInVm = createInjectedScriptVm(store);
const parserWindow = {
isDestroyed: () => false,
webContents: {
executeJavaScript: async (script: string) => {
try {
return await runInjectedScriptInVm(script, store);
return await runInjectedScriptInVm(script);
} catch (vmError) {
const recorded = scriptResults.get(hashInjectedScript(script));
if (recorded) {
@@ -8,6 +8,7 @@ import {
isKanaChar,
isKanaOnlyText,
isTokenPos2Excluded,
normalizeKana,
} from './token-classification';
const POS1_EXCLUSIONS = new Set(['助詞']);
@@ -29,6 +30,26 @@ function makeNoun(surface: string): MergedToken {
};
}
test('kana normalization folds halfwidth kana, composing the voiced pairs', () => {
// カ + ゙ is two code points for one character: without composing them, a
// halfwidth word counts as longer than the reading that spells it, which
// disqualifies the reading from known-word matching.
assert.equal(normalizeKana('ガク'), normalizeKana('ガク'));
assert.equal(normalizeKana('パン'), normalizeKana('パン'));
assert.equal(normalizeKana('ミナト'), 'みなと');
assert.ok(isKanaOnlyText('ガク'));
});
test('kana normalization leaves characters other than halfwidth kana alone', () => {
// The composition is scoped to the halfwidth runs: applied to the whole
// string, NFKC would also rewrite these into something the dictionary, the
// known-word list, and the frequency data were never keyed on.
assert.equal(normalizeKana('①ガ'), '①が');
assert.equal(normalizeKana('Aガ'), 'Aが');
assert.equal(normalizeKana('㍑ガ'), '㍑が');
assert.equal(normalizeKana('fiガ'), 'fiが');
});
test('kana classification excludes the katakana-hiragana double hyphen', () => {
assert.equal(isKanaChar(''), false);
assert.equal(isKanaOnlyText(''), false);
@@ -4,8 +4,20 @@ const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
const KATAKANA_CODEPOINT_START = 0x30a1;
const KATAKANA_CODEPOINT_END = 0x30f6;
// No `u` flag: the range is entirely BMP so it changes nothing here, and
// Bun's unicode-mode matcher mis-handles this class next to certain ligatures.
const HALFWIDTH_KANA_RUN = /[\uff66-\uff9f]+/g;
// NFKC over the halfwidth kana only, never the whole string: it composes the
// voiced pairs (カ + ゙) into single characters so ガク compares equal to ガク
// instead of counting one character longer than the word it spells, but run
// over everything it would also rewrite unrelated text (① → 1, ㍑ → リットル).
function composeHalfwidthKana(text: string): string {
return text.replace(HALFWIDTH_KANA_RUN, (run) => run.normalize('NFKC'));
}
export function normalizeKana(text: string): string {
const raw = text.trim();
const raw = composeHalfwidthKana(text).trim();
if (!raw) {
return '';
}
@@ -0,0 +1,150 @@
// Dictionary classification for the injected scan runtime: which dictionaries
// an entry came from, and whether it is a SubMiner character entry for the
// media being watched. Both walk nested entry data, so both are memoized on the
// entry object by the runtime that hosts them.
import { CHARACTER_DICTIONARY_TITLE_PREFIX } from './character-dictionary-title';
// The prefix is interpolated into generated regex source, so metacharacters in
// it would change what the pattern matches (or fail to compile).
const ESCAPED_TITLE_PREFIX = CHARACTER_DICTIONARY_TITLE_PREFIX.replace(
/[.*+?^${}()|[\]\\]/g,
'\\$&',
);
const TITLE_MEDIA_ID_PATTERN = ESCAPED_TITLE_PREFIX + String.raw`[^\d]*(?:AniList\s*)?(\d+)`;
export const YOMITAN_DICTIONARY_CLASSIFICATION_HELPERS = String.raw`
function normalizeWordClasses(headword) {
if (!Array.isArray(headword?.wordClasses)) { return undefined; }
const classes = headword.wordClasses.filter((wordClass) => typeof wordClass === "string" && wordClass.trim().length > 0);
return classes.length > 0 ? classes : undefined;
}
function appendDictionaryNames(target, value) {
if (!value || typeof value !== 'object') {
return;
}
const candidates = [
value.dictionary,
value.dictionaryName,
value.name,
value.title,
value.dictionaryTitle,
value.dictionaryAlias
];
for (const candidate of candidates) {
if (typeof candidate === 'string' && candidate.trim().length > 0) {
target.push(candidate.trim());
}
}
}
// Memoized on the entry object: termsFind results are cached across
// lines, so the same entries come back for every repeated lookup, and
// each one is classified several times per scan (name pre-pass,
// headword preference, every retry window).
function getDictionaryEntryNames(entry) {
if (!entry || typeof entry !== 'object') { return []; }
const cached = dictionaryEntryNamesCache.get(entry);
if (cached !== undefined) { return cached; }
const names = [];
appendDictionaryNames(names, entry);
for (const definition of entry?.definitions || []) {
appendDictionaryNames(names, definition);
}
for (const frequency of entry?.frequencies || []) {
appendDictionaryNames(names, frequency);
}
for (const pronunciation of entry?.pronunciations || []) {
appendDictionaryNames(names, pronunciation);
}
dictionaryEntryNamesCache.set(entry, names);
return names;
}
// Cached per scan rather than per runtime: the answer depends on
// includeNameMatchMetadata, which is a per-call parameter.
const nameDictionaryEntryCache = new WeakMap();
function isNameDictionaryEntry(entry) {
if (!includeNameMatchMetadata || !entry || typeof entry !== 'object') {
return false;
}
const cached = nameDictionaryEntryCache.get(entry);
if (cached !== undefined) { return cached; }
const isName = getDictionaryEntryNames(entry).some((name) => name.startsWith(${JSON.stringify(CHARACTER_DICTIONARY_TITLE_PREFIX)}));
nameDictionaryEntryCache.set(entry, isName);
return isName;
}
const TITLE_MEDIA_ID_REGEX = new RegExp(${JSON.stringify(TITLE_MEDIA_ID_PATTERN)}, 'i');
function parseSubMinerMediaIdFromString(value) {
const imageMatch = value.match(/\bimg\/m(\d+)-/i);
if (imageMatch) {
const parsed = Number.parseInt(imageMatch[1], 10);
if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
}
const titleMatch = value.match(TITLE_MEDIA_ID_REGEX);
if (titleMatch) {
const parsed = Number.parseInt(titleMatch[1], 10);
if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
}
return null;
}
function parseSubMinerMediaIdCandidate(value) {
if (typeof value === 'number' && Number.isSafeInteger(value) && value > 0) {
return value;
}
if (typeof value === 'string' && /^\d+$/.test(value.trim())) {
const parsed = Number.parseInt(value.trim(), 10);
if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
}
return null;
}
function collectSubMinerMediaIds(value, target) {
if (typeof value === 'string') {
const parsed = parseSubMinerMediaIdFromString(value);
if (parsed !== null) { target.add(parsed); }
return;
}
if (!value || typeof value !== 'object') {
return;
}
if (Array.isArray(value)) {
for (const item of value) { collectSubMinerMediaIds(item, target); }
return;
}
const mediaIdCandidates = [
value.subminerMediaId,
value.subMinerMediaId,
value.characterDictionaryMediaId,
value.data?.subminerMediaId,
value.data?.subMinerMediaId,
value.data?.characterDictionaryMediaId
];
for (const candidate of mediaIdCandidates) {
const parsed = parseSubMinerMediaIdCandidate(candidate);
if (parsed !== null) { target.add(parsed); }
}
for (const child of Object.values(value)) {
collectSubMinerMediaIds(child, target);
}
}
// Walking an entry collects media ids from every nested value, so this
// is the most expensive classification step; memoized on the entry for
// the same reason as the dictionary names above.
function getSubMinerMediaIds(entry) {
if (!entry || typeof entry !== 'object') { return EMPTY_MEDIA_ID_SET; }
const cached = subMinerMediaIdsCache.get(entry);
if (cached !== undefined) { return cached; }
const mediaIds = new Set();
collectSubMinerMediaIds(entry, mediaIds);
subMinerMediaIdsCache.set(entry, mediaIds);
return mediaIds;
}
function isCurrentMediaNameDictionaryEntry(entry) {
if (!isNameDictionaryEntry(entry)) {
return false;
}
if (currentCharacterDictionaryMediaId === null) {
return true;
}
const mediaIds = getSubMinerMediaIds(entry);
return mediaIds.size === 0 || mediaIds.has(currentCharacterDictionaryMediaId);
}
`;
@@ -0,0 +1,135 @@
// Frequency-rank resolution for the injected scan runtime: reads the many
// shapes a Yomitan frequency entry can take and picks the best rank for a
// headword, honouring per-dictionary priority and occurrence-vs-rank mode.
export const YOMITAN_FREQUENCY_HELPERS = String.raw`
function parsePositiveFrequencyNumber(value) {
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
return Math.max(1, Math.floor(value));
}
if (typeof value === 'string') {
const numericMatch = value.trim().match(/[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?/)?.[0];
if (!numericMatch) { return null; }
const parsed = Number.parseFloat(numericMatch);
if (!Number.isFinite(parsed) || parsed <= 0) { return null; }
return Math.max(1, Math.floor(parsed));
}
if (Array.isArray(value)) {
for (const item of value) {
const parsed = parsePositiveFrequencyNumber(item);
if (parsed !== null) { return parsed; }
}
}
return null;
}
function parseDisplayFrequencyNumber(value) {
if (typeof value === 'string') {
const leadingDigits = value.trim().match(/^\d+/)?.[0];
if (!leadingDigits) { return null; }
const parsed = Number.parseInt(leadingDigits, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
}
return parsePositiveFrequencyNumber(value);
}
function getFrequencyDictionaryName(frequency) {
const candidates = [
frequency?.dictionary,
frequency?.dictionaryName,
frequency?.name,
frequency?.title,
frequency?.dictionaryTitle,
frequency?.dictionaryAlias
];
for (const candidate of candidates) {
if (typeof candidate === 'string' && candidate.trim().length > 0) {
return candidate.trim();
}
}
return null;
}
function getBestFrequencyRank(dictionaryEntry, headwordIndex, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
let best = null;
const headwordCount = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords.length : 0;
for (const frequency of dictionaryEntry?.frequencies || []) {
if (!frequency || typeof frequency !== 'object') { continue; }
const frequencyHeadwordIndex = frequency.headwordIndex;
if (typeof frequencyHeadwordIndex === 'number') {
if (frequencyHeadwordIndex !== headwordIndex) { continue; }
} else if (headwordCount > 1) {
continue;
}
const dictionary = getFrequencyDictionaryName(frequency);
if (!dictionary) { continue; }
if (dictionaryFrequencyModeByName[dictionary] === 'occurrence-based') { continue; }
const rank =
parseDisplayFrequencyNumber(frequency.displayValue) ??
parsePositiveFrequencyNumber(frequency.frequency);
if (rank === null) { continue; }
const priorityRaw = dictionaryPriorityByName[dictionary];
const fallbackPriority =
typeof frequency.dictionaryIndex === 'number' && Number.isFinite(frequency.dictionaryIndex)
? Math.max(0, Math.floor(frequency.dictionaryIndex))
: Number.MAX_SAFE_INTEGER;
const priority =
typeof priorityRaw === 'number' && Number.isFinite(priorityRaw)
? Math.max(0, Math.floor(priorityRaw))
: fallbackPriority;
if (best === null || priority < best.priority || (priority === best.priority && rank < best.rank)) {
best = { priority, rank };
}
}
return best?.rank ?? null;
}
function hasExactSource(headword, token, requirePrimary) {
for (const src of headword?.sources || []) {
if (src.originalText !== token) { continue; }
if (requirePrimary && !src.isPrimary) { continue; }
if (src.matchType !== 'exact') { continue; }
return true;
}
return false;
}
function collectExactHeadwordMatches(dictionaryEntries, token, requirePrimary) {
const matches = [];
for (const dictionaryEntry of dictionaryEntries || []) {
const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
const headword = headwords[headwordIndex];
if (!hasExactSource(headword, token, requirePrimary)) { continue; }
matches.push({ dictionaryEntry, headword, headwordIndex });
}
}
return matches;
}
function sameHeadword(match, preferredMatch) {
if (!match || !preferredMatch) {
return false;
}
if (match.headword?.term !== preferredMatch.headword?.term) {
return false;
}
const matchReading = typeof match.headword?.reading === 'string' ? match.headword.reading : '';
const preferredReading =
typeof preferredMatch.headword?.reading === 'string' ? preferredMatch.headword.reading : '';
if (!matchReading || !preferredReading) {
return true;
}
return matchReading === preferredReading;
}
function getBestFrequencyRankForMatches(matches, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
let best = null;
for (const match of matches) {
const rank = getBestFrequencyRank(
match.dictionaryEntry,
match.headwordIndex,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
);
if (rank === null) { continue; }
if (best === null || rank < best) {
best = rank;
}
}
return best;
}
`;
@@ -0,0 +1,170 @@
// Furigana distribution for the injected scan runtime: splits a headword and
// its reading into the segments a token carries, including the inflected case
// where the matched source text differs from the dictionary form.
export const YOMITAN_FURIGANA_HELPERS = String.raw`
function createFuriganaSegment(text, reading) { return {text, reading}; }
function getSegmentReadingContribution(segment) {
if (typeof segment.reading === "string" && segment.reading.length > 0) { return segment.reading; }
const segmentText = typeof segment.text === "string" ? segment.text : "";
const isKanaOnly = segmentText.length > 0 && [...segmentText].every((char) => isCodePointKana(char.codePointAt(0)));
return isKanaOnly ? convertHalfwidthKanaToKatakana(segmentText) : "";
}
function getProlongedHiragana(previousCharacter) {
switch (previousCharacter) {
case "あ": case "か": case "が": case "さ": case "ざ": case "た": case "だ": case "な": case "は": case "ば": case "ぱ": case "ま": case "や": case "ら": case "わ": case "ぁ": case "ゃ": case "ゎ": return "あ";
case "い": case "き": case "ぎ": case "し": case "じ": case "ち": case "ぢ": case "に": case "ひ": case "び": case "ぴ": case "み": case "り": case "ぃ": return "い";
case "う": case "く": case "ぐ": case "す": case "ず": case "つ": case "づ": case "ぬ": case "ふ": case "ぶ": case "ぷ": case "む": case "ゆ": case "る": case "ぅ": case "ゅ": return "う";
case "え": case "け": case "げ": case "せ": case "ぜ": case "て": case "で": case "ね": case "へ": case "べ": case "ぺ": case "め": case "れ": case "ぇ": return "え";
case "お": case "こ": case "ご": case "そ": case "ぞ": case "と": case "ど": case "の": case "ほ": case "ぼ": case "ぽ": case "も": case "よ": case "ろ": case "を": case "ぉ": case "ょ": return "う";
default: return null;
}
}
function getFuriganaKanaSegments(text, reading) {
const newSegments = [];
let start = 0;
let state = (reading[0] === text[0]);
for (let i = 1; i < text.length; ++i) {
const newState = (reading[i] === text[i]);
if (state === newState) { continue; }
newSegments.push(createFuriganaSegment(text.substring(start, i), state ? '' : reading.substring(start, i)));
state = newState;
start = i;
}
newSegments.push(createFuriganaSegment(text.substring(start), state ? '' : reading.substring(start)));
return newSegments;
}
function convertKatakanaToHiragana(text, keepProlongedSoundMarks = false) {
let result = '';
const offset = (HIRAGANA_CONVERSION_RANGE[0] - KATAKANA_CONVERSION_RANGE[0]);
for (let char of text) {
const codePoint = char.codePointAt(0);
switch (codePoint) {
case KATAKANA_SMALL_KA_CODE_POINT:
case KATAKANA_SMALL_KE_CODE_POINT:
break;
case KANA_PROLONGED_SOUND_MARK_CODE_POINT:
case HALFWIDTH_KANA_PROLONGED_SOUND_MARK_CODE_POINT:
char = "ー";
if (!keepProlongedSoundMarks && result.length > 0) {
const char2 = getProlongedHiragana(result[result.length - 1]);
if (char2 !== null) { char = char2; }
}
break;
default:
if (isCodePointInRange(codePoint, KATAKANA_CONVERSION_RANGE)) {
char = String.fromCodePoint(codePoint + offset);
break;
}
// Halfwidth katakana folds too, or a name written that way would
// match neither a candidate form nor its own reading.
const halfwidthHiragana = convertHalfwidthKanaCodePointToHiragana(codePoint);
if (halfwidthHiragana !== null) { char = halfwidthHiragana; }
break;
}
result += char;
}
return result;
}
function segmentizeFurigana(reading, readingNormalized, groups, groupsStart) {
const groupCount = groups.length - groupsStart;
if (groupCount <= 0) { return reading.length === 0 ? [] : null; }
const group = groups[groupsStart];
const {isKana, text} = group;
if (isKana) {
if (group.textNormalized !== null && readingNormalized.startsWith(group.textNormalized)) {
const segments = segmentizeFurigana(reading.substring(text.length), readingNormalized.substring(text.length), groups, groupsStart + 1);
if (segments !== null) {
if (reading.startsWith(text)) { segments.unshift(createFuriganaSegment(text, '')); }
else { segments.unshift(...getFuriganaKanaSegments(text, reading)); }
return segments;
}
}
return null;
}
let result = null;
for (let i = reading.length; i >= text.length; --i) {
const segments = segmentizeFurigana(reading.substring(i), readingNormalized.substring(i), groups, groupsStart + 1);
if (segments !== null) {
if (result !== null) { return null; }
segments.unshift(createFuriganaSegment(text, reading.substring(0, i)));
result = segments;
}
if (groupCount === 1) { break; }
}
return result;
}
function distributeFurigana(term, reading) {
if (reading === term) { return [createFuriganaSegment(term, '')]; }
const groups = [];
let groupPre = null;
let isKanaPre = null;
for (const c of term) {
const isKana = isCodePointKana(c.codePointAt(0));
if (isKana === isKanaPre) { groupPre.text += c; }
else {
groupPre = {isKana, text: c, textNormalized: null};
groups.push(groupPre);
isKanaPre = isKana;
}
}
for (const group of groups) {
if (group.isKana) { group.textNormalized = convertKatakanaToHiragana(group.text); }
}
const segments = segmentizeFurigana(reading, convertKatakanaToHiragana(reading), groups, 0);
return segments !== null ? segments : [createFuriganaSegment(term, reading)];
}
function getStemLength(text1, text2) {
const minLength = Math.min(text1.length, text2.length);
if (minLength === 0) { return 0; }
let i = 0;
while (true) {
const char1 = text1.codePointAt(i);
const char2 = text2.codePointAt(i);
if (char1 !== char2) { break; }
const charLength = String.fromCodePoint(char1).length;
i += charLength;
if (i >= minLength) {
if (i > minLength) { i -= charLength; }
break;
}
}
return i;
}
function distributeFuriganaInflected(term, reading, source) {
const termNormalized = convertKatakanaToHiragana(term);
const readingNormalized = convertKatakanaToHiragana(reading);
const sourceNormalized = convertKatakanaToHiragana(source);
let mainText = term;
let stemLength = getStemLength(termNormalized, sourceNormalized);
const readingStemLength = getStemLength(readingNormalized, sourceNormalized);
if (readingStemLength > 0 && readingStemLength >= stemLength) {
mainText = reading;
stemLength = readingStemLength;
reading = source.substring(0, stemLength) + reading.substring(stemLength);
}
const segments = [];
if (stemLength > 0) {
mainText = source.substring(0, stemLength) + mainText.substring(stemLength);
const segments2 = distributeFurigana(mainText, reading);
let consumed = 0;
for (const segment of segments2) {
const start = consumed;
consumed += segment.text.length;
if (consumed < stemLength) { segments.push(segment); }
else if (consumed === stemLength) { segments.push(segment); break; }
else {
if (start < stemLength) { segments.push(createFuriganaSegment(mainText.substring(start, stemLength), '')); }
break;
}
}
}
if (stemLength < source.length) {
const remainder = source.substring(stemLength);
const last = segments[segments.length - 1];
if (last && last.reading.length === 0) { last.text += remainder; }
else { segments.push(createFuriganaSegment(remainder, '')); }
}
return segments;
}
`;
@@ -0,0 +1,45 @@
// Kana classification and normalization for the injected scan runtime: the
// code-point ranges the walk tests every character against, and the folds that
// let halfwidth and katakana spellings compare equal to their dictionary form.
import { HAN_CODE_POINT_RANGES } from '../../text/han-code-points';
export const YOMITAN_KANA_HELPERS = String.raw`
const HIRAGANA_CONVERSION_RANGE = [0x3041, 0x3096];
const KATAKANA_CONVERSION_RANGE = [0x30a1, 0x30f6];
const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc;
const KATAKANA_SMALL_KA_CODE_POINT = 0x30f5;
const KATAKANA_SMALL_KE_CODE_POINT = 0x30f6;
const KANA_RANGES = [[0x3040, 0x309f], [0x30a0, 0x30ff], [0xff66, 0xff9f]];
const HALFWIDTH_KATAKANA_RANGE = [0xff66, 0xff9d];
const HALFWIDTH_KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0xff70;
// Folded one code point to one, so every index into a normalized string
// still lines up with the original text — the name-candidate prefilter
// and the furigana stem matching both index back into it. The standalone
// voiced marks (゙ ゚) have no one-character equivalent and stay as they are.
const HALFWIDTH_KATAKANA_TO_HIRAGANA = "をぁぃぅぇぉゃゅょっーあいうえおかきくけこさしすせそたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわん";
function convertHalfwidthKanaCodePointToHiragana(codePoint) {
if (codePoint < HALFWIDTH_KATAKANA_RANGE[0] || codePoint > HALFWIDTH_KATAKANA_RANGE[1]) { return null; }
return HALFWIDTH_KATAKANA_TO_HIRAGANA[codePoint - HALFWIDTH_KATAKANA_RANGE[0]] || null;
}
// Halfwidth katakana is kana here but not to the rest of the pipeline
// (known-word matching and frequency lookups only fold fullwidth), so a
// reading taken from halfwidth text is written the way the fullwidth
// katakana path already writes it. NFKC rather than the per-code-point
// table: this is the one place where nothing indexes back into the
// result, so a voiced pair (カ + ゙) can compose into the single ガ it
// means instead of leaving a stray combining mark in the reading. Scoped
// to the halfwidth runs, because NFKC over everything else rewrites
// characters that have nothing to do with kana (① → 1, ㍑ → リットル).
function convertHalfwidthKanaToKatakana(text) {
return text.replace(/[ヲ-゚]+/g, (run) => run.normalize("NFKC"));
}
// Han ranges come from the shared table so the scan walk and the character
// dictionary agree on what a kanji is (supplementary planes included).
// Halfwidth katakana counts as Japanese text: a name written that way has
// to reach the greedy pre-pass, which has its own handling for it.
const JAPANESE_RANGES = [[0x3040, 0x30ff], [0xff66, 0xff9f], ...${JSON.stringify(HAN_CODE_POINT_RANGES)}];
function isCodePointInRange(codePoint, range) { return codePoint >= range[0] && codePoint <= range[1]; }
function isCodePointInRanges(codePoint, ranges) { return ranges.some((range) => isCodePointInRange(codePoint, range)); }
function isCodePointKana(codePoint) { return isCodePointInRanges(codePoint, KANA_RANGES); }
function isCodePointJapanese(codePoint) { return isCodePointInRanges(codePoint, JAPANESE_RANGES); }
`;
@@ -0,0 +1,79 @@
// Match selection for the injected scan runtime: picks the headword a position
// tokenizes to, and the longest name or generic match in a window, which is how
// the greedy name pre-pass decides what to reserve.
export const YOMITAN_MATCH_SELECTION_HELPERS = String.raw`
function findLongestNameMatch(dictionaryEntries, textWindow) {
let best = null;
for (const dictionaryEntry of dictionaryEntries || []) {
if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; }
const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
const headword = headwords[headwordIndex];
for (const src of headword?.sources || []) {
if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
const originalText = typeof src.originalText === 'string' ? src.originalText : '';
if (!originalText || !textWindow.startsWith(originalText)) { continue; }
if (best === null || originalText.length > best.sourceLength) {
best = { dictionaryEntry, headword, headwordIndex, sourceLength: originalText.length };
}
}
}
}
return best;
}
function findLongestGenericMatchLength(dictionaryEntries, textWindow) {
let best = 0;
for (const dictionaryEntry of dictionaryEntries || []) {
if (isNameDictionaryEntry(dictionaryEntry)) { continue; }
const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
for (const headword of headwords) {
for (const src of headword?.sources || []) {
if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
const originalText = typeof src.originalText === 'string' ? src.originalText : '';
if (!originalText || !textWindow.startsWith(originalText)) { continue; }
if (originalText.length > best) { best = originalText.length; }
}
}
}
return best;
}
function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
const currentMediaDictionaryEntries =
currentCharacterDictionaryMediaId === null
? (dictionaryEntries || [])
: (dictionaryEntries || []).filter((entry) => {
if (!isNameDictionaryEntry(entry)) { return true; }
return isCurrentMediaNameDictionaryEntry(entry);
});
const exactPrimaryMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, true);
let matchedNameDictionary = false;
if (includeNameMatchMetadata) {
// Every match already comes from currentMediaDictionaryEntries, so
// classifying its own entry is enough.
for (const match of exactPrimaryMatches) {
if (!isCurrentMediaNameDictionaryEntry(match.dictionaryEntry)) { continue; }
matchedNameDictionary = true;
break;
}
}
const preferredMatch = exactPrimaryMatches[0];
if (preferredMatch) {
const exactFrequencyMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, false)
.filter((match) => sameHeadword(match, preferredMatch));
return {
term: preferredMatch.headword.term,
reading: preferredMatch.headword.reading,
wordClasses: normalizeWordClasses(preferredMatch.headword),
isNameMatch:
matchedNameDictionary || isCurrentMediaNameDictionaryEntry(preferredMatch.dictionaryEntry),
frequencyRank: getBestFrequencyRankForMatches(
exactFrequencyMatches.length > 0 ? exactFrequencyMatches : exactPrimaryMatches,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
)
};
}
return null;
}
`;
File diff suppressed because it is too large Load Diff
@@ -3,6 +3,14 @@ import * as fs from 'fs';
import * as http from 'http';
import * as path from 'path';
import { selectYomitanParseTokens } from './parser-selection-stage';
import {
buildYomitanScanCallScript,
buildYomitanScanNameCandidatesScript,
CHARACTER_DICTIONARY_TITLE_PREFIX,
YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT,
YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL,
type YomitanFrequencyMode,
} from './yomitan-scan-runtime-script';
interface LoggerLike {
error: (message: string, ...args: unknown[]) => void;
@@ -22,8 +30,6 @@ interface YomitanParserRuntimeDeps {
createYomitanExtensionWindow?: (pageName: string) => Promise<BrowserWindow | null>;
}
type YomitanFrequencyMode = 'occurrence-based' | 'rank-based';
export interface YomitanDictionaryInfo {
title: string;
revision?: string | number;
@@ -74,13 +80,19 @@ export interface YomitanAddNoteResult {
}
const DEFAULT_YOMITAN_SCAN_LENGTH = 40;
const CHARACTER_DICTIONARY_TITLE_PREFIX = 'SubMiner Character Dictionary';
const yomitanProfileMetadataByWindow = new WeakMap<BrowserWindow, YomitanProfileMetadata>();
const yomitanProfileDiagnosticsLoggedByWindow = new WeakSet<BrowserWindow>();
const yomitanFrequencyCacheByWindow = new WeakMap<
BrowserWindow,
Map<string, YomitanTermFrequency[]>
>();
// Epoch passed with every scan request; the in-window termsFind cache clears
// itself when the epoch changes (dictionary imports, settings changes).
const yomitanScanCacheEpochByWindow = new WeakMap<BrowserWindow, number>();
function getYomitanScanCacheEpoch(window: BrowserWindow): number {
return yomitanScanCacheEpochByWindow.get(window) ?? 0;
}
function isObject(value: unknown): value is Record<string, unknown> {
return Boolean(value && typeof value === 'object');
@@ -99,6 +111,7 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
typeof entry.startPos === 'number' &&
typeof entry.endPos === 'number' &&
(entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean') &&
(entry.isUnparsedRun === undefined || typeof entry.isUnparsedRun === 'boolean') &&
(entry.frequencyRank === undefined || typeof entry.frequencyRank === 'number') &&
(entry.wordClasses === undefined ||
(Array.isArray(entry.wordClasses) &&
@@ -107,13 +120,9 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
);
}
function scanTokenSpanKey(token: YomitanScanToken): string {
return `${token.startPos}:${token.endPos}:${token.surface}`;
}
// Maps a parse-selected token to the scanner-token shape carried out of the
// parser runtime. Shared by both selectYomitanParseTokens fallback paths so the
// projected fields stay in sync as the shape changes.
// parser runtime, used by the parseText fallback path when the in-window
// scanner is unavailable.
function toYomitanScanToken(token: {
surface: string;
reading: string;
@@ -132,66 +141,6 @@ function toYomitanScanToken(token: {
};
}
// parseText segmentation is authoritative (it emits filler chunks for text the
// termsFind scanner skips), but only the termsFind scanner carries annotation
// metadata (isNameMatch, frequencyRank, headwordReading, wordClasses). Graft
// scanner tokens onto the parseText segmentation per matching span so one
// unmatched chunk degrades only itself instead of dropping the whole line's
// metadata.
//
// Exception: character-name tokens. The greedy name scan can re-segment text
// around a name (e.g. とヨータ → と + ヨータ instead of とヨー + タ), so
// parseText segmentation cannot be authoritative there. Each name span is
// expanded until it aligns with token boundaries in both segmentations, then
// the parse tokens inside are replaced with the scanner tokens.
function mergeScannerTokensIntoParseTokens(
parseScanTokens: YomitanScanToken[],
scannerTokens: YomitanScanToken[],
): YomitanScanToken[] {
const scannerTokensBySpan = new Map<string, YomitanScanToken>();
for (const token of scannerTokens) {
scannerTokensBySpan.set(scanTokenSpanKey(token), token);
}
const graftedTokens = parseScanTokens.map(
(token) => scannerTokensBySpan.get(scanTokenSpanKey(token)) ?? token,
);
const nameTokens = scannerTokens.filter((token) => token.isNameMatch === true);
if (nameTokens.length === 0) {
return graftedTokens;
}
const regions = nameTokens.map((token) => ({ start: token.startPos, end: token.endPos }));
const allTokens = [...parseScanTokens, ...scannerTokens];
let expanded = true;
while (expanded) {
expanded = false;
for (const region of regions) {
for (const token of allTokens) {
const overlaps = token.startPos < region.end && token.endPos > region.start;
const extendsBeyond = token.startPos < region.start || token.endPos > region.end;
if (overlaps && extendsBeyond) {
region.start = Math.min(region.start, token.startPos);
region.end = Math.max(region.end, token.endPos);
expanded = true;
}
}
}
}
const isInsideNameRegion = (token: YomitanScanToken): boolean =>
regions.some((region) => token.startPos >= region.start && token.endPos <= region.end);
const merged = graftedTokens.filter((token) => !isInsideNameRegion(token));
for (const token of scannerTokens) {
if (isInsideNameRegion(token)) {
merged.push(token);
}
}
merged.sort((a, b) => a.startPos - b.startPos || a.endPos - b.endPos);
return merged;
}
function makeTermReadingCacheKey(term: string, reading: string | null): string {
return `${term}\u0000${reading ?? ''}`;
}
@@ -208,6 +157,7 @@ function getWindowFrequencyCache(window: BrowserWindow): Map<string, YomitanTerm
function clearWindowCaches(window: BrowserWindow): void {
yomitanProfileMetadataByWindow.delete(window);
yomitanFrequencyCacheByWindow.delete(window);
yomitanScanCacheEpochByWindow.set(window, getYomitanScanCacheEpoch(window) + 1);
}
export function clearYomitanParserCachesForWindow(window: BrowserWindow): void {
clearWindowCaches(window);
@@ -704,6 +654,10 @@ async function ensureYomitanParserWindow(
if (readyPromise) {
await readyPromise;
}
// Eagerly install the scan runtime so the first subtitle line does not
// pay the install round trip; failures fall back to the per-request
// install-and-retry path.
await installYomitanScanRuntime(parserWindow).catch(() => {});
return true;
} catch (err) {
@@ -877,668 +831,42 @@ async function serveDictionaryZipOnce<T>(
}
}
const YOMITAN_SCANNING_HELPERS = String.raw`
const HIRAGANA_CONVERSION_RANGE = [0x3041, 0x3096];
const KATAKANA_CONVERSION_RANGE = [0x30a1, 0x30f6];
const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc;
const KATAKANA_SMALL_KA_CODE_POINT = 0x30f5;
const KATAKANA_SMALL_KE_CODE_POINT = 0x30f6;
const KANA_RANGES = [[0x3040, 0x309f], [0x30a0, 0x30ff]];
const JAPANESE_RANGES = [[0x3040, 0x30ff], [0x3400, 0x9fff]];
function isCodePointInRange(codePoint, range) { return codePoint >= range[0] && codePoint <= range[1]; }
function isCodePointInRanges(codePoint, ranges) { return ranges.some((range) => isCodePointInRange(codePoint, range)); }
function isCodePointKana(codePoint) { return isCodePointInRanges(codePoint, KANA_RANGES); }
function isCodePointJapanese(codePoint) { return isCodePointInRanges(codePoint, JAPANESE_RANGES); }
function createFuriganaSegment(text, reading) { return {text, reading}; }
function getSegmentReadingContribution(segment) {
if (typeof segment.reading === "string" && segment.reading.length > 0) { return segment.reading; }
const segmentText = typeof segment.text === "string" ? segment.text : "";
const isKanaOnly = segmentText.length > 0 && [...segmentText].every((char) => isCodePointKana(char.codePointAt(0)));
return isKanaOnly ? segmentText : "";
}
function getProlongedHiragana(previousCharacter) {
switch (previousCharacter) {
case "あ": case "か": case "が": case "さ": case "ざ": case "た": case "だ": case "な": case "は": case "ば": case "ぱ": case "ま": case "や": case "ら": case "わ": case "ぁ": case "ゃ": case "ゎ": return "あ";
case "い": case "き": case "ぎ": case "し": case "じ": case "ち": case "ぢ": case "に": case "ひ": case "び": case "ぴ": case "み": case "り": case "ぃ": return "い";
case "う": case "く": case "ぐ": case "す": case "ず": case "つ": case "づ": case "ぬ": case "ふ": case "ぶ": case "ぷ": case "む": case "ゆ": case "る": case "ぅ": case "ゅ": return "う";
case "え": case "け": case "げ": case "せ": case "ぜ": case "て": case "で": case "ね": case "へ": case "べ": case "ぺ": case "め": case "れ": case "ぇ": return "え";
case "お": case "こ": case "ご": case "そ": case "ぞ": case "と": case "ど": case "の": case "ほ": case "ぼ": case "ぽ": case "も": case "よ": case "ろ": case "を": case "ぉ": case "ょ": return "う";
default: return null;
}
}
function getFuriganaKanaSegments(text, reading) {
const newSegments = [];
let start = 0;
let state = (reading[0] === text[0]);
for (let i = 1; i < text.length; ++i) {
const newState = (reading[i] === text[i]);
if (state === newState) { continue; }
newSegments.push(createFuriganaSegment(text.substring(start, i), state ? '' : reading.substring(start, i)));
state = newState;
start = i;
}
newSegments.push(createFuriganaSegment(text.substring(start), state ? '' : reading.substring(start)));
return newSegments;
}
function convertKatakanaToHiragana(text, keepProlongedSoundMarks = false) {
let result = '';
const offset = (HIRAGANA_CONVERSION_RANGE[0] - KATAKANA_CONVERSION_RANGE[0]);
for (let char of text) {
const codePoint = char.codePointAt(0);
switch (codePoint) {
case KATAKANA_SMALL_KA_CODE_POINT:
case KATAKANA_SMALL_KE_CODE_POINT:
break;
case KANA_PROLONGED_SOUND_MARK_CODE_POINT:
if (!keepProlongedSoundMarks && result.length > 0) {
const char2 = getProlongedHiragana(result[result.length - 1]);
if (char2 !== null) { char = char2; }
}
break;
default:
if (isCodePointInRange(codePoint, KATAKANA_CONVERSION_RANGE)) {
char = String.fromCodePoint(codePoint + offset);
}
break;
}
result += char;
}
return result;
}
function segmentizeFurigana(reading, readingNormalized, groups, groupsStart) {
const groupCount = groups.length - groupsStart;
if (groupCount <= 0) { return reading.length === 0 ? [] : null; }
const group = groups[groupsStart];
const {isKana, text} = group;
if (isKana) {
if (group.textNormalized !== null && readingNormalized.startsWith(group.textNormalized)) {
const segments = segmentizeFurigana(reading.substring(text.length), readingNormalized.substring(text.length), groups, groupsStart + 1);
if (segments !== null) {
if (reading.startsWith(text)) { segments.unshift(createFuriganaSegment(text, '')); }
else { segments.unshift(...getFuriganaKanaSegments(text, reading)); }
return segments;
}
}
return null;
}
let result = null;
for (let i = reading.length; i >= text.length; --i) {
const segments = segmentizeFurigana(reading.substring(i), readingNormalized.substring(i), groups, groupsStart + 1);
if (segments !== null) {
if (result !== null) { return null; }
segments.unshift(createFuriganaSegment(text, reading.substring(0, i)));
result = segments;
}
if (groupCount === 1) { break; }
}
return result;
}
function distributeFurigana(term, reading) {
if (reading === term) { return [createFuriganaSegment(term, '')]; }
const groups = [];
let groupPre = null;
let isKanaPre = null;
for (const c of term) {
const isKana = isCodePointKana(c.codePointAt(0));
if (isKana === isKanaPre) { groupPre.text += c; }
else {
groupPre = {isKana, text: c, textNormalized: null};
groups.push(groupPre);
isKanaPre = isKana;
}
}
for (const group of groups) {
if (group.isKana) { group.textNormalized = convertKatakanaToHiragana(group.text); }
}
const segments = segmentizeFurigana(reading, convertKatakanaToHiragana(reading), groups, 0);
return segments !== null ? segments : [createFuriganaSegment(term, reading)];
}
function getStemLength(text1, text2) {
const minLength = Math.min(text1.length, text2.length);
if (minLength === 0) { return 0; }
let i = 0;
while (true) {
const char1 = text1.codePointAt(i);
const char2 = text2.codePointAt(i);
if (char1 !== char2) { break; }
const charLength = String.fromCodePoint(char1).length;
i += charLength;
if (i >= minLength) {
if (i > minLength) { i -= charLength; }
break;
}
}
return i;
}
function distributeFuriganaInflected(term, reading, source) {
const termNormalized = convertKatakanaToHiragana(term);
const readingNormalized = convertKatakanaToHiragana(reading);
const sourceNormalized = convertKatakanaToHiragana(source);
let mainText = term;
let stemLength = getStemLength(termNormalized, sourceNormalized);
const readingStemLength = getStemLength(readingNormalized, sourceNormalized);
if (readingStemLength > 0 && readingStemLength >= stemLength) {
mainText = reading;
stemLength = readingStemLength;
reading = source.substring(0, stemLength) + reading.substring(stemLength);
}
const segments = [];
if (stemLength > 0) {
mainText = source.substring(0, stemLength) + mainText.substring(stemLength);
const segments2 = distributeFurigana(mainText, reading);
let consumed = 0;
for (const segment of segments2) {
const start = consumed;
consumed += segment.text.length;
if (consumed < stemLength) { segments.push(segment); }
else if (consumed === stemLength) { segments.push(segment); break; }
else {
if (start < stemLength) { segments.push(createFuriganaSegment(mainText.substring(start, stemLength), '')); }
break;
}
}
}
if (stemLength < source.length) {
const remainder = source.substring(stemLength);
const last = segments[segments.length - 1];
if (last && last.reading.length === 0) { last.text += remainder; }
else { segments.push(createFuriganaSegment(remainder, '')); }
}
return segments;
}
function parsePositiveFrequencyNumber(value) {
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
return Math.max(1, Math.floor(value));
}
if (typeof value === 'string') {
const numericMatch = value.trim().match(/[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?/)?.[0];
if (!numericMatch) { return null; }
const parsed = Number.parseFloat(numericMatch);
if (!Number.isFinite(parsed) || parsed <= 0) { return null; }
return Math.max(1, Math.floor(parsed));
}
if (Array.isArray(value)) {
for (const item of value) {
const parsed = parsePositiveFrequencyNumber(item);
if (parsed !== null) { return parsed; }
}
}
return null;
}
function parseDisplayFrequencyNumber(value) {
if (typeof value === 'string') {
const leadingDigits = value.trim().match(/^\d+/)?.[0];
if (!leadingDigits) { return null; }
const parsed = Number.parseInt(leadingDigits, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
}
return parsePositiveFrequencyNumber(value);
}
function getFrequencyDictionaryName(frequency) {
const candidates = [
frequency?.dictionary,
frequency?.dictionaryName,
frequency?.name,
frequency?.title,
frequency?.dictionaryTitle,
frequency?.dictionaryAlias
];
for (const candidate of candidates) {
if (typeof candidate === 'string' && candidate.trim().length > 0) {
return candidate.trim();
}
}
return null;
}
function getBestFrequencyRank(dictionaryEntry, headwordIndex, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
let best = null;
const headwordCount = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords.length : 0;
for (const frequency of dictionaryEntry?.frequencies || []) {
if (!frequency || typeof frequency !== 'object') { continue; }
const frequencyHeadwordIndex = frequency.headwordIndex;
if (typeof frequencyHeadwordIndex === 'number') {
if (frequencyHeadwordIndex !== headwordIndex) { continue; }
} else if (headwordCount > 1) {
continue;
}
const dictionary = getFrequencyDictionaryName(frequency);
if (!dictionary) { continue; }
if (dictionaryFrequencyModeByName[dictionary] === 'occurrence-based') { continue; }
const rank =
parseDisplayFrequencyNumber(frequency.displayValue) ??
parsePositiveFrequencyNumber(frequency.frequency);
if (rank === null) { continue; }
const priorityRaw = dictionaryPriorityByName[dictionary];
const fallbackPriority =
typeof frequency.dictionaryIndex === 'number' && Number.isFinite(frequency.dictionaryIndex)
? Math.max(0, Math.floor(frequency.dictionaryIndex))
: Number.MAX_SAFE_INTEGER;
const priority =
typeof priorityRaw === 'number' && Number.isFinite(priorityRaw)
? Math.max(0, Math.floor(priorityRaw))
: fallbackPriority;
if (best === null || priority < best.priority || (priority === best.priority && rank < best.rank)) {
best = { priority, rank };
}
}
return best?.rank ?? null;
}
function hasExactSource(headword, token, requirePrimary) {
for (const src of headword.sources || []) {
if (src.originalText !== token) { continue; }
if (requirePrimary && !src.isPrimary) { continue; }
if (src.matchType !== 'exact') { continue; }
return true;
}
return false;
}
function collectExactHeadwordMatches(dictionaryEntries, token, requirePrimary) {
const matches = [];
for (const dictionaryEntry of dictionaryEntries || []) {
const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
const headword = headwords[headwordIndex];
if (!hasExactSource(headword, token, requirePrimary)) { continue; }
matches.push({ dictionaryEntry, headword, headwordIndex });
}
}
return matches;
}
function sameHeadword(match, preferredMatch) {
if (!match || !preferredMatch) {
return false;
}
if (match.headword?.term !== preferredMatch.headword?.term) {
return false;
}
const matchReading = typeof match.headword?.reading === 'string' ? match.headword.reading : '';
const preferredReading =
typeof preferredMatch.headword?.reading === 'string' ? preferredMatch.headword.reading : '';
if (!matchReading || !preferredReading) {
return true;
}
return matchReading === preferredReading;
}
function getBestFrequencyRankForMatches(matches, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
let best = null;
for (const match of matches) {
const rank = getBestFrequencyRank(
match.dictionaryEntry,
match.headwordIndex,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
);
if (rank === null) { continue; }
if (best === null || rank < best) {
best = rank;
}
}
return best;
}
function normalizeWordClasses(headword) {
if (!Array.isArray(headword?.wordClasses)) { return undefined; }
const classes = headword.wordClasses.filter((wordClass) => typeof wordClass === "string" && wordClass.trim().length > 0);
return classes.length > 0 ? classes : undefined;
}
function appendDictionaryNames(target, value) {
if (!value || typeof value !== 'object') {
return;
}
const candidates = [
value.dictionary,
value.dictionaryName,
value.name,
value.title,
value.dictionaryTitle,
value.dictionaryAlias
];
for (const candidate of candidates) {
if (typeof candidate === 'string' && candidate.trim().length > 0) {
target.push(candidate.trim());
}
}
}
function getDictionaryEntryNames(entry) {
const names = [];
appendDictionaryNames(names, entry);
for (const definition of entry?.definitions || []) {
appendDictionaryNames(names, definition);
}
for (const frequency of entry?.frequencies || []) {
appendDictionaryNames(names, frequency);
}
for (const pronunciation of entry?.pronunciations || []) {
appendDictionaryNames(names, pronunciation);
}
return names;
}
function isNameDictionaryEntry(entry) {
if (!includeNameMatchMetadata || !entry || typeof entry !== 'object') {
return false;
}
return getDictionaryEntryNames(entry).some((name) => name.startsWith(${JSON.stringify(CHARACTER_DICTIONARY_TITLE_PREFIX)}));
}
function parseSubMinerMediaIdFromString(value) {
const imageMatch = value.match(/\bimg\/m(\d+)-/i);
if (imageMatch) {
const parsed = Number.parseInt(imageMatch[1], 10);
if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
}
const titleMatch = value.match(/${CHARACTER_DICTIONARY_TITLE_PREFIX}[^\d]*(?:AniList\s*)?(\d+)/i);
if (titleMatch) {
const parsed = Number.parseInt(titleMatch[1], 10);
if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
}
return null;
}
function parseSubMinerMediaIdCandidate(value) {
if (typeof value === 'number' && Number.isSafeInteger(value) && value > 0) {
return value;
}
if (typeof value === 'string' && /^\d+$/.test(value.trim())) {
const parsed = Number.parseInt(value.trim(), 10);
if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
}
return null;
}
function collectSubMinerMediaIds(value, target) {
if (typeof value === 'string') {
const parsed = parseSubMinerMediaIdFromString(value);
if (parsed !== null) { target.add(parsed); }
return;
}
if (!value || typeof value !== 'object') {
return;
}
if (Array.isArray(value)) {
for (const item of value) { collectSubMinerMediaIds(item, target); }
return;
}
const mediaIdCandidates = [
value.subminerMediaId,
value.subMinerMediaId,
value.characterDictionaryMediaId,
value.data?.subminerMediaId,
value.data?.subMinerMediaId,
value.data?.characterDictionaryMediaId
];
for (const candidate of mediaIdCandidates) {
const parsed = parseSubMinerMediaIdCandidate(candidate);
if (parsed !== null) { target.add(parsed); }
}
for (const child of Object.values(value)) {
collectSubMinerMediaIds(child, target);
}
}
function getSubMinerMediaIds(entry) {
const mediaIds = new Set();
collectSubMinerMediaIds(entry, mediaIds);
return mediaIds;
}
function isCurrentMediaNameDictionaryEntry(entry) {
if (!isNameDictionaryEntry(entry)) {
return false;
}
if (currentCharacterDictionaryMediaId === null) {
return true;
}
const mediaIds = getSubMinerMediaIds(entry);
return mediaIds.size === 0 || mediaIds.has(currentCharacterDictionaryMediaId);
}
function findLongestNameMatch(dictionaryEntries, textWindow) {
let best = null;
for (const dictionaryEntry of dictionaryEntries || []) {
if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; }
const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
const headword = headwords[headwordIndex];
for (const src of headword?.sources || []) {
if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
const originalText = typeof src.originalText === 'string' ? src.originalText : '';
if (!originalText || !textWindow.startsWith(originalText)) { continue; }
if (best === null || originalText.length > best.sourceLength) {
best = { dictionaryEntry, headword, headwordIndex, sourceLength: originalText.length };
}
}
}
}
return best;
}
function findLongestGenericMatchLength(dictionaryEntries, textWindow) {
let best = 0;
for (const dictionaryEntry of dictionaryEntries || []) {
if (isNameDictionaryEntry(dictionaryEntry)) { continue; }
const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
for (const headword of headwords) {
for (const src of headword?.sources || []) {
if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
const originalText = typeof src.originalText === 'string' ? src.originalText : '';
if (!originalText || !textWindow.startsWith(originalText)) { continue; }
if (originalText.length > best) { best = originalText.length; }
}
}
}
return best;
}
function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
const currentMediaDictionaryEntries =
currentCharacterDictionaryMediaId === null
? (dictionaryEntries || [])
: (dictionaryEntries || []).filter((entry) => {
if (!isNameDictionaryEntry(entry)) { return true; }
return isCurrentMediaNameDictionaryEntry(entry);
});
const exactPrimaryMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, true);
let matchedNameDictionary = false;
if (includeNameMatchMetadata) {
for (const dictionaryEntry of currentMediaDictionaryEntries || []) {
if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; }
for (const match of exactPrimaryMatches) {
if (match.dictionaryEntry !== dictionaryEntry) { continue; }
matchedNameDictionary = true;
break;
}
if (matchedNameDictionary) { break; }
}
}
const preferredMatch = exactPrimaryMatches[0];
if (preferredMatch) {
const exactFrequencyMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, false)
.filter((match) => sameHeadword(match, preferredMatch));
return {
term: preferredMatch.headword.term,
reading: preferredMatch.headword.reading,
wordClasses: normalizeWordClasses(preferredMatch.headword),
isNameMatch:
matchedNameDictionary || isCurrentMediaNameDictionaryEntry(preferredMatch.dictionaryEntry),
frequencyRank: getBestFrequencyRankForMatches(
exactFrequencyMatches.length > 0 ? exactFrequencyMatches : exactPrimaryMatches,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
)
};
}
return null;
}
`;
async function installYomitanScanRuntime(parserWindow: BrowserWindow): Promise<void> {
await parserWindow.webContents.executeJavaScript(YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT, true);
// A fresh runtime has no candidate list; force the next scan to reinstall it.
yomitanScanNameCandidateKeyByWindow.delete(parserWindow);
}
function buildYomitanScanningScript(
text: string,
profileIndex: number,
scanLength: number,
includeNameMatchMetadata: boolean,
greedyNameScanEnabled: boolean,
currentCharacterDictionaryMediaId: number | null,
dictionaryPriorityByName: Record<string, number>,
dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>,
): string {
return `
(async () => {
const invoke = (action, params) =>
new Promise((resolve, reject) => {
chrome.runtime.sendMessage({ action, params }, (response) => {
if (chrome.runtime.lastError) {
reject(new Error(chrome.runtime.lastError.message));
return;
}
if (!response || typeof response !== "object") {
reject(new Error("Invalid response from Yomitan backend"));
return;
}
if (response.error) {
reject(new Error(response.error.message || "Yomitan backend error"));
return;
}
resolve(response.result);
});
});
${YOMITAN_SCANNING_HELPERS}
const includeNameMatchMetadata = ${includeNameMatchMetadata ? 'true' : 'false'};
const greedyNameScanEnabled = ${greedyNameScanEnabled ? 'true' : 'false'};
const currentCharacterDictionaryMediaId = ${
currentCharacterDictionaryMediaId !== null
? String(currentCharacterDictionaryMediaId)
: 'null'
};
const dictionaryPriorityByName = ${JSON.stringify(dictionaryPriorityByName)};
const dictionaryFrequencyModeByName = ${JSON.stringify(dictionaryFrequencyModeByName)};
const text = ${JSON.stringify(text)};
const details = {matchType: "exact", deinflect: true};
const tokens = [];
const termsFindCache = new Map();
async function termsFindAt(position, windowLength) {
const cacheKey = position + ":" + windowLength;
const cached = termsFindCache.get(cacheKey);
if (cached) { return cached; }
const substring = text.substring(position, position + windowLength);
const result = await invoke("termsFind", { text: substring, details, optionsContext: { index: ${profileIndex} } });
termsFindCache.set(cacheKey, result);
return result;
}
function buildScanToken(position, source, preferredHeadword) {
const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
const tokenPayload = {
surface: segments.map((segment) => segment.text).join("") || source,
reading: segments.map(getSegmentReadingContribution).join(""),
headword: preferredHeadword.term,
headwordReading: reading || undefined,
startPos: position,
endPos: position + source.length,
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
frequencyRank:
typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
: undefined,
};
if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
tokenPayload.wordClasses = preferredHeadword.wordClasses;
}
return tokenPayload;
}
async function findTokenAt(position, windowLength) {
const codePoint = text.codePointAt(position);
const character = String.fromCodePoint(codePoint);
const result = await termsFindAt(position, windowLength);
const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
const originalTextLength = typeof result?.originalTextLength === "number" ? result.originalTextLength : 0;
if (dictionaryEntries.length === 0 || originalTextLength <= 0 || (originalTextLength === character.length && !isCodePointJapanese(codePoint))) {
return { token: null, matchedLength: 0 };
}
const source = text.substring(position, position + originalTextLength);
const preferredHeadword = getPreferredHeadword(
dictionaryEntries,
source,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
);
if (!preferredHeadword || typeof preferredHeadword.term !== "string") {
return { token: null, matchedLength: originalTextLength };
}
return { token: buildScanToken(position, source, preferredHeadword), matchedLength: originalTextLength };
}
// Greedy name pre-pass: character-name matches claim their spans before
// the left-to-right walk, so a longer generic match starting earlier
// (e.g. とヨー → 渡洋) cannot swallow the start of a name (ヨータ).
const nameTokens = [];
if (greedyNameScanEnabled) {
let namePos = 0;
while (namePos < text.length) {
const codePoint = text.codePointAt(namePos);
if (!isCodePointJapanese(codePoint)) {
namePos += String.fromCodePoint(codePoint).length;
continue;
}
const result = await termsFindAt(namePos, ${scanLength});
const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
const textWindow = text.substring(namePos, namePos + ${scanLength});
const nameMatch = findLongestNameMatch(dictionaryEntries, textWindow);
// A name only claims its span when no strictly longer generic word
// starts at the same position (a character named 空 must not split
// 空気). Ties go to the name. Generic matches that start earlier and
// overlap the name are still blocked by the reservation.
if (
!nameMatch ||
findLongestGenericMatchLength(dictionaryEntries, textWindow) > nameMatch.sourceLength
) {
namePos += String.fromCodePoint(codePoint).length;
continue;
}
const source = text.substring(namePos, namePos + nameMatch.sourceLength);
nameTokens.push(buildScanToken(namePos, source, {
term: nameMatch.headword.term,
reading: nameMatch.headword.reading,
wordClasses: normalizeWordClasses(nameMatch.headword),
isNameMatch: true,
frequencyRank: getBestFrequencyRank(
nameMatch.dictionaryEntry,
nameMatch.headwordIndex,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
)
}));
namePos += nameMatch.sourceLength;
}
}
let i = 0;
let nameIndex = 0;
while (i < text.length) {
while (nameIndex < nameTokens.length && nameTokens[nameIndex].startPos < i) { nameIndex += 1; }
const nextNameToken = nameIndex < nameTokens.length ? nameTokens[nameIndex] : null;
if (nextNameToken && nextNameToken.startPos === i) {
tokens.push(nextNameToken);
i = nextNameToken.endPos;
nameIndex += 1;
continue;
}
// Cap the window at the next reserved name span so a generic match
// cannot consume into it.
const windowLength = nextNameToken ? Math.min(${scanLength}, nextNameToken.startPos - i) : ${scanLength};
let attempt = await findTokenAt(i, windowLength);
// Yomitan text normalization can consume characters (whitespace,
// punctuation) beyond the matched term, leaving no headword whose
// source equals the consumed text. Retry with shorter windows so a
// valid prefix term (e.g. a character name before a paren) still
// tokenizes instead of the position being skipped.
let retryLength = Math.min(attempt.matchedLength, windowLength) - 1;
while (!attempt.token && retryLength >= 1) {
const retry = await findTokenAt(i, retryLength);
if (retry.token) {
attempt = retry;
break;
}
retryLength = Math.min(retryLength - 1, retry.matchedLength - 1);
}
if (attempt.token) {
tokens.push(attempt.token);
i += attempt.matchedLength;
continue;
}
i += String.fromCodePoint(text.codePointAt(i)).length;
}
return tokens;
})();
`;
// Key of the character-name candidate list currently installed in each parser
// window, so an unchanged list costs nothing per line.
const yomitanScanNameCandidateKeyByWindow = new WeakMap<BrowserWindow, string>();
async function ensureYomitanScanNameCandidates(
parserWindow: BrowserWindow,
nameCandidates: { key: string; forms: string[] } | null,
logger: LoggerLike,
): Promise<void> {
const installedKey = yomitanScanNameCandidateKeyByWindow.get(parserWindow);
const nextKey = nameCandidates?.key ?? '';
if (installedKey === nextKey) {
return;
}
try {
await parserWindow.webContents.executeJavaScript(
buildYomitanScanNameCandidatesScript(nameCandidates),
true,
);
yomitanScanNameCandidateKeyByWindow.set(parserWindow, nextKey);
} catch (err) {
// The scan falls back to checking every position when the list is absent,
// so a failed install costs speed, never a missed name.
logger.warn?.(
'Failed to install Yomitan character-name scan candidates:',
(err as Error).message,
);
yomitanScanNameCandidateKeyByWindow.delete(parserWindow);
}
}
export async function requestYomitanParseResults(
@@ -1635,6 +963,20 @@ export async function requestYomitanParseResults(
}
}
// parseText fallback for when the in-window scanner cannot run (script eval
// failure, unexpected payload). The scanner walk is the primary tokenizer and
// emits its own filler runs, so this extra full parse only happens on errors.
async function requestYomitanParseFallbackTokens(
text: string,
deps: YomitanParserRuntimeDeps,
logger: LoggerLike,
): Promise<YomitanScanToken[] | null> {
const parseResults = await requestYomitanParseResults(text, deps, logger);
const selectedTokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
const parseScanTokens = selectedTokens?.map(toYomitanScanToken) ?? null;
return parseScanTokens && parseScanTokens.length > 0 ? parseScanTokens : null;
}
export async function requestYomitanScanTokens(
text: string,
deps: YomitanParserRuntimeDeps,
@@ -1642,6 +984,7 @@ export async function requestYomitanScanTokens(
options?: {
includeNameMatchMetadata?: boolean;
currentCharacterDictionaryMediaId?: number | null;
nameCandidates?: { key: string; forms: string[] } | null;
},
): Promise<YomitanScanToken[] | null> {
const yomitanExt = deps.getYomitanExt();
@@ -1655,10 +998,6 @@ export async function requestYomitanScanTokens(
return null;
}
const parseResults = await requestYomitanParseResults(text, deps, logger);
const selectedParseTokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
const parseScanTokens = selectedParseTokens?.map(toYomitanScanToken) ?? null;
const metadata = await requestYomitanProfileMetadata(parserWindow, logger);
const profileIndex = metadata?.profileIndex ?? 0;
const scanLength = metadata?.scanLength ?? DEFAULT_YOMITAN_SCAN_LENGTH;
@@ -1669,44 +1008,63 @@ export async function requestYomitanScanTokens(
name.startsWith(CHARACTER_DICTIONARY_TITLE_PREFIX),
);
// Candidate name forms let the in-page pre-pass skip positions where no
// character name can start. Installed only when it changes (per media), so
// the per-line call stays a single tiny script.
const nameCandidates = greedyNameScanEnabled ? (options?.nameCandidates ?? null) : null;
await ensureYomitanScanNameCandidates(parserWindow, nameCandidates, logger);
const callScript = buildYomitanScanCallScript({
text,
profileIndex,
scanLength,
includeNameMatchMetadata,
greedyNameScanEnabled,
currentCharacterDictionaryMediaId:
typeof options?.currentCharacterDictionaryMediaId === 'number' &&
Number.isFinite(options.currentCharacterDictionaryMediaId) &&
options.currentCharacterDictionaryMediaId > 0
? Math.floor(options.currentCharacterDictionaryMediaId)
: null,
dictionaryPriorityByName: metadata?.dictionaryPriorityByName ?? {},
dictionaryFrequencyModeByName: metadata?.dictionaryFrequencyModeByName ?? {},
cacheEpoch: getYomitanScanCacheEpoch(parserWindow),
nameCandidateKey: nameCandidates?.key ?? null,
});
try {
const rawResult = await parserWindow.webContents.executeJavaScript(
buildYomitanScanningScript(
text,
profileIndex,
scanLength,
includeNameMatchMetadata,
greedyNameScanEnabled,
typeof options?.currentCharacterDictionaryMediaId === 'number' &&
Number.isFinite(options.currentCharacterDictionaryMediaId) &&
options.currentCharacterDictionaryMediaId > 0
? Math.floor(options.currentCharacterDictionaryMediaId)
: null,
metadata?.dictionaryPriorityByName ?? {},
metadata?.dictionaryFrequencyModeByName ?? {},
),
true,
);
if (isScanTokenArray(rawResult)) {
if (parseScanTokens && parseScanTokens.length > 0) {
return mergeScannerTokensIntoParseTokens(parseScanTokens, rawResult);
let rawResult = await parserWindow.webContents.executeJavaScript(callScript, true);
if (rawResult === YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL) {
// First request for this window, or the page reloaded and dropped the
// installed runtime: install and retry once. The candidate list lives in
// the same page state, so it has to be reinstalled alongside it.
await installYomitanScanRuntime(parserWindow);
await ensureYomitanScanNameCandidates(parserWindow, nameCandidates, logger);
rawResult = await parserWindow.webContents.executeJavaScript(callScript, true);
}
// The scanner reports a line where a position ran out of shrinking-window
// retries: it stopped short of windows an uncapped ladder would have tried,
// so a real term may be sitting in an unparsed run. One parseText for the
// line is the bounded way to get the exhaustive answer back (this is the
// parse the scanner replaced, and it only runs for these rare lines).
if (isObject(rawResult) && rawResult.retryBudgetExhausted === true) {
logger.info?.('Yomitan scanner exhausted its retry budget; parsing the line as a fallback.');
const fallbackTokens = await requestYomitanParseFallbackTokens(text, deps, logger);
if (fallbackTokens) {
return fallbackTokens;
}
return rawResult;
rawResult = rawResult.tokens;
}
if (Array.isArray(rawResult)) {
const selectedTokens = selectYomitanParseTokens(rawResult, () => false, 'headword');
return selectedTokens?.map(toYomitanScanToken) ?? null;
if (isScanTokenArray(rawResult)) {
// Filler-only results carry no dictionary match; keep the historical
// contract of returning null so callers fall back to raw text.
return rawResult.some((token) => token.isUnparsedRun !== true) ? rawResult : null;
}
if (parseScanTokens && parseScanTokens.length > 0) {
return parseScanTokens;
}
return null;
logger.error('Yomitan scanner returned an unexpected payload; using parseText fallback.');
return await requestYomitanParseFallbackTokens(text, deps, logger);
} catch (err) {
if (parseScanTokens && parseScanTokens.length > 0) {
return parseScanTokens;
}
logger.error('Yomitan scanner request failed:', (err as Error).message);
return null;
return await requestYomitanParseFallbackTokens(text, deps, logger);
}
}
@@ -0,0 +1,563 @@
// In-page Yomitan scan runtime: the scan walk that gets installed once per
// parser window as globalThis.__subminerYomitanScan, plus the tiny per-line
// call script. Kept separate from the host runtime module so the injected
// script text (which is data, not executed here) does not dominate that file;
// the helper bundle it embeds is composed in yomitan-scanning-helpers-script.ts
// from the yomitan-*-script.ts fragments.
import { YOMITAN_SCANNING_HELPERS } from './yomitan-scanning-helpers-script';
export { CHARACTER_DICTIONARY_TITLE_PREFIX } from './yomitan-scanning-helpers-script';
export type YomitanFrequencyMode = 'occurrence-based' | 'rank-based';
// Bump whenever the install script below changes so already-loaded parser
// windows re-install the new scan runtime instead of running the stale one.
export const YOMITAN_SCAN_RUNTIME_VERSION = 12;
export const YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL = '__subminer-yomitan-scan-runtime-missing__';
export interface YomitanScanRequestParams {
text: string;
profileIndex: number;
scanLength: number;
includeNameMatchMetadata: boolean;
greedyNameScanEnabled: boolean;
currentCharacterDictionaryMediaId: number | null;
dictionaryPriorityByName: Record<string, number>;
dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>;
cacheEpoch: number;
/**
* Key of the character-name candidate list installed for the current media,
* or null to scan every Japanese position (see the pre-pass prefilter).
*/
nameCandidateKey: string | null;
}
// Installed once per parser window (and re-installed after in-page reloads):
// keeps V8 from re-parsing the helper bundle on every subtitle line, and hosts
// the cross-line termsFind cache. Each subtitle line then only evaluates a tiny
// call into globalThis.__subminerYomitanScan.
export const YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT = String.raw`
(() => {
if (globalThis.__subminerYomitanScanVersion === ${YOMITAN_SCAN_RUNTIME_VERSION}) {
return true;
}
const invoke = (action, params) =>
new Promise((resolve, reject) => {
chrome.runtime.sendMessage({ action, params }, (response) => {
if (chrome.runtime.lastError) {
reject(new Error(chrome.runtime.lastError.message));
return;
}
if (!response || typeof response !== "object") {
reject(new Error("Invalid response from Yomitan backend"));
return;
}
if (response.error) {
reject(new Error(response.error.message || "Yomitan backend error"));
return;
}
resolve(response.result);
});
});
// Cross-line termsFind LRU keyed by profile + substring: subtitle lines
// repeat particles and inflections constantly, so most lookups hit here.
// Entries hold in-flight promises so concurrent identical lookups dedupe.
const termsFindCache = new Map();
// Two bounds. The key count keeps the map itself small; the accumulated
// dictionary-entry count stands in for retained bytes, because a single
// lookup over a common prefix can hold hundreds of entries with their full
// glossaries and a key-count cap alone would not bound that.
const TERMS_FIND_CACHE_LIMIT = 2000;
const TERMS_FIND_CACHE_DICTIONARY_ENTRY_LIMIT = 20000;
let termsFindCacheDictionaryEntries = 0;
let termsFindCacheEpoch = -1;
function dropCachedTermsFind(cacheKey, entry) {
if (termsFindCache.get(cacheKey) !== entry) { return; }
termsFindCache.delete(cacheKey);
termsFindCacheDictionaryEntries -= entry.dictionaryEntryCount;
}
// Runs on insert and again once a lookup resolves: an entry is only worth
// its estimated weight of 1 until then, so a single oversized response
// would otherwise sit in the cache forever, over the limit and reused.
function evictOverflowingTermsFindEntries() {
while (
termsFindCache.size > TERMS_FIND_CACHE_LIMIT ||
termsFindCacheDictionaryEntries > TERMS_FIND_CACHE_DICTIONARY_ENTRY_LIMIT
) {
const oldest = termsFindCache.entries().next().value;
if (oldest === undefined) { break; }
dropCachedTermsFind(oldest[0], oldest[1]);
}
}
// Classification of a dictionary entry (which dictionaries it came from,
// which media ids it mentions) depends only on the entry object, so it is
// memoized for as long as that object lives. Entries are shared with the
// termsFind cache above, which is what makes this worth keeping: the same
// objects come back for every repeated lookup, on every line.
const dictionaryEntryNamesCache = new WeakMap();
const subMinerMediaIdsCache = new WeakMap();
const EMPTY_MEDIA_ID_SET = new Set();
// Only blind ladder steps are capped (see the retry loop): those are the
// ones that would otherwise degrade into O(scanLength) lookups at a single
// position. Steps the backend guides by reporting a shorter consumed length
// stay uncapped, so a valid prefix term is still found on lines where
// normalization eats a long tail.
const MAX_BLIND_SHRINKING_WINDOW_RETRIES = 4;
// Character-name candidate forms for the current media, installed
// separately from the per-line scan call so the per-line script stays tiny.
// Stored raw here; the normalized lookup index is built inside the scan,
// where the kana-normalization helper is in scope, and reused by key.
let rawNameCandidates = null;
let nameCandidateIndex = null;
globalThis.__subminerYomitanScanSetNameCandidates = (key, forms) => {
if (!key || !Array.isArray(forms) || forms.length === 0) {
rawNameCandidates = null;
nameCandidateIndex = null;
return false;
}
rawNameCandidates = { key, forms };
nameCandidateIndex = null;
return true;
};
globalThis.__subminerYomitanScanVersion = ${YOMITAN_SCAN_RUNTIME_VERSION};
globalThis.__subminerYomitanScan = async (scanParams) => {
const {
text,
profileIndex,
scanLength,
includeNameMatchMetadata,
greedyNameScanEnabled,
currentCharacterDictionaryMediaId,
dictionaryPriorityByName,
dictionaryFrequencyModeByName,
cacheEpoch,
nameCandidateKey
} = scanParams;
if (cacheEpoch !== termsFindCacheEpoch) {
termsFindCache.clear();
termsFindCacheDictionaryEntries = 0;
termsFindCacheEpoch = cacheEpoch;
}
${YOMITAN_SCANNING_HELPERS}
const CAPTION_OPENING_BRACKETS = new Set(["(", "", "[", "", "{", "", "「", "『", "【", "〈", "《", "≪", "", "<"]);
function shouldEmitUnparsedRunAsToken(runText) {
if (!/[\p{L}\p{N}]/u.test(runText)) { return false; }
const firstChar = Array.from(runText.trim())[0];
return firstChar !== undefined && !CAPTION_OPENING_BRACKETS.has(firstChar);
}
function isLookupWorthyCodePoint(codePoint) {
if (isCodePointJapanese(codePoint)) { return true; }
return /[\p{L}\p{N}]/u.test(String.fromCodePoint(codePoint));
}
function isKanaOnlyRunText(runText) {
const chars = Array.from(runText);
return chars.length > 0 && chars.every((char) => isCodePointKana(char.codePointAt(0)));
}
const details = {matchType: "exact", deinflect: true};
const tokens = [];
async function termsFindAt(position, windowLength) {
const substring = text.substring(position, position + windowLength);
const cacheKey = profileIndex + "\u0000" + substring;
const cached = termsFindCache.get(cacheKey);
if (cached !== undefined) {
termsFindCache.delete(cacheKey);
termsFindCache.set(cacheKey, cached);
return await cached.promise;
}
// An in-flight lookup counts as one entry until it resolves; the real
// weight replaces that estimate once the result is known.
const entry = { promise: null, dictionaryEntryCount: 1 };
entry.promise = invoke("termsFind", { text: substring, details, optionsContext: { index: profileIndex } })
.then((result) => {
const resolvedCount =
1 + (Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries.length : 0);
const isCached = termsFindCache.get(cacheKey) === entry;
if (isCached) {
termsFindCacheDictionaryEntries += resolvedCount - entry.dictionaryEntryCount;
}
entry.dictionaryEntryCount = resolvedCount;
// The real weight can push the cache over its budget, and a single
// response can exceed it on its own, so re-check here.
if (isCached) { evictOverflowingTermsFindEntries(); }
return result;
});
termsFindCache.set(cacheKey, entry);
termsFindCacheDictionaryEntries += entry.dictionaryEntryCount;
evictOverflowingTermsFindEntries();
try {
return await entry.promise;
} catch (error) {
dropCachedTermsFind(cacheKey, entry);
throw error;
}
}
// Text the walk skips accumulates into unparsed runs, mirroring the
// filler chunks the parseText segmentation used to provide: runs stay
// hoverable (flagged isUnparsedRun) unless they are punctuation-only or
// caption-style asides, and kana continuations of a longer headword
// extend the previous token instead.
function flushUnparsedRun(runStart, runEnd) {
if (runStart === null || runEnd <= runStart) { return; }
const runText = text.substring(runStart, runEnd);
const previousToken = tokens[tokens.length - 1];
if (
previousToken &&
previousToken.endPos === runStart &&
isKanaOnlyRunText(runText) &&
typeof previousToken.headword === "string" &&
previousToken.headword.length > previousToken.surface.length &&
previousToken.headword.startsWith(previousToken.surface + runText)
) {
previousToken.surface += runText;
// The run is kana-only, so its reading is itself: append it or the
// reading stops covering the surface, which disables the known-word
// reading fallback (isCompleteReadingForSurface) downstream.
previousToken.reading += runText;
// The run is kana-only, so its reading is itself: append it or the
// reading stops covering the surface, which disables the known-word
// reading fallback (isCompleteReadingForSurface) downstream.
previousToken.endPos = runEnd;
return;
}
if (!shouldEmitUnparsedRunAsToken(runText)) { return; }
tokens.push({
surface: runText,
reading: "",
headword: runText,
startPos: runStart,
endPos: runEnd,
isUnparsedRun: true
});
}
function buildScanToken(position, source, preferredHeadword) {
const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
const tokenPayload = {
surface: segments.map((segment) => segment.text).join("") || source,
reading: segments.map(getSegmentReadingContribution).join(""),
headword: preferredHeadword.term,
headwordReading: reading || undefined,
startPos: position,
endPos: position + source.length,
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
frequencyRank:
typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
: undefined,
};
if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
tokenPayload.wordClasses = preferredHeadword.wordClasses;
}
return tokenPayload;
}
// findTokenAt plus the shrinking-window ladder below it: Yomitan text
// normalization can consume characters (whitespace, punctuation) beyond
// the matched term, leaving no headword whose source equals the consumed
// text. Retry with shorter windows so a valid prefix term (e.g. a
// character name before a paren) still tokenizes instead of the position
// being skipped.
// Every window at or above the consumed length repeats the same result,
// so the next informative window sits just below it. A lookup that
// consumed its whole window reports nothing to aim at, and the step down
// from it is a blind guess: only those are budgeted.
// The window can run past the end of the line, so blindness is judged
// against the text the lookup actually saw.
// Set when a position stopped short of windows an uncapped ladder would
// still have tried; the line then escalates to parseText at the end.
let blindRetryBudgetExhausted = false;
async function resolveTokenAt(position, windowLength) {
let attempt = await findTokenAt(position, windowLength);
const scannedLength = Math.min(windowLength, text.length - position);
let retryLength = Math.min(attempt.matchedLength, scannedLength) - 1;
let stepIsBlind = attempt.matchedLength >= scannedLength;
let blindRetriesRemaining = MAX_BLIND_SHRINKING_WINDOW_RETRIES;
while (!attempt.token && retryLength >= 1) {
if (stepIsBlind) {
if (blindRetriesRemaining <= 0) {
blindRetryBudgetExhausted = true;
break;
}
blindRetriesRemaining -= 1;
}
const retry = await findTokenAt(position, retryLength);
if (retry.token) { return retry; }
const guidedLength = retry.matchedLength - 1;
stepIsBlind = guidedLength >= retryLength - 1;
retryLength = Math.min(retryLength - 1, guidedLength);
}
return attempt;
}
async function findTokenAt(position, windowLength) {
const codePoint = text.codePointAt(position);
const character = String.fromCodePoint(codePoint);
const result = await termsFindAt(position, windowLength);
const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
const originalTextLength = typeof result?.originalTextLength === "number" ? result.originalTextLength : 0;
if (dictionaryEntries.length === 0 || originalTextLength <= 0 || (originalTextLength === character.length && !isCodePointJapanese(codePoint))) {
return { token: null, matchedLength: 0 };
}
const source = text.substring(position, position + originalTextLength);
const preferredHeadword = getPreferredHeadword(
dictionaryEntries,
source,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
);
if (!preferredHeadword || typeof preferredHeadword.term !== "string") {
return { token: null, matchedLength: originalTextLength };
}
return { token: buildScanToken(position, source, preferredHeadword), matchedLength: originalTextLength };
}
// Kana normalization folds halfwidth katakana one code point to one, so an
// unvoiced halfwidth spelling prefix-matches a candidate form like any
// other. What it cannot fold is a voiced pair: カ + ゙ stays two characters
// where the candidate form carries the single が, so the comparison fails
// at that character. That break can sit anywhere inside the name, not
// just at its first character (山ガク starts on a kanji), so the bypass is
// keyed on the region a candidate could cover, not on how it starts.
function isHalfwidthKanaVoicedMarkCodePoint(codePoint) {
return codePoint === 0xff9e || codePoint === 0xff9f;
}
// Build (once per candidate list) a first-character bucket index of the
// normalized name forms, so the pre-pass can reject a position with a
// single map hit instead of a backend round trip.
if (rawNameCandidates && nameCandidateIndex?.key !== rawNameCandidates.key) {
const byFirstChar = new Map();
for (const form of rawNameCandidates.forms) {
const normalized = typeof form === "string" ? convertKatakanaToHiragana(form.trim()) : "";
if (!normalized) { continue; }
const bucket = byFirstChar.get(normalized[0]);
if (bucket) { bucket.push(normalized); } else { byFirstChar.set(normalized[0], [normalized]); }
}
nameCandidateIndex = byFirstChar.size > 0 ? { key: rawNameCandidates.key, byFirstChar } : null;
} else if (!rawNameCandidates) {
nameCandidateIndex = null;
}
// Only meaningful when the installed list matches the media this scan is
// for; otherwise fall back to scanning every position.
const activeNameCandidateIndex =
nameCandidateKey !== null && nameCandidateIndex?.key === nameCandidateKey
? nameCandidateIndex
: null;
const normalizedText = activeNameCandidateIndex ? convertKatakanaToHiragana(text) : "";
// Yomitan collapses emphatic sequences before matching (すっっごーーい →
// すごい), so a stretched name still resolves to its entry. Skipping these
// characters keeps such spellings candidates; the filter only ever grows
// the probe set, so a false positive costs one lookup, never a name.
const EMPHATIC_SKIP_CHARS = new Set(["ぁ", "ぃ", "ぅ", "ぇ", "ぉ", "っ", "ゃ", "ゅ", "ょ", "ー"]);
function matchesCandidateFormAt(form, position) {
let textIndex = position;
for (let formIndex = 0; formIndex < form.length; formIndex += 1) {
while (
textIndex < normalizedText.length &&
normalizedText[textIndex] !== form[formIndex] &&
EMPHATIC_SKIP_CHARS.has(normalizedText[textIndex])
) {
textIndex += 1;
}
if (normalizedText[textIndex] !== form[formIndex]) { return false; }
textIndex += 1;
}
return true;
}
// Where the folding gives up, listed once per line. Matching may skip any
// number of emphatic characters on its way through a form (山ーーーーーーガク),
// so there is no shorter honest bound than the window a name lookup
// covers: scanLength. The list is almost always empty, which is what
// keeps the check below free on ordinary lines.
const halfwidthVoicedMarkPositions = [];
if (activeNameCandidateIndex) {
for (let index = 0; index < text.length; index += 1) {
if (isHalfwidthKanaVoicedMarkCodePoint(text.charCodeAt(index))) {
halfwidthVoicedMarkPositions.push(index);
}
}
}
function hasHalfwidthVoicedMarkInScanWindow(position) {
const end = position + scanLength;
for (const markPosition of halfwidthVoicedMarkPositions) {
if (markPosition >= position && markPosition < end) { return true; }
}
return false;
}
// A name written ガ... folds to か + ゙, so its first character never leads
// to the が bucket the candidate form is filed under. Nothing else can
// find it, so such a position is always worth a probe.
function startsHalfwidthVoicedPair(position, codePoint) {
if (codePoint < 0xff66 || codePoint > 0xff9d) { return false; }
return isHalfwidthKanaVoicedMarkCodePoint(text.charCodeAt(position + 1));
}
function couldNameStartAt(position, codePoint) {
// Nothing starts with a combining voiced mark, whether or not the
// prefilter is active.
if (isHalfwidthKanaVoicedMarkCodePoint(codePoint)) { return false; }
if (!activeNameCandidateIndex) { return true; }
const bucket = activeNameCandidateIndex.byFirstChar.get(normalizedText[position]);
if (!bucket) {
// No candidate begins with this character, and the window search
// below would only ever say yes to positions like this one, so an
// unrelated ガ elsewhere in the line must not drag them in.
return startsHalfwidthVoicedPair(position, codePoint);
}
for (const form of bucket) {
if (matchesCandidateFormAt(form, position)) { return true; }
}
// A candidate does start here but did not match: an unfoldable voiced
// pair anywhere in the window is a reason the comparison could not see
// it (山ガク, 山ーーーーーーガク), so probe rather than drop the name.
return hasHalfwidthVoicedMarkInScanWindow(position);
}
// Greedy name pre-pass: character-name matches claim their spans before
// the left-to-right walk, so a longer generic match starting earlier
// (e.g. とヨー → 渡洋) cannot swallow the start of a name (ヨータ).
const nameTokens = [];
if (greedyNameScanEnabled) {
let namePos = 0;
while (namePos < text.length) {
const codePoint = text.codePointAt(namePos);
if (!isCodePointJapanese(codePoint) || !couldNameStartAt(namePos, codePoint)) {
namePos += String.fromCodePoint(codePoint).length;
continue;
}
const result = await termsFindAt(namePos, scanLength);
const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
const textWindow = text.substring(namePos, namePos + scanLength);
const nameMatch = findLongestNameMatch(dictionaryEntries, textWindow);
// A name only claims its span when no strictly longer generic word
// starts at the same position (a character named 空 must not split
// 空気). Ties go to the name. Generic matches that start earlier and
// overlap the name are still blocked by the reservation.
if (
!nameMatch ||
findLongestGenericMatchLength(dictionaryEntries, textWindow) > nameMatch.sourceLength
) {
namePos += String.fromCodePoint(codePoint).length;
continue;
}
const source = text.substring(namePos, namePos + nameMatch.sourceLength);
nameTokens.push(buildScanToken(namePos, source, {
term: nameMatch.headword.term,
reading: nameMatch.headword.reading,
wordClasses: normalizeWordClasses(nameMatch.headword),
isNameMatch: true,
frequencyRank: getBestFrequencyRank(
nameMatch.dictionaryEntry,
nameMatch.headwordIndex,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
)
}));
namePos += nameMatch.sourceLength;
}
}
// First reserved name span that a match ending at endPos would leave
// half-consumed. Spans the match covers entirely are not returned: those
// lose to the longer word instead of splitting it.
function findSplitNameToken(startIndex, endPos) {
for (let index = startIndex; index < nameTokens.length; index += 1) {
const nameToken = nameTokens[index];
if (nameToken.startPos >= endPos) { return null; }
if (nameToken.endPos > endPos) { return nameToken; }
}
return null;
}
let i = 0;
let nameIndex = 0;
let unparsedRunStart = null;
while (i < text.length) {
while (nameIndex < nameTokens.length && nameTokens[nameIndex].startPos < i) { nameIndex += 1; }
const nextNameToken = nameIndex < nameTokens.length ? nameTokens[nameIndex] : null;
if (nextNameToken && nextNameToken.startPos === i) {
flushUnparsedRun(unparsedRunStart, i);
unparsedRunStart = null;
tokens.push(nextNameToken);
i = nextNameToken.endPos;
nameIndex += 1;
continue;
}
const codePoint = text.codePointAt(i);
// Punctuation and whitespace can never start a token: skip the backend
// round trip entirely. Latin letters and digits stay lookup-worthy
// (terms like Tシャツ start on an ASCII letter).
if (!isLookupWorthyCodePoint(codePoint)) {
if (unparsedRunStart === null) { unparsedRunStart = i; }
i += String.fromCodePoint(codePoint).length;
continue;
}
// A reservation only outranks generic matches that would cut into it.
// Look the position up unrestricted first: a generic word that starts
// earlier and covers the whole name span (写真 over a character named
// 真) is the better reading, so the reservation yields rather than
// splitting the word. Only a match that ends inside a name span gets
// re-run against a window capped at that span.
let attempt = await resolveTokenAt(i, scanLength);
if (attempt.token) {
const splitNameToken = findSplitNameToken(nameIndex, attempt.token.endPos);
if (splitNameToken) {
attempt = await resolveTokenAt(i, splitNameToken.startPos - i);
}
}
if (attempt.token) {
flushUnparsedRun(unparsedRunStart, i);
unparsedRunStart = null;
tokens.push(attempt.token);
i += attempt.matchedLength;
continue;
}
if (unparsedRunStart === null) { unparsedRunStart = i; }
i += String.fromCodePoint(text.codePointAt(i)).length;
}
flushUnparsedRun(unparsedRunStart, text.length);
if (blindRetryBudgetExhausted) {
// A position gave up with shorter windows still worth trying. The walk
// is the only tokenizer now, so stopping there would leave a real term
// as an unparsed run; report it so the host can spend one parseText on
// the line instead of letting the ladder run to O(scanLength) lookups.
return { tokens, retryBudgetExhausted: true };
}
return tokens;
};
return true;
})();
`;
// Installs (or clears) the character-name candidate forms for the current
// media. Runs only when the list changes, not per line. Passing null restores
// the exhaustive every-position pre-pass.
export function buildYomitanScanNameCandidatesScript(
nameCandidates: { key: string; forms: string[] } | null,
): string {
if (!nameCandidates) {
return `
(() => {
if (typeof globalThis.__subminerYomitanScanSetNameCandidates !== "function") {
return false;
}
return globalThis.__subminerYomitanScanSetNameCandidates(null, null);
})();
`;
}
return `
(() => {
if (typeof globalThis.__subminerYomitanScanSetNameCandidates !== "function") {
return false;
}
return globalThis.__subminerYomitanScanSetNameCandidates(
${JSON.stringify(nameCandidates.key)},
${JSON.stringify(nameCandidates.forms)}
);
})();
`;
}
export function buildYomitanScanCallScript(params: YomitanScanRequestParams): string {
return `
(async () => {
if (typeof globalThis.__subminerYomitanScan !== "function") {
return ${JSON.stringify(YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL)};
}
return await globalThis.__subminerYomitanScan(${JSON.stringify(params)});
})();
`;
}
@@ -0,0 +1,304 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { requestYomitanScanTokens } from './yomitan-parser-runtime';
import {
countTermsFindLookups,
createNameScanDeps,
NAME_SCAN_WORDS,
} from './yomitan-scan-test-harness';
// Behaviour of the in-page scan runtime around character names and kana:
// which positions the greedy pre-pass probes, and what the walk makes of
// halfwidth spellings. Driven end to end through requestYomitanScanTokens
// because the runtime only exists inside the parser window.
const NAME_SCAN_LINE = 'ミナトはまだ学校にいない';
test('requestYomitanScanTokens skips name pre-pass lookups where no candidate name can start', async () => {
const exhaustiveLookups: string[] = [];
const exhaustive = await requestYomitanScanTokens(
NAME_SCAN_LINE,
createNameScanDeps(exhaustiveLookups),
{ error: () => undefined },
{ includeNameMatchMetadata: true },
);
const prefilteredLookups: string[] = [];
const prefiltered = await requestYomitanScanTokens(
NAME_SCAN_LINE,
createNameScanDeps(prefilteredLookups),
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
// Terms and readings the generated dictionary exposes for this media.
nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] },
},
);
// Same tokenization, including the name match, with fewer round trips.
assert.deepEqual(prefiltered, exhaustive);
assert.equal(prefiltered?.[0]?.surface, 'ミナト');
assert.equal(prefiltered?.[0]?.isNameMatch, true);
assert.ok(
prefilteredLookups.length < exhaustiveLookups.length,
`expected fewer lookups with candidates (${prefilteredLookups.length} vs ${exhaustiveLookups.length})`,
);
// Mid-token positions are exactly what the pre-pass used to probe (a name can
// start mid-token); with candidates they cost nothing, while the main walk's
// own token-start lookups are unaffected.
assert.ok(countTermsFindLookups(exhaustiveLookups, '校に') > 0);
assert.equal(countTermsFindLookups(prefilteredLookups, '校に'), 0);
});
test('requestYomitanScanTokens matches a katakana name from its kana-normalized candidate form', async () => {
const lookups: string[] = [];
const result = await requestYomitanScanTokens(
NAME_SCAN_LINE,
createNameScanDeps(lookups),
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
// Only the hiragana reading is listed; the katakana surface in the line
// must still be found through kana normalization.
nameCandidates: { key: 'media-1', forms: ['みなと'] },
},
);
assert.equal(result?.[0]?.surface, 'ミナト');
assert.equal(result?.[0]?.isNameMatch, true);
});
// Kana normalization folds halfwidth katakana, so a name written that way does
// prefix-match a candidate form — but only if the position counts as Japanese
// in the first place. The generic word here reaches into the name, so only a
// pre-pass reservation can keep the name whole.
const HALFWIDTH_NAME_SCAN_WORDS: Array<[string, string, string, boolean]> = [
['ネコ', 'ネコ', 'ねこ', false],
['まだミ', 'まだミ', 'まだみ', false],
['まだ', 'まだ', 'まだ', false],
['ミナト', 'ミナト', 'みなと', true],
];
test('requestYomitanScanTokens probes halfwidth katakana positions during the name pre-pass', async () => {
const lookups: string[] = [];
const result = await requestYomitanScanTokens(
'ネコまだミナト',
createNameScanDeps(lookups, HALFWIDTH_NAME_SCAN_WORDS),
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
// Fullwidth forms only, as the generated dictionary stores them.
nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] },
},
);
assert.equal(countTermsFindLookups(lookups, 'ミナト'), 1);
// コ is mid-token, so only the pre-pass would ever look it up, and it matches
// no candidate: folding halfwidth made those positions indexable, so they no
// longer cost a round trip apiece.
assert.equal(countTermsFindLookups(lookups, 'コ'), 0);
assert.deepEqual(
result?.map((token) => token.surface),
['ネコ', 'まだ', 'ミナト'],
);
assert.equal(result?.[2]?.isNameMatch, true);
// The reading is written the way the fullwidth katakana path writes it
// (surface spelling, fullwidth): halfwidth kana is not kana to the known-word
// and frequency code downstream, and an empty reading there disables the
// reading fallback entirely.
assert.equal(result?.[2]?.reading, 'ミナト');
assert.equal(result?.[2]?.headwordReading, 'みなと');
});
test('a voiced halfwidth name still bypasses the candidate prefilter', async () => {
const lookups: string[] = [];
const result = await requestYomitanScanTokens(
'まだガク',
createNameScanDeps(lookups, [
['まだカ', 'まだカ', 'まだか', false],
['まだ', 'まだ', 'まだ', false],
['ガク', 'ガク', 'がく', true],
]),
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
nameCandidates: { key: 'media-1', forms: ['ガク', 'がく'] },
},
);
// カ + ゙ folds to か + ゙, which cannot prefix-match が, so the prefilter would
// drop this position; the voiced-mark bypass is what keeps the name.
assert.deepEqual(
result?.map((token) => token.surface),
['まだ', 'ガク'],
);
assert.equal(result?.[1]?.isNameMatch, true);
});
test('an unrelated halfwidth voiced word does not restore the exhaustive pre-pass', async () => {
const baseline: string[] = [];
await requestYomitanScanTokens(
NAME_SCAN_LINE,
createNameScanDeps(baseline),
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] },
},
);
const withVoicedTail: string[] = [];
await requestYomitanScanTokens(
`${NAME_SCAN_LINE}ガ`,
createNameScanDeps(withVoicedTail),
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] },
},
);
// Mid-token positions are the ones only the pre-pass would ever probe. A ガ
// anywhere in the line used to drag every position within scanLength of it
// back in; now only the voiced pair itself, which the fold cannot index, is
// added to what the line already looked up.
for (const midTokenPrefix of ['ナト', 'だ学', '校に', 'ない']) {
assert.equal(countTermsFindLookups(baseline, midTokenPrefix), 0, midTokenPrefix);
assert.equal(countTermsFindLookups(withVoicedTail, midTokenPrefix), 0, midTokenPrefix);
}
assert.ok(
withVoicedTail.length - baseline.length <= 3,
`expected the ガ tail to add only its own lookups, saw ${JSON.stringify(withVoicedTail)}`,
);
});
test('a mixed-width voiced name survives the candidate prefilter', async () => {
const lookups: string[] = [];
const result = await requestYomitanScanTokens(
'まだ山ガク',
createNameScanDeps(lookups, [
['まだ山', 'まだ山', 'まだやま', false],
['まだ', 'まだ', 'まだ', false],
['山ガク', '山ガク', 'やまがく', true],
]),
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
nameCandidates: { key: 'media-1', forms: ['山ガク', 'やまがく'] },
},
);
// The name starts on a kanji, so the fold only breaks mid-name: 山ガク
// normalizes to 山がく, which still cannot match the candidate 山がく. The
// bypass is keyed on the scan window rather than the first character, so the
// position is still probed and the generic まだ山 cannot swallow the 山.
assert.deepEqual(
result?.map((token) => token.surface),
['まだ', '山ガク'],
);
assert.equal(result?.[1]?.isNameMatch, true);
});
test('a stretched mixed-width voiced name survives the candidate prefilter', async () => {
const lookups: string[] = [];
const result = await requestYomitanScanTokens(
'まだ山ーーーーーーガク',
createNameScanDeps(lookups, [
['まだ山', 'まだ山', 'まだやま', false],
['まだ', 'まだ', 'まだ', false],
['山ーーーーーーガク', '山ガク', 'やまがく', true],
]),
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
nameCandidates: { key: 'media-1', forms: ['山ガク', 'やまがく'] },
},
);
// Matching skips any number of emphatic characters, so the voiced mark that
// defeats the fold can sit arbitrarily far into the name: the search for it
// has to cover the whole lookup window, not a multiple of the form length.
assert.deepEqual(
result?.map((token) => token.surface),
['まだ', '山ーーーーーーガク'],
);
assert.equal(result?.[1]?.isNameMatch, true);
});
test('halfwidth voiced kana compose into the reading instead of leaving a stray mark', async () => {
const lookups: string[] = [];
const result = await requestYomitanScanTokens(
'ガク パン',
createNameScanDeps(lookups, [
['ガク', 'ガク', 'がく', false],
['パン', 'パン', 'ぱん', false],
]),
{ error: () => undefined },
{ includeNameMatchMetadata: true },
);
// The name pre-pass runs over every position here (no candidate list), but a
// standalone voiced mark can never start a name, so it costs no lookup.
assert.equal(countTermsFindLookups(lookups, '゙'), 0);
assert.equal(countTermsFindLookups(lookups, '゚'), 0);
const readings = (result ?? [])
.filter((token) => token.isUnparsedRun !== true)
.map((token) => [token.surface, token.reading]);
assert.deepEqual(readings, [
['ガク', 'ガク'],
['パン', 'パン'],
]);
});
test('requestYomitanScanTokens falls back to the exhaustive name scan without candidates', async () => {
const withoutLookups: string[] = [];
const withoutCandidates = await requestYomitanScanTokens(
NAME_SCAN_LINE,
createNameScanDeps(withoutLookups),
{ error: () => undefined },
{ includeNameMatchMetadata: true, currentCharacterDictionaryMediaId: 1, nameCandidates: null },
);
assert.equal(withoutCandidates?.[0]?.isNameMatch, true);
// No candidate list means every Japanese position is probed, as before.
assert.ok(countTermsFindLookups(withoutLookups, '校に') > 0);
});
test('requestYomitanScanTokens reinstalls name candidates when the media changes', async () => {
const lookups: string[] = [];
const deps = createNameScanDeps(lookups);
// First media's candidates cannot match this line's name.
const otherMedia = await requestYomitanScanTokens(
NAME_SCAN_LINE,
deps,
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 2,
nameCandidates: { key: 'media-2', forms: ['カズマ'] },
},
);
assert.equal(otherMedia?.[0]?.isNameMatch, undefined);
const correctMedia = await requestYomitanScanTokens(
NAME_SCAN_LINE,
deps,
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
nameCandidates: { key: 'media-1', forms: ['ミナト'] },
},
);
assert.equal(correctMedia?.[0]?.surface, 'ミナト');
assert.equal(correctMedia?.[0]?.isNameMatch, true);
});
@@ -0,0 +1,166 @@
// Shared harness for the Yomitan parser-runtime and scan-runtime tests: fake
// parser-window deps whose injected scripts run in a vm context, plus the
// backend stubs the scanner tests drive them with. Kept out of the test files
// so the runtime tests and the in-page scanner tests can share one setup.
import * as vm from 'node:vm';
export function createDeps(
executeJavaScript: (script: string) => Promise<unknown>,
options?: {
createYomitanExtensionWindow?: (pageName: string) => Promise<unknown>;
},
) {
const parserWindow = {
isDestroyed: () => false,
webContents: {
executeJavaScript: async (script: string) => await executeJavaScript(script),
},
};
return {
getYomitanExt: () => ({ id: 'ext-id' }) as never,
getYomitanParserWindow: () => parserWindow as never,
setYomitanParserWindow: () => undefined,
getYomitanParserReadyPromise: () => null,
setYomitanParserReadyPromise: () => undefined,
getYomitanParserInitPromise: () => null,
setYomitanParserInitPromise: () => undefined,
createYomitanExtensionWindow: options?.createYomitanExtensionWindow as never,
};
}
function createYomitanScriptSandbox(handler: (action: string, params: unknown) => unknown) {
return {
chrome: {
runtime: {
lastError: null,
sendMessage: (
payload: { action?: string; params?: unknown },
callback: (response: { result?: unknown; error?: { message?: string } }) => void,
) => {
try {
callback({ result: handler(payload.action ?? '', payload.params) });
} catch (error) {
callback({ error: { message: (error as Error).message } });
}
},
},
},
Array,
Error,
JSON,
Map,
Math,
Number,
Object,
Promise,
RegExp,
Set,
String,
};
}
export async function runInjectedYomitanScript(
script: string,
handler: (action: string, params: unknown) => unknown,
): Promise<unknown> {
return await vm.runInNewContext(script, createYomitanScriptSandbox(handler));
}
// Persistent page context shared across executeJavaScript calls, matching the
// real parser window: the scan runtime is installed once via
// globalThis.__subminerYomitanScan and per-line calls reuse it (and its
// cross-line termsFind cache).
function createPersistentYomitanScriptRunner(
handler: (action: string, params: unknown) => unknown,
): (script: string) => Promise<unknown> {
const context = vm.createContext(createYomitanScriptSandbox(handler));
return async (script: string) => await vm.runInContext(script, context);
}
// Deps whose parser window executes every injected script (profile metadata,
// scan runtime install, per-line scan calls, parseText fallback) inside one
// persistent vm context, dispatching backend actions to `handler`.
export function createScanDeps(
handler: (action: string, params: unknown) => unknown,
options?: { onScript?: (script: string) => void },
) {
const runScript = createPersistentYomitanScriptRunner(handler);
return createDeps(async (script) => {
options?.onScript?.(script);
return await runScript(script);
});
}
export function countTermsFindLookups(lookups: string[], prefix: string): number {
return lookups.filter((lookupText) => lookupText.startsWith(prefix)).length;
}
// Backend stub for the greedy name pre-pass: one character name (ミナト) in a
// line of ordinary words, with the SubMiner character dictionary enabled.
export const NAME_SCAN_WORDS: Array<[string, string, string, boolean]> = [
['ミナト', 'ミナト', 'みなと', true],
['は', 'は', 'は', false],
['まだ', 'まだ', 'まだ', false],
['学校', '学校', 'がっこう', false],
['に', 'に', 'に', false],
['いない', 'いる', 'いる', false],
];
export function createNameScanDeps(
lookups: string[],
words: Array<[string, string, string, boolean]> = NAME_SCAN_WORDS,
) {
return createScanDeps((action, params) => {
if (action === 'optionsGetFull') {
return {
profileCurrent: 0,
profiles: [
{
options: {
scanning: { length: 40 },
dictionaries: [
{ name: 'JMdict', enabled: true, id: 0 },
{
name: 'SubMiner Character Dictionary (AniList 1)',
enabled: true,
id: 1,
},
],
},
},
],
};
}
if (action === 'getDictionaryInfo') {
return [];
}
if (action !== 'termsFind') {
throw new Error(`unexpected action: ${action}`);
}
const text = (params as { text?: string } | undefined)?.text ?? '';
lookups.push(text);
for (const [surface, term, reading, isName] of words) {
if (text.startsWith(surface)) {
return {
originalTextLength: surface.length,
dictionaryEntries: [
{
headwords: [
{
term,
reading,
sources: [{ originalText: surface, isPrimary: true, matchType: 'exact' }],
},
],
definitions: [
{ dictionary: isName ? 'SubMiner Character Dictionary (AniList 1)' : 'JMdict' },
],
},
],
};
}
}
return { originalTextLength: 0, dictionaryEntries: [] };
});
}
@@ -0,0 +1,21 @@
// Helper bundle for the in-page Yomitan scan runtime, composed from the
// fragments below. Injected as text into the parser window by
// yomitan-scan-runtime-script.ts, so it is data here, not code this process
// runs. The fragments are concatenated into a single function body and share
// one lexical scope: every function in them is hoisted, but the constants are
// not, so kana stays first — the later fragments read its ranges as they run.
import { YOMITAN_DICTIONARY_CLASSIFICATION_HELPERS } from './yomitan-dictionary-classification-script';
import { YOMITAN_FREQUENCY_HELPERS } from './yomitan-frequency-script';
import { YOMITAN_FURIGANA_HELPERS } from './yomitan-furigana-script';
import { YOMITAN_KANA_HELPERS } from './yomitan-kana-script';
import { YOMITAN_MATCH_SELECTION_HELPERS } from './yomitan-match-selection-script';
export { CHARACTER_DICTIONARY_TITLE_PREFIX } from './character-dictionary-title';
export const YOMITAN_SCANNING_HELPERS = [
YOMITAN_KANA_HELPERS,
YOMITAN_FURIGANA_HELPERS,
YOMITAN_FREQUENCY_HELPERS,
YOMITAN_DICTIONARY_CLASSIFICATION_HELPERS,
YOMITAN_MATCH_SELECTION_HELPERS,
].join('\n');
+54
View File
@@ -0,0 +1,54 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { HAN_CODE_POINT_RANGES, HAN_REGEXP_CLASS_BODY, isHanCodePoint } from './han-code-points';
test('every range boundary is inside the table', () => {
for (const [start, end] of HAN_CODE_POINT_RANGES) {
for (const codePoint of [start, end]) {
assert.ok(isHanCodePoint(codePoint), `expected U+${codePoint.toString(16)} to be Han`);
}
}
// Extension J (Unicode 17) and the Compatibility blocks are the ones a
// BMP-only table used to miss.
assert.ok(isHanCodePoint(0x323b0));
assert.ok(isHanCodePoint(0x33479));
assert.ok(isHanCodePoint(0xf900));
assert.ok(isHanCodePoint(0x2f800));
});
test('no unified ideograph the runtime knows about falls outside the table', () => {
// One direction only: a runtime with older Unicode data simply checks fewer
// code points, where asserting the reverse would fail on Extension J.
const unifiedIdeograph = /\p{Unified_Ideograph}/u;
for (let codePoint = 0x3000; codePoint <= 0x40000; codePoint += 1) {
if (unifiedIdeograph.test(String.fromCodePoint(codePoint))) {
assert.ok(
isHanCodePoint(codePoint),
`expected unified ideograph U+${codePoint.toString(16)} to be in the table`,
);
}
}
});
test('code points just outside the table are rejected', () => {
for (const codePoint of [0x33ff, 0x4dc0, 0xa000, 0x1f000, 0x3347a]) {
assert.equal(
isHanCodePoint(codePoint),
false,
`expected U+${codePoint.toString(16)} not to be Han`,
);
}
});
test('the regexp class body matches the same code points as the predicate', () => {
const classRegExp = new RegExp(`^[${HAN_REGEXP_CLASS_BODY}]$`, 'u');
for (const codePoint of [0x3400, 0x4e00, 0x9fff, 0xf900, 0x20000, 0x323b0, 0x33479]) {
assert.match(String.fromCodePoint(codePoint), classRegExp);
}
for (const codePoint of [0x3040, 0x30ff, 0x33fa, 0x3347a]) {
assert.doesNotMatch(String.fromCodePoint(codePoint), classRegExp);
}
});
+31
View File
@@ -0,0 +1,31 @@
// Single source of truth for "this code point is a Han character", shared by
// the main-process character dictionary and the in-page Yomitan scan runtime.
// The two used to carry separate range lists, and they drifted: a name written
// with a supplementary-plane kanji could enter the generated dictionary while
// the scanner's greedy name pre-pass refused to probe the position.
//
// Ranges rather than \p{Script=Han}: the scan walk tests one code point per
// character of every subtitle line, where an integer compare beats building a
// string for a regex, and the script is injected as text into a page where a
// shared helper cannot be imported.
export const HAN_CODE_POINT_RANGES: ReadonlyArray<readonly [number, number]> = [
[0x3400, 0x4dbf], // Extension A
[0x4e00, 0x9fff], // CJK Unified Ideographs
[0xf900, 0xfaff], // Compatibility Ideographs
[0x20000, 0x2a6df], // Extension B
[0x2a700, 0x2ebef], // Extensions C-F
[0x2ebf0, 0x2ee5f], // Extension I
[0x2f800, 0x2fa1f], // Compatibility Ideographs Supplement
[0x30000, 0x3134f], // Extension G
[0x31350, 0x323af], // Extension H
[0x323b0, 0x33479], // Extension J (Unicode 17)
];
export function isHanCodePoint(codePoint: number): boolean {
return HAN_CODE_POINT_RANGES.some(([start, end]) => codePoint >= start && codePoint <= end);
}
/** The same ranges as a regular expression character class body (needs the `u` flag). */
export const HAN_REGEXP_CLASS_BODY = HAN_CODE_POINT_RANGES.map(
([start, end]) => `\\u{${start.toString(16)}}-\\u{${end.toString(16)}}`,
).join('');
+203
View File
@@ -0,0 +1,203 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import test from 'node:test';
import { parseChangelog, resolveChangelogGroupKey } from './changelog-parse';
const SAMPLE = `# Changelog
## v0.19.2 (2026-08-04)
### Changed
- Subsync: picks both tracks now.
### Fixed
- Overlay: shows the plain line immediately.
<details>
<summary>Internal changes</summary>
### Internal
- Patched \`undici\`.
</details>
## v0.19.1 (2026-08-01)
### Added
- **Word Card Type:**
- Adds a setting.
- Flags clear each other.
## v0.18.0 (2026-07-01)
### Fixed
- Something older.
`;
test('changelog parser reads versions, dates, and sections in file order', () => {
const entries = parseChangelog(SAMPLE);
assert.deepEqual(
entries.map((entry) => `${entry.version}@${entry.date}`),
['0.19.2@2026-08-04', '0.19.1@2026-08-01', '0.18.0@2026-07-01'],
);
assert.deepEqual(
entries[0]?.sections.map((section) => section.heading),
['Changed', 'Fixed', 'Internal'],
);
assert.deepEqual(entries[0]?.sections[1]?.items, [
{ text: 'Overlay: shows the plain line immediately.', children: [] },
]);
});
test('changelog parser flags sections inside the details block as internal', () => {
const entries = parseChangelog(SAMPLE);
const sections = entries[0]?.sections ?? [];
assert.deepEqual(
sections.map((section) => section.internal),
[false, false, true],
);
assert.deepEqual(sections[2]?.items, [{ text: 'Patched `undici`.', children: [] }]);
});
test('changelog parser groups entries by major.minor', () => {
const entries = parseChangelog(SAMPLE);
assert.deepEqual(
entries.map((entry) => entry.groupKey),
['0.19', '0.19', '0.18'],
);
assert.equal(resolveChangelogGroupKey('1.2.3'), '1.2');
});
test('changelog parser keeps bullets that precede any section heading', () => {
const entries = parseChangelog('## v0.1.0 (2025-01-01)\n\n- Initial release.\n');
assert.deepEqual(entries[0]?.sections, [
{
heading: 'Changes',
items: [{ text: 'Initial release.', children: [] }],
internal: false,
},
]);
});
test('changelog parser drops empty sections and tolerates missing dates', () => {
const entries = parseChangelog('## v0.2.0\n\n### Added\n\n### Fixed\n- One fix.\n');
assert.equal(entries[0]?.date, '');
assert.deepEqual(
entries[0]?.sections.map((section) => section.heading),
['Fixed'],
);
});
test('changelog parser keeps indented sub-bullets nested under their lead bullet', () => {
const entries = parseChangelog(SAMPLE);
const added = entries[1]?.sections.find((section) => section.heading === 'Added');
assert.deepEqual(added?.items, [
{
text: '**Word Card Type:**',
children: [
{ text: 'Adds a setting.', children: [] },
{ text: 'Flags clear each other.', children: [] },
],
},
]);
});
test('changelog parser nests three bullet levels and rejoins wrapped lines', () => {
const entries = parseChangelog(
[
'## v0.9.0 (2025-05-05)',
'',
'### Added',
'- Top level',
' - Second level',
' - Third level',
' continued on the next line',
' - Back to second level',
'- Another top level',
'',
].join('\n'),
);
assert.deepEqual(entries[0]?.sections[0]?.items, [
{
text: 'Top level',
children: [
{
text: 'Second level',
children: [{ text: 'Third level continued on the next line', children: [] }],
},
{ text: 'Back to second level', children: [] },
],
},
{ text: 'Another top level', children: [] },
]);
});
test('changelog parser reads prerelease and build metadata version headings', () => {
const entries = parseChangelog(
[
'## v0.16.0 (2026-06-01)',
'',
'### Added',
'- New in 0.16.',
'',
'## v0.15.0-rc.1+build.2 (2026-05-29)',
'',
'### Added',
'- Release candidate note.',
'',
].join('\n'),
);
// The prerelease heading has to become its own entry. Asserting the exact
// version list is what catches the failure mode: a heading the regex misses
// is not skipped, its notes silently fold into the release above it.
assert.deepEqual(
entries.map((entry) => entry.version),
['0.16.0', '0.15.0-rc.1+build.2'],
);
assert.equal(entries[1]?.date, '2026-05-29');
assert.equal(entries[1]?.groupKey, '0.15');
assert.equal(entries[0]?.sections.length, 1);
// The prerelease body has to land on its own entry, not fold into 0.16.0.
assert.deepEqual(entries[1]?.sections, [
{
heading: 'Added',
items: [{ text: 'Release candidate note.', children: [] }],
internal: false,
},
]);
});
test('changelog parser handles the repo CHANGELOG.md', () => {
const markdown = fs.readFileSync(path.join(process.cwd(), 'CHANGELOG.md'), 'utf8');
const entries = parseChangelog(markdown);
assert.ok(entries.length > 3);
for (const entry of entries) {
assert.match(entry.version, /^\d+\.\d+\.\d+/);
assert.ok(entry.sections.length > 0, `expected sections for v${entry.version}`);
for (const section of entry.sections) {
for (const item of section.items) {
assert.ok(item.text.length > 0, `empty bullet in v${entry.version}`);
}
}
}
// Older entries group notes under a bold lead bullet; nesting must survive.
const breaking = entries
.find((entry) => entry.version === '0.15.0')
?.sections.find((section) => section.heading === 'Breaking Changes');
assert.deepEqual(
breaking?.items.map((item) => `${item.text}:${item.children.length}`),
['**Subsync:**:2', '**N+1 Highlighting:**:2'],
);
});
+128
View File
@@ -0,0 +1,128 @@
import type { ChangelogEntry, ChangelogItem, ChangelogSection } from '../../types/changelog';
// Prerelease and build metadata are matched separately: a single `[-+]`-led
// group cannot span `-rc.1+build.2`, and an unmatched heading silently folds
// that release's notes into the previous entry.
const VERSION_HEADING =
/^##\s+v(\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)\s*(?:\(([^)]*)\))?\s*$/;
const SECTION_HEADING = /^###\s+(.+?)\s*$/;
const BULLET = /^(\s*)[-*]\s+(.*)$/;
/**
* Entries are grouped by `major.minor` so the whole current minor line renders
* expanded, matching how docs-site/changelog.md splits current vs previous.
*/
export function resolveChangelogGroupKey(version: string): string {
const match = version.match(/^(\d+)\.(\d+)/);
if (!match) return version;
return `${match[1]}.${match[2]}`;
}
/**
* Parses the repo CHANGELOG.md into version entries. Bullets keep their inline
* markdown and their nesting: older entries group related notes under a bold
* lead bullet with indented children, and flattening them loses that structure.
*/
export function parseChangelog(markdown: string): ChangelogEntry[] {
const entries: ChangelogEntry[] = [];
let entry: ChangelogEntry | null = null;
let section: ChangelogSection | null = null;
let internal = false;
// Open bullets from outermost to innermost, used to place the next bullet.
let openItems: Array<{ indent: number; item: ChangelogItem }> = [];
function startSection(heading: string): void {
section = { heading, items: [], internal };
openItems = [];
entry?.sections.push(section);
}
function addBullet(indent: number, text: string): void {
if (!section) {
// Bullets before any "###" heading (older entries) land in a generic group.
startSection('Changes');
}
const item: ChangelogItem = { text, children: [] };
while (openItems.length > 0 && (openItems[openItems.length - 1]?.indent ?? 0) >= indent) {
openItems.pop();
}
const parent = openItems[openItems.length - 1];
if (parent) {
parent.item.children.push(item);
} else {
section?.items.push(item);
}
openItems.push({ indent, item });
}
function appendContinuation(text: string): void {
const current = openItems[openItems.length - 1];
if (!current) return;
current.item.text = `${current.item.text} ${text}`;
}
for (const rawLine of markdown.split(/\r?\n/)) {
const line = rawLine.trimEnd();
const trimmed = line.trim();
const versionMatch = trimmed.match(VERSION_HEADING);
if (versionMatch) {
const version = versionMatch[1] ?? '';
entry = {
version,
date: versionMatch[2]?.trim() ?? '',
groupKey: resolveChangelogGroupKey(version),
sections: [],
};
entries.push(entry);
section = null;
internal = false;
openItems = [];
continue;
}
if (!entry) continue;
if (trimmed.startsWith('<details')) {
internal = true;
section = null;
openItems = [];
continue;
}
if (trimmed.startsWith('</details')) {
internal = false;
section = null;
openItems = [];
continue;
}
if (trimmed.startsWith('<summary')) continue;
const sectionMatch = trimmed.match(SECTION_HEADING);
if (sectionMatch) {
startSection(sectionMatch[1] ?? '');
continue;
}
const bulletMatch = line.match(BULLET);
if (bulletMatch) {
addBullet((bulletMatch[1] ?? '').length, bulletMatch[2] ?? '');
continue;
}
// An indented non-bullet line continues the bullet above it, including
// across a blank line: that is CommonMark's continuation paragraph, and
// dropping the open bullets here would silently discard the text.
if (!trimmed) {
continue;
}
if (/^\s/.test(line)) {
appendContinuation(trimmed);
}
}
return entries.map((item) => ({
...item,
sections: item.sections.filter((entrySection) => entrySection.items.length > 0),
}));
}
+59
View File
@@ -0,0 +1,59 @@
/**
* Loose semver ordering shared by the updater and the changelog UI.
* Returns >0 when `a` is newer, <0 when older, 0 when equal.
*/
export function compareSemverLike(a: string, b: string): number {
const parse = (
value: string,
): {
core: number[];
prerelease: Array<number | string>;
} => {
// Build metadata ("+build.2") is not part of precedence per semver, and
// leaving it attached makes it leak into the prerelease comparison.
const normalized = value.replace(/^v/i, '').split('+', 1)[0] ?? '';
const [coreText = '', ...prereleaseParts] = normalized.split('-');
const core = coreText
.split('.')
.slice(0, 3)
.map((part) => Number.parseInt(part, 10) || 0);
while (core.length < 3) core.push(0);
const prereleaseText = prereleaseParts.join('-');
return {
core,
prerelease: prereleaseText
? prereleaseText.split('.').map((part) => {
const numeric = Number.parseInt(part, 10);
return /^\d+$/.test(part) ? numeric : part;
})
: [],
};
};
const left = parse(a);
const right = parse(b);
for (let i = 0; i < 3; i += 1) {
const diff = (left.core[i] ?? 0) - (right.core[i] ?? 0);
if (diff !== 0) return diff;
}
if (left.prerelease.length === 0 && right.prerelease.length === 0) return 0;
if (left.prerelease.length === 0) return 1;
if (right.prerelease.length === 0) return -1;
const length = Math.max(left.prerelease.length, right.prerelease.length);
for (let i = 0; i < length; i += 1) {
const leftPart = left.prerelease[i];
const rightPart = right.prerelease[i];
if (leftPart === undefined && rightPart === undefined) return 0;
if (leftPart === undefined) return -1;
if (rightPart === undefined) return 1;
if (leftPart === rightPart) continue;
if (typeof leftPart === 'number' && typeof rightPart === 'number') {
return leftPart - rightPart;
}
if (typeof leftPart === 'number') return -1;
if (typeof rightPart === 'number') return 1;
return leftPart > rightPart ? 1 : -1;
}
return 0;
}
+93 -13
View File
@@ -295,6 +295,7 @@ import {
importYomitanDictionaryFromZip,
initializeOverlayAnkiIntegration as initializeOverlayAnkiIntegrationCore,
initializeOverlayRuntime as initializeOverlayRuntimeCore,
isSubtitleAnnotationUpgrade,
isOverlayWindowContentReady,
jellyfinTicksToSecondsRuntime,
listJellyfinItemsRuntime,
@@ -467,6 +468,8 @@ import { openJimakuModal as openJimakuModalRuntime } from './main/runtime/jimaku
import { openTsukihimeModal as openTsukihimeModalRuntime } from './main/runtime/tsukihime-open';
import { openSubsyncManualModal as openSubsyncManualModalRuntime } from './main/runtime/subsync-open';
import { openSessionHelpModal as openSessionHelpModalRuntime } from './main/runtime/session-help-open';
import { openChangelogModal as openChangelogModalRuntime } from './main/runtime/changelog-open';
import { createChangelogRuntime } from './main/runtime/changelog/changelog-runtime';
import { openCharacterDictionaryManagerModal as openCharacterDictionaryManagerModalRuntime } from './main/runtime/character-dictionary-open';
import { openControllerSelectModal as openControllerSelectModalRuntime } from './main/runtime/controller-select-open';
import { openControllerDebugModal as openControllerDebugModalRuntime } from './main/runtime/controller-debug-open';
@@ -486,6 +489,7 @@ import { createOverlayVisibilityRuntimeService } from './main/overlay-visibility
import { createDiscordPresenceRuntime } from './main/runtime/discord-presence-runtime';
import { createCharacterDictionaryRuntimeService } from './main/character-dictionary-runtime';
import { createCharacterDictionaryImageLookup } from './main/character-dictionary-runtime/image-lookup';
import { createCharacterNameCandidateLookup } from './main/character-dictionary-runtime/name-candidates';
import {
createCharacterDictionaryAutoSyncRuntimeService,
getCharacterDictionaryManagerSnapshot,
@@ -505,6 +509,7 @@ import { createStartupOsdSequencer } from './main/runtime/startup-osd-sequencer'
import {
INSTALL_UPDATE_ACTION_ID,
UPDATE_AVAILABLE_NOTIFICATION_ID,
VIEW_CHANGELOG_ACTION_ID,
} from './main/runtime/update/update-notifications';
import { createOverlayNotificationsRuntime } from './main/runtime/overlay-notifications-runtime';
import {
@@ -1815,8 +1820,10 @@ function withCurrentSubtitleTiming(payload: SubtitleData): SubtitleData {
endTime: appState.mpvClient?.currentSubEnd ?? null,
};
}
function emitSubtitlePayload(payload: SubtitleData): void {
function emitSubtitlePayload(payload: SubtitleData, options?: { resumePrefetch?: boolean }): void {
const timedPayload = withCurrentSubtitleTiming(payload);
const currentSubtitleData = appState.currentSubtitleData;
const isAnnotationUpgrade = isSubtitleAnnotationUpgrade(currentSubtitleData, timedPayload);
const frequencyDictionary = configService.getConfig().subtitleStyle.frequencyDictionary;
const frequencyOptions = {
enabled: frequencyDictionary.enabled,
@@ -1825,10 +1832,18 @@ function emitSubtitlePayload(payload: SubtitleData): void {
};
appState.currentSubtitleData = timedPayload;
overlayManager.broadcastToOverlayWindows('subtitle:set', timedPayload);
subtitleWsService.broadcast(timedPayload, frequencyOptions);
if (!isAnnotationUpgrade) {
subtitleWsService.broadcast(timedPayload, frequencyOptions);
}
annotationSubtitleWsService.broadcast(timedPayload, frequencyOptions);
autoplayReadyGate.maybeSignalPluginAutoplayReady(timedPayload, { forceWhilePaused: true });
subtitlePrefetchService?.resume();
// resumePrefetch: false marks an emit that is not the end of the work for
// this line; prefetch stays paused until the subtitle processing controller
// settles so it does not compete with the on-screen line for the single
// Yomitan parser window.
if (options?.resumePrefetch !== false) {
subtitlePrefetchService?.resume();
}
}
function getCurrentAutoplaySubtitlePayload(): SubtitleData | null {
const payload = appState.currentSubtitleData;
@@ -1884,7 +1899,17 @@ const buildSubtitleProcessingControllerMainDepsHandler =
createBuildSubtitleProcessingControllerMainDepsHandler({
tokenizeSubtitle: async (text: string) =>
tokenizeSubtitleDeferred ? await tokenizeSubtitleDeferred(text) : { text, tokens: null },
emitSubtitle: (payload) => emitSubtitlePayload(payload),
// Controller emits never release the prefetch pause: the first emit for an
// uncached line is the provisional plain payload, sent before tokenization
// starts, so resuming on it would put prefetch back in contention with the
// on-screen line for the single parser window.
emitSubtitle: (payload) => emitSubtitlePayload(payload, { resumePrefetch: false }),
// The pause is released once the controller has no work left, which covers
// the runs that end without an emit (suppressed duplicate, failed
// tokenization) as well as the ones that deliver a payload.
onProcessingSettled: () => {
subtitlePrefetchService?.resume();
},
logDebug: (message) => {
logger.debug(`[subtitle-processing] ${message}`);
},
@@ -1921,7 +1946,7 @@ const autoplaySubtitlePrimingRuntime = createAutoplaySubtitlePrimingRuntime({
appState.activeParsedSubtitleMediaPath = mediaPath;
},
subtitleProcessingController,
emitSubtitlePayload: (payload) => emitSubtitlePayload(payload),
emitSubtitlePayload: (payload, options) => emitSubtitlePayload(payload, options),
getSubtitlePrefetchService: () => subtitlePrefetchService,
getLastObservedTimePos: () => lastObservedTimePos,
getVisibleOverlayVisible: () => overlayManager.getVisibleOverlayVisible(),
@@ -1955,7 +1980,6 @@ const subtitlePrefetchInitController = createSubtitlePrefetchInitController({
subtitleProcessingController.preCacheTokenization(text, data);
},
hasCachedTokenization: (text) => subtitleProcessingController.hasCachedSubtitle(text),
isCacheFull: () => subtitleProcessingController.isCacheFull(),
logInfo: (message) => logger.info(message),
logWarn: (message) => logger.warn(message),
onParsedSubtitleCuesChanged: (cues, sourceKey) => {
@@ -1982,15 +2006,22 @@ const resolveActiveSubtitleSidebarSourceHandler = createResolveActiveSubtitleSid
getFfmpegPath: () => configService.getConfig().subsync.ffmpeg_path.trim() || 'ffmpeg',
extractInternalSubtitleTrack: (ffmpegPath, videoPath, track) =>
extractInternalSubtitleTrackToTempFile(ffmpegPath, videoPath, track),
logDebug: (message) => logger.debug(message),
});
const refreshSubtitlePrefetchFromActiveTrackHandler =
createRefreshSubtitlePrefetchFromActiveTrackHandler({
getMpvClient: () => appState.mpvClient,
getLastObservedTimePos: () => lastObservedTimePos,
shouldKeepExistingCuesOnMissingSource: (videoPath) => isYoutubeMediaPath(videoPath),
// Remote media has no extractable on-disk track to fall back to, so a transient
// resolve miss (sid briefly 'no', a cycle onto an embedded stream track) would
// otherwise drop a working cue list for the rest of the episode.
shouldKeepExistingCuesOnMissingSource: (videoPath) =>
isYoutubeMediaPath(videoPath) || isRemoteMediaPath(videoPath),
subtitlePrefetchInitController,
resolveActiveSubtitleSidebarSource: (input) => resolveActiveSubtitleSidebarSourceHandler(input),
logDebug: (message) => logger.debug(message),
logWarn: (message) => logger.warn(message),
});
const subtitlePrefetchRuntime = {
@@ -2521,6 +2552,10 @@ const characterDictionaryAutoSyncRuntime = createCharacterDictionaryAutoSyncRunt
},
{
hasParserWindow: () => Boolean(appState.yomitanParserWindow),
invalidateCharacterDictionaryLookups: () => {
characterDictionaryImageLookup.invalidate();
characterNameCandidateLookup.invalidate();
},
clearParserCaches: () => {
if (appState.yomitanParserWindow) {
clearYomitanParserCachesForWindow(appState.yomitanParserWindow);
@@ -2546,6 +2581,13 @@ const characterDictionaryImageLookup = createCharacterDictionaryImageLookup({
getCurrentMediaId: () => characterDictionaryAutoSyncRuntime.getCurrentMediaId(),
});
// Lets the Yomitan scan runtime skip name lookups at positions where no
// character name can start; absent candidates just mean the exhaustive scan.
const characterNameCandidateLookup = createCharacterNameCandidateLookup({
userDataPath: USER_DATA_PATH,
getCurrentMediaId: () => characterDictionaryAutoSyncRuntime.getCurrentMediaId(),
});
const overlayVisibilityRuntime = createOverlayVisibilityRuntimeService(
createBuildOverlayVisibilityRuntimeMainDepsHandler({
getMainWindow: () => overlayManager.getMainWindow(),
@@ -2817,6 +2859,14 @@ function openSessionHelpOverlay(): void {
);
}
function openChangelogOverlay(): void {
openOverlayHostedModalWithOsd(
openChangelogModalRuntime,
'Changelog overlay unavailable.',
'Failed to open changelog overlay.',
);
}
function openCharacterDictionaryManagerOverlay(): void {
openCharacterDictionaryManagerWithConfigGate({
isCharacterDictionaryEnabled: () => configService.getConfig().subtitleStyle.nameMatchEnabled,
@@ -2994,6 +3044,8 @@ const {
streamIndex,
delaySeconds,
}),
initSubtitlePrefetch: (sourcePath) =>
subtitlePrefetchRuntime.refreshSubtitleSidebarFromSource(sourcePath),
logDebug: (message, error) => {
logger.debug(message, error);
},
@@ -3958,7 +4010,10 @@ const refreshCurrentSubtitleAfterKnownWordUpdate = (): void => {
}
subtitleProcessingController.invalidateTokenizationCache();
subtitlePrefetchService?.onSeek(lastObservedTimePos);
subtitleProcessingController.refreshCurrentSubtitle(appState.currentSubText);
if (!subtitleProcessingController.refreshCurrentSubtitle(appState.currentSubText)) {
// Idle controller: no settle is coming to release the pause above.
subtitlePrefetchService?.resume();
}
};
let hasAttemptedImmersionTrackerStartup = false;
const ensureImmersionTrackerStarted = (): void => {
@@ -4359,13 +4414,20 @@ const {
emitSubtitlePayload(payload);
},
onSubtitleChange: (text) => {
// Pause only; restarting the prefetch run here would discard in-flight
// tokenization work on every line. Real seeks restart via onTimePosUpdate.
subtitlePrefetchService?.pause();
subtitlePrefetchService?.onSeek(lastObservedTimePos);
subtitleProcessingController.onSubtitleChange(text);
if (!subtitleProcessingController.onSubtitleChange(text)) {
// Repeat of the current text: the controller is idle, so no settle is
// coming to release the pause. Resume now instead of idling prefetch
// for the rest of the cue.
subtitlePrefetchService?.resume();
}
},
refreshDiscordPresence: () => {
discordPresenceRuntime.publishDiscordPresence();
},
logSubtitleProcessingDebug: (message: string) => logger.debug(message),
ensureImmersionTrackerInitialized: () => {
ensureImmersionTrackerStarted();
},
@@ -4604,6 +4666,7 @@ const {
getCharacterNameImage: (term) => characterDictionaryImageLookup.get(term),
getCurrentCharacterDictionaryMediaId: () =>
characterDictionaryAutoSyncRuntime.getCurrentMediaId(),
getCharacterNameCandidates: () => characterNameCandidateLookup.get(),
getFrequencyDictionaryEnabled: () =>
getRuntimeBooleanOption(
'subtitle.annotation.frequency',
@@ -5081,6 +5144,18 @@ flushPendingMpvLogWrites = () => {
void flushMpvLog();
};
const { getChangelogSnapshot } = createChangelogRuntime({
getInstalledVersion: () => app.getVersion(),
getUpdateChannel: () => configService.getConfig().updates.channel,
resourcesPath: process.resourcesPath,
appPath: app.getAppPath(),
dirname: __dirname,
joinPath: (...parts) => path.join(...parts),
fileExists: (candidate) => fs.existsSync(candidate),
readFile: (candidate) => fs.readFileSync(candidate, 'utf8'),
logWarn: (message) => logger.warn(message),
});
const { getUpdateService } = createUpdateServiceRuntime({
userDataPath: USER_DATA_PATH,
getUpdatesConfig: () => configService.getConfig().updates,
@@ -5449,6 +5524,12 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
logger.warn('Failed to install update from overlay notification action:', error);
});
}
if (
notificationId === UPDATE_AVAILABLE_NOTIFICATION_ID &&
actionId === VIEW_CHANGELOG_ACTION_ID
) {
openChangelogOverlay();
}
if (actionId === OPEN_ANKI_CARD_ACTION_ID && noteId !== undefined) {
void openAnkiCardFromNotification(noteId).catch((error) => {
logger.warn('Failed to open Anki card from overlay notification action:', error);
@@ -5658,7 +5739,6 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
if (result.ok && result.rebuildRequired) {
try {
await characterDictionaryAutoSyncRuntime.runSyncNow();
characterDictionaryImageLookup.invalidate();
} catch (error) {
logger.warn('Failed to rebuild character dictionary after manager override:', error);
}
@@ -5689,7 +5769,6 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
if (result.ok && result.rebuildRequired) {
try {
await characterDictionaryAutoSyncRuntime.runSyncNow();
characterDictionaryImageLookup.invalidate();
} catch (error) {
logger.warn('Failed to rebuild character dictionary after manager removal:', error);
}
@@ -5706,7 +5785,6 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
if (result.ok && result.rebuildRequired) {
try {
await characterDictionaryAutoSyncRuntime.runSyncNow();
characterDictionaryImageLookup.invalidate();
} catch (error) {
logger.warn('Failed to rebuild character dictionary after manager reorder:', error);
}
@@ -5714,6 +5792,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
return result;
},
appendClipboardVideoToQueue: () => appendClipboardVideoToQueueHandler(),
getChangelogSnapshot: (options) => getChangelogSnapshot(options),
...playlistBrowserMainDeps,
getImmersionTracker: () => appState.immersionTracker,
},
@@ -6104,6 +6183,7 @@ const { ensureTray: ensureTrayHandler, destroyTray: destroyTrayHandler } =
initializeOverlayRuntime: () => initializeOverlayRuntime(),
isOverlayRuntimeInitialized: () => appState.overlayRuntimeInitialized,
openSessionHelpModal: () => openSessionHelpOverlay(),
openChangelogModal: () => openChangelogOverlay(),
openTexthookerInBrowser: () =>
handleCliCommand(parseArgs(['--texthooker', '--open-browser'])),
showTexthookerPage: () => shouldShowTexthookerTrayEntry(configService.getConfig()),
@@ -1,7 +1,7 @@
export const ANILIST_GRAPHQL_URL = 'https://graphql.anilist.co';
export const ANILIST_REQUEST_DELAY_MS = 2000;
export const CHARACTER_IMAGE_DOWNLOAD_DELAY_MS = 250;
export const CHARACTER_DICTIONARY_FORMAT_VERSION = 19;
export const CHARACTER_DICTIONARY_FORMAT_VERSION = 20;
export const CHARACTER_DICTIONARY_MERGED_TITLE = 'SubMiner Character Dictionary';
export const HONORIFIC_SUFFIXES = [
@@ -0,0 +1,163 @@
import assert from 'node:assert/strict';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import test from 'node:test';
import { CHARACTER_DICTIONARY_FORMAT_VERSION } from './constants';
import { createCharacterNameCandidateLookup } from './name-candidates';
function writeSnapshot(outputDir: string, mediaId: number, entries: Array<[string, string]>): void {
const snapshotsDir = path.join(outputDir, 'snapshots');
fs.mkdirSync(snapshotsDir, { recursive: true });
fs.writeFileSync(
path.join(snapshotsDir, `anilist-${mediaId}.json`),
JSON.stringify({
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
mediaId,
mediaTitle: `title-${mediaId}`,
entryCount: entries.length,
updatedAt: 1,
termEntries: entries.map(([term, reading]) => [
term,
reading,
'name main',
'',
100,
[],
0,
'',
]),
images: [],
}),
);
}
function withTempDir<T>(run: (dir: string) => T): T {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-name-candidates-'));
try {
return run(dir);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
}
test('collects terms and readings for the current media', () => {
withTempDir((dir) => {
writeSnapshot(dir, 1, [
['ミナト', 'みなと'],
['湊', 'みなと'],
]);
writeSnapshot(dir, 2, [['カズマ', 'かずま']]);
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => 1,
});
const candidates = lookup.get();
assert.ok(candidates);
assert.deepEqual([...candidates.forms].sort(), ['みなと', 'ミナト', '湊'].sort());
// Deduplicated: both entries share the みなと reading.
assert.equal(candidates.forms.length, 3);
});
});
test('returns null without a media scope so the scanner stays exhaustive', () => {
withTempDir((dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => null,
});
assert.equal(lookup.get(), null);
});
});
test('returns null for a media with no cached snapshot', () => {
withTempDir((dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => 999,
});
assert.equal(lookup.get(), null);
});
});
test('key changes when the snapshot content changes', () => {
withTempDir((dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => 1,
});
const first = lookup.get();
writeSnapshot(dir, 1, [
['ミナト', 'みなと'],
['アクア', 'あくあ'],
]);
lookup.invalidate();
const second = lookup.get();
assert.ok(first && second);
assert.notEqual(first.key, second.key);
assert.equal(second.forms.length, 4);
});
});
// The lookup runs once per subtitle line, so it must not stat the snapshot
// directory every call. Asserted behaviorally: an unannounced on-disk change is
// invisible until the recheck interval elapses, which can only be true if the
// filesystem is not consulted per lookup.
test('does not re-read the snapshot directory on every lookup', () => {
withTempDir((dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
let nowMs = 1_000_000;
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => 1,
now: () => nowMs,
});
assert.equal(lookup.get()?.forms.length, 2);
writeSnapshot(dir, 1, [
['ミナト', 'みなと'],
['アクア', 'あくあ'],
]);
nowMs += 1000;
assert.equal(lookup.get()?.forms.length, 2, 'expected the cached list within the interval');
nowMs += 10_000;
assert.equal(lookup.get()?.forms.length, 4, 'expected a refresh past the interval');
});
});
test('invalidate picks up a snapshot change immediately', () => {
withTempDir((dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
let nowMs = 1_000_000;
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => 1,
now: () => nowMs,
});
assert.equal(lookup.get()?.forms.length, 2);
writeSnapshot(dir, 1, [
['ミナト', 'みなと'],
['アクア', 'あくあ'],
]);
nowMs += 1;
lookup.invalidate();
assert.equal(lookup.get()?.forms.length, 4);
});
});
@@ -0,0 +1,159 @@
import * as fs from 'fs';
import * as path from 'path';
import { readCachedSnapshots } from './cache';
import type { CharacterDictionarySnapshot } from './types';
// Candidate name forms for the greedy name pre-pass in the Yomitan scan
// runtime. The scanner otherwise has to ask the backend at every Japanese
// position, because a character name can start mid-token; knowing which forms
// exist lets it look up only where a name can actually begin.
//
// A form is any string Yomitan could match a character entry by: the term and
// its reading. Both come from the dictionary SubMiner generated, so the pair is
// the complete matchable set for an entry. Callers treat a missing list as
// "scan every position", so a stale or absent snapshot costs speed, never a
// missed name.
function getSnapshotsDir(outputDir: string): string {
return path.join(outputDir, 'snapshots');
}
function collectSnapshotNameForms(snapshot: CharacterDictionarySnapshot): string[] {
const forms = new Set<string>();
for (const entry of snapshot.termEntries) {
const term = typeof entry[0] === 'string' ? entry[0].trim() : '';
if (term) {
forms.add(term);
}
const reading = typeof entry[1] === 'string' ? entry[1].trim() : '';
if (reading) {
forms.add(reading);
}
}
return [...forms];
}
// The signature grows with the size of the dictionary library, and it rides
// along in every per-line scan call, so it is folded into a fixed-width digest
// first. Collisions only matter against the immediately previous signature (the
// runtime compares keys for equality), and FNV-1a over the file list is far
// beyond what that needs.
function digestSnapshotDirectorySignature(signature: string): string {
let hash = 0x811c9dc5;
for (let index = 0; index < signature.length; index += 1) {
hash ^= signature.charCodeAt(index);
hash = Math.imul(hash, 0x01000193);
}
return (hash >>> 0).toString(36);
}
function getSnapshotDirectorySignature(outputDir: string): string {
let entries: fs.Dirent[] = [];
try {
entries = fs.readdirSync(getSnapshotsDir(outputDir), { withFileTypes: true });
} catch {
return '';
}
const parts: string[] = [];
for (const entry of entries) {
if (!entry.isFile() || !/^anilist-\d+\.json$/.test(entry.name)) {
continue;
}
try {
const stat = fs.statSync(path.join(getSnapshotsDir(outputDir), entry.name));
parts.push(`${entry.name}:${stat.mtimeMs}:${stat.size}`);
} catch {
// Ignore files that disappear during a refresh; the next lookup rebuilds.
}
}
return parts.sort().join('|');
}
export interface CharacterNameCandidateSet {
/** Identifies this exact form list, so the scan runtime can cache it. */
key: string;
forms: string[];
}
// This lookup is consulted once per subtitle line, so it must not stat the
// snapshot directory every time. Dictionary writes are rare and always call
// invalidate(), which forces the next lookup to re-read; the interval only
// bounds staleness from changes made behind our back.
const SNAPSHOT_SIGNATURE_RECHECK_INTERVAL_MS = 5000;
export function createCharacterNameCandidateLookup(deps: {
userDataPath?: string;
outputDir?: string;
getCurrentMediaId?: () => number | null | undefined;
now?: () => number;
}): {
get: (mediaId?: number | null) => CharacterNameCandidateSet | null;
invalidate: () => void;
} {
const outputDir =
deps.outputDir ??
(deps.userDataPath ? path.join(deps.userDataPath, 'character-dictionaries') : '');
const now = deps.now ?? (() => Date.now());
let signature: string | null = null;
let lastSignatureCheckAtMs = 0;
let formsByMediaId = new Map<number, string[]>();
function refreshIfNeeded(): void {
if (!outputDir) {
formsByMediaId = new Map<number, string[]>();
signature = '';
return;
}
const nowMs = now();
if (
signature !== null &&
nowMs - lastSignatureCheckAtMs < SNAPSHOT_SIGNATURE_RECHECK_INTERVAL_MS
) {
return;
}
lastSignatureCheckAtMs = nowMs;
const nextSignature = getSnapshotDirectorySignature(outputDir);
if (nextSignature === signature) {
return;
}
signature = nextSignature;
formsByMediaId = new Map<number, string[]>();
for (const snapshot of readCachedSnapshots(outputDir)) {
const forms = collectSnapshotNameForms(snapshot);
if (forms.length > 0) {
formsByMediaId.set(snapshot.mediaId, forms);
}
}
}
return {
get(mediaId?: number | null): CharacterNameCandidateSet | null {
refreshIfNeeded();
const rawMediaId = mediaId ?? deps.getCurrentMediaId?.() ?? null;
const normalizedMediaId =
typeof rawMediaId === 'number' && Number.isFinite(rawMediaId) && rawMediaId > 0
? Math.floor(rawMediaId)
: null;
// Without a media scope the pre-pass would need every character of every
// cached title, which is both slow to match and pointless: report no
// candidates so the scanner keeps its exhaustive behavior.
if (normalizedMediaId === null) {
return null;
}
const forms = formsByMediaId.get(normalizedMediaId);
if (!forms || forms.length === 0) {
return null;
}
return {
key: `${digestSnapshotDirectorySignature(signature ?? '')}:${normalizedMediaId}`,
forms,
};
},
invalidate(): void {
signature = null;
lastSignatureCheckAtMs = 0;
},
};
}
@@ -1,3 +1,4 @@
import { isHanCodePoint } from '../../core/text/han-code-points';
import { HONORIFIC_SUFFIXES } from './constants';
import type { JapaneseNameParts, NameReadings, ResolvedNameSplits } from './types';
@@ -26,10 +27,12 @@ export function buildReading(term: string): string {
return katakanaToHiragana(compact);
}
// Code points, not code units: a supplementary-plane kanji (𠮷, U+20BB7) is a
// surrogate pair, and reading only the high surrogate would classify a real
// single-character name as non-kanji and drop it.
export function containsKanji(value: string): boolean {
for (const char of value) {
const code = char.charCodeAt(0);
if ((code >= 0x4e00 && code <= 0x9fff) || (code >= 0x3400 && code <= 0x4dbf)) {
if (isHanCodePoint(char.codePointAt(0) ?? 0)) {
return true;
}
}
@@ -36,3 +36,134 @@ test('buildNameTerms adds surname honorifics from Japanese localized aliases', (
assert.ok(terms.includes('馬渕さん'));
assert.ok(!terms.includes('송치'));
});
test('buildNameTerms drops the disambiguator letter of a mob character name', () => {
const terms = buildNameTerms(
characterRecord({
firstNameHint: '',
lastNameHint: '',
fullName: 'Joshi A',
nativeName: '女子A',
}),
);
// ア would match every あ〜 in the subtitles; the letter is a disambiguator
// (Girl A / Girl B), not a name.
assert.ok(!terms.includes('ア'));
assert.ok(!terms.includes('アさん'));
assert.ok(terms.includes('女子A'));
assert.ok(terms.includes('ジョシア'));
});
test('buildNameTerms keeps a character whose whole name is one kana', () => {
const terms = buildNameTerms(
characterRecord({
firstNameHint: '',
lastNameHint: '',
fullName: 'A',
nativeName: 'あ',
}),
);
// The mob-label rule only judges parts a name was split into; a name the
// source gives us whole is the character's actual name.
assert.ok(terms.includes('あ'));
assert.ok(terms.includes('あさん'));
// Romanized forms are never lookup targets (the subtitles are Japanese), and
// the single-kana alias "A" transliterates to is dropped as a collision.
assert.ok(!terms.includes('A'));
assert.ok(!terms.includes('ア'));
});
test('buildNameTerms keeps a one-character name written in another script', () => {
const terms = buildNameTerms(
characterRecord({
firstNameHint: '',
lastNameHint: '',
fullName: 'Byeol',
nativeName: '별',
alternativeNames: ['Я'],
}),
);
assert.ok(terms.includes('별'));
assert.ok(terms.includes('별さん'));
assert.ok(terms.includes('Я'));
});
test('buildNameTerms yields nothing for a character whose only name is a bare letter', () => {
// Documented policy rather than an oversight: a romanized name is never a
// term on its own (the subtitles are Japanese), and the single kana a bare
// letter transliterates to would match every あ〜 in the line.
assert.deepEqual(
buildNameTerms(
characterRecord({
firstNameHint: '',
lastNameHint: '',
fullName: 'A',
nativeName: '',
}),
),
[],
);
});
test('buildNameTerms keeps one-character split parts that are not mob labels', () => {
const hangul = buildNameTerms(
characterRecord({
firstNameHint: '',
lastNameHint: '',
fullName: 'Byeol Kim',
nativeName: '별 김',
}),
);
assert.ok(hangul.includes('별'));
assert.ok(hangul.includes('김'));
const middleDot = buildNameTerms(
characterRecord({
firstNameHint: '',
lastNameHint: '',
fullName: 'A Be',
nativeName: 'ア・ベ',
}),
);
assert.ok(middleDot.includes('ア'));
assert.ok(middleDot.includes('ベ'));
});
test('buildNameTerms keeps a single-kanji name part', () => {
// The name is an alias, not the native name, so the parts come from the
// space split rather than from the native-name split.
const terms = buildNameTerms(
characterRecord({
firstNameHint: 'Sora',
lastNameHint: 'Yamada',
fullName: 'Sora Yamada',
nativeName: '',
alternativeNames: ['山田 空'],
}),
);
assert.ok(terms.includes('山田'));
assert.ok(terms.includes('空'));
});
test('buildNameTerms keeps a single supplementary-plane kanji name part', () => {
// 𠮷 (U+20BB7) is a surrogate pair: a code-unit kanji check reads only the
// high surrogate and drops the part as if it were a mob disambiguator.
const terms = buildNameTerms(
characterRecord({
firstNameHint: 'Tsukasa',
lastNameHint: 'Yoshi',
fullName: 'Tsukasa Yoshi',
nativeName: '',
alternativeNames: ['𠮷 司'],
}),
);
assert.ok(terms.includes('𠮷'));
assert.ok(terms.includes('司'));
});
@@ -1,3 +1,4 @@
import { HAN_REGEXP_CLASS_BODY } from '../../core/text/han-code-points';
import { HONORIFIC_SUFFIXES } from './constants';
import {
addRomanizedKanaAliases,
@@ -42,11 +43,29 @@ export function expandRawNameVariants(rawName: string): string[] {
return [...variants];
}
// The label AniList appends to unnamed mob characters: one letter or digit,
// halfwidth or fullwidth (女子A / "Joshi A" / 女子1). Nothing else qualifies —
// a one-character part in any script is a real name part (별 김, ア・ベ, 山田 空).
const SINGLE_LABEL_CHARACTER = /^[0-9A-Za-z\uff10-\uff19\uff21-\uff3a\uff41-\uff5a]$/u;
// Judged on split parts only: a name the source gives us whole in a script the
// subtitles can contain is kept whatever it looks like, because a character
// really can be called あ or 별. (A romanized name is a separate matter: it is
// never a term on its own, only a source of kana aliases. See below.)
function isUsableNameSplitPart(part: string): boolean {
return !SINGLE_LABEL_CHARACTER.test(part);
}
// Kana, Han (shared ranges), and the marks that only ever appear inside a
// Japanese name: iteration marks and the small ka/ke used in place names.
const JAPANESE_NAME_CHARACTERS = new RegExp(
`^[\\u3040-\\u30ff${HAN_REGEXP_CLASS_BODY}\u3005\u3006\u30f5\u30f6\u30fc]+$`,
'u',
);
export function isJapaneseNameSplitCandidate(name: string): boolean {
const compact = name.replace(/[\s\u3000・・·•]/g, '');
return (
containsKanji(compact) && /^[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff々〆ヵヶー]+$/.test(compact)
);
return containsKanji(compact) && JAPANESE_NAME_CHARACTERS.test(compact);
}
function addJapaneseNameParts(
@@ -97,8 +116,11 @@ export function buildNameTerms(
const split = name.split(/[\s\u3000]+/).filter((part) => part.trim().length > 0);
if (split.length === 2) {
target.add(split[0]!);
target.add(split[1]!);
for (const part of split) {
if (isUsableNameSplitPart(part)) {
target.add(part);
}
}
}
const splitByMiddleDot = name
@@ -107,7 +129,9 @@ export function buildNameTerms(
.filter((part) => part.length > 0);
if (splitByMiddleDot.length >= 2) {
for (const part of splitByMiddleDot) {
target.add(part);
if (isUsableNameSplitPart(part)) {
target.add(part);
}
}
}
@@ -117,7 +141,15 @@ export function buildNameTerms(
}
}
// Romanized names never become terms themselves — the subtitles are Japanese,
// so "Joshi A" would never appear in one — they only contribute the kana a
// Japanese writer would spell them with.
for (const alias of addRomanizedKanaAliases(romanizedBase)) {
// Except when the whole name is one letter: it transliterates to a single
// kana (A → ア) that matches every あ〜 in the subtitles. A character whose
// only recorded name is a bare letter therefore yields no terms at all,
// which is the intended outcome: those are unnamed mob characters.
if ([...alias].length === 1) continue;
base.add(alias);
}
+2
View File
@@ -109,6 +109,7 @@ export interface MainIpcRuntimeServiceDepsParams {
removeCharacterDictionaryManagedEntry?: IpcDepsRuntimeOptions['removeCharacterDictionaryManagedEntry'];
moveCharacterDictionaryManagedEntry?: IpcDepsRuntimeOptions['moveCharacterDictionaryManagedEntry'];
appendClipboardVideoToQueue: IpcDepsRuntimeOptions['appendClipboardVideoToQueue'];
getChangelogSnapshot?: IpcDepsRuntimeOptions['getChangelogSnapshot'];
getPlaylistBrowserSnapshot: IpcDepsRuntimeOptions['getPlaylistBrowserSnapshot'];
appendPlaylistBrowserFile: IpcDepsRuntimeOptions['appendPlaylistBrowserFile'];
playPlaylistBrowserIndex: IpcDepsRuntimeOptions['playPlaylistBrowserIndex'];
@@ -302,6 +303,7 @@ export function createMainIpcRuntimeServiceDeps(
removeCharacterDictionaryManagedEntry: params.removeCharacterDictionaryManagedEntry,
moveCharacterDictionaryManagedEntry: params.moveCharacterDictionaryManagedEntry,
appendClipboardVideoToQueue: params.appendClipboardVideoToQueue,
getChangelogSnapshot: params.getChangelogSnapshot,
getPlaylistBrowserSnapshot: params.getPlaylistBrowserSnapshot,
appendPlaylistBrowserFile: params.appendPlaylistBrowserFile,
playPlaylistBrowserIndex: params.playPlaylistBrowserIndex,
+85 -10
View File
@@ -176,6 +176,29 @@ test('subtitle sidebar media path tag is assigned after prefetch succeeds', () =
);
});
test('remote media keeps parsed cues when the active subtitle source cannot be resolved', () => {
const source = readMainSource();
const actionBlock = source.match(
/createRefreshSubtitlePrefetchFromActiveTrackHandler\(\{(?<body>[\s\S]*?)\n \}\);/,
)?.groups?.body;
assert.ok(actionBlock);
assert.match(actionBlock, /isYoutubeMediaPath\(videoPath\) \|\| isRemoteMediaPath\(videoPath\)/);
});
test('jellyfin subtitle preload seeds the tokenization prefetch directly', () => {
const source = readMainSource();
const actionBlock = source.match(
/preloadJellyfinExternalSubtitlesMainDeps:\s*\{(?<body>[\s\S]*?)\n \},/,
)?.groups?.body;
assert.ok(actionBlock);
assert.match(
actionBlock,
/initSubtitlePrefetch: \(sourcePath\) =>\s*subtitlePrefetchRuntime\.refreshSubtitleSidebarFromSource\(sourcePath\),/,
);
});
test('update overlay notification action triggers install flow', () => {
const source = readMainSource();
const runtimeSource = readSource('src/main/runtime/overlay-notifications-runtime.ts');
@@ -200,7 +223,7 @@ test('update overlay notification action triggers install flow', () => {
assert.match(runtimeSource, /fallbackClient\.openNoteInBrowser\(noteId\)/);
});
test('subtitle change re-prioritizes prefetch around live playback before tokenizing current line', () => {
test('subtitle change pauses prefetch without restarting its run before tokenizing current line', () => {
const source = readMainSource();
const actionBlock = source.match(
/onSubtitleChange:\s*\(text\)\s*=>\s*\{(?<body>[\s\S]*?)\n \},\n refreshDiscordPresence:/,
@@ -208,15 +231,19 @@ test('subtitle change re-prioritizes prefetch around live playback before tokeni
assert.ok(actionBlock);
assert.match(actionBlock, /subtitlePrefetchService\?\.pause\(\);/);
assert.match(actionBlock, /subtitlePrefetchService\?\.onSeek\(lastObservedTimePos\);/);
assert.match(actionBlock, /subtitleProcessingController\.onSubtitleChange\(text\);/);
// Restarting the run per line (onSeek) discards in-flight prefetch work;
// only real seeks restart via onTimePosUpdate.
assert.doesNotMatch(actionBlock, /subtitlePrefetchService\?\.onSeek\(/);
assert.match(actionBlock, /subtitleProcessingController\.onSubtitleChange\(text\)/);
assert.ok(
actionBlock.indexOf('subtitlePrefetchService?.pause();') <
actionBlock.indexOf('subtitlePrefetchService?.onSeek(lastObservedTimePos);'),
actionBlock.indexOf('subtitleProcessingController.onSubtitleChange(text)'),
);
assert.ok(
actionBlock.indexOf('subtitlePrefetchService?.onSeek(lastObservedTimePos);') <
actionBlock.indexOf('subtitleProcessingController.onSubtitleChange(text);'),
// A repeated subtitle emits nothing, so the pause has to be released here or
// prefetching idles until the next distinct line.
assert.match(
actionBlock,
/if \(!subtitleProcessingController\.onSubtitleChange\(text\)\) \{[\s\S]*?subtitlePrefetchService\?\.resume\(\);/,
);
});
@@ -466,16 +493,35 @@ test('known-word updates invalidate prefetched tokenizations before refreshing c
assert.match(actionBlock, /subtitlePrefetchService\?\.onSeek\(lastObservedTimePos\);/);
assert.match(
actionBlock,
/subtitleProcessingController\.refreshCurrentSubtitle\(appState\.currentSubText\);/,
/if \(!subtitleProcessingController\.refreshCurrentSubtitle\(appState\.currentSubText\)\) \{[\s\S]*?subtitlePrefetchService\?\.resume\(\);/,
);
assert.ok(
actionBlock.indexOf('subtitleProcessingController.invalidateTokenizationCache();') <
actionBlock.indexOf(
'subtitleProcessingController.refreshCurrentSubtitle(appState.currentSubText);',
'subtitleProcessingController.refreshCurrentSubtitle(appState.currentSubText)',
),
);
});
test('subtitle processing controller resumes prefetch on settle, not on its emits', () => {
const source = readMainSource();
const depsBlock = source.match(
/createBuildSubtitleProcessingControllerMainDepsHandler\(\{(?<body>[\s\S]*?)\n \}\);/,
)?.groups?.body;
assert.ok(depsBlock);
// A controller emit can be the provisional plain payload sent before the
// scan runs, so it must not release the prefetch pause.
assert.match(
depsBlock,
/emitSubtitle: \(payload\) => emitSubtitlePayload\(payload, \{ resumePrefetch: false \}\),/,
);
assert.match(
depsBlock,
/onProcessingSettled: \(\) => \{\s+subtitlePrefetchService\?\.resume\(\);/,
);
});
test('manual visible overlay changes notify mpv plugin visibility state', () => {
const source = readMainSource();
const setBlock = source.match(
@@ -570,7 +616,7 @@ test('YouTube media cache lifecycle routes through configured status notificatio
test('subtitle broadcasts share one frequency options snapshot per emitted payload', () => {
const source = readMainSource();
const emitBlock = source.match(
/function emitSubtitlePayload\(payload: SubtitleData\): void \{(?<body>[\s\S]*?)\n\}/,
/function emitSubtitlePayload\([\s\S]*?\): void \{(?<body>[\s\S]*?)\n\}/,
)?.groups?.body;
const frequencyOptionsSnapshot = emitBlock?.match(
/const frequencyDictionary = configService\.getConfig\(\)\.subtitleStyle\.frequencyDictionary;(?<body>[\s\S]*?)\n \};/,
@@ -590,6 +636,35 @@ test('subtitle broadcasts share one frequency options snapshot per emitted paylo
);
});
test('annotation upgrades skip the duplicate basic websocket event', () => {
const source = readMainSource();
const emitBlock = source.match(
/function emitSubtitlePayload\([\s\S]*?\): void \{(?<body>[\s\S]*?)\n\}/,
)?.groups?.body;
assert.ok(emitBlock);
assert.match(
emitBlock,
/const isAnnotationUpgrade = isSubtitleAnnotationUpgrade\(currentSubtitleData, timedPayload\);/,
);
assert.match(
emitBlock,
/if \(!isAnnotationUpgrade\) \{\s+subtitleWsService\.broadcast\(timedPayload, frequencyOptions\);\s+\}/,
);
assert.equal(
(emitBlock.match(/overlayManager\.broadcastToOverlayWindows\('subtitle:set'/g) ?? []).length,
1,
);
assert.equal(
(
emitBlock.match(
/annotationSubtitleWsService\.broadcast\(timedPayload, frequencyOptions\)/g,
) ?? []
).length,
1,
);
});
test('websocket frequency options callbacks each read one configuration snapshot', () => {
const source = readMainSource();
const subtitleBlock = source.match(
+14 -2
View File
@@ -326,7 +326,13 @@ test('handleOverlayModalClosed hides modal window only after all pending modals
});
runtime.sendToActiveOverlayWindow(
'subsync:open-manual',
{ ffsubsyncAvailable: true, sourceTracks: [] },
{
ffsubsyncAvailable: true,
videoReferenceAvailable: true,
subtitleTracks: [],
defaultReferenceTrackId: null,
defaultTargetTrackId: null,
},
{
restoreOnModalClose: 'subsync',
},
@@ -560,7 +566,13 @@ test('modal runtime notifies callers when modal input state becomes active/inact
});
runtime.sendToActiveOverlayWindow(
'subsync:open-manual',
{ ffsubsyncAvailable: true, sourceTracks: [] },
{
ffsubsyncAvailable: true,
videoReferenceAvailable: true,
subtitleTracks: [],
defaultReferenceTrackId: null,
defaultTargetTrackId: null,
},
{
restoreOnModalClose: 'subsync',
},
@@ -1,5 +1,7 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { createSubtitleProcessingController } from '../../core/services/subtitle-processing-controller';
import type { SubtitleData } from '../../types';
import {
createAutoplaySubtitlePrimingRuntime,
setMpvCurrentSecondarySubText,
@@ -42,8 +44,9 @@ test('scheduleSubtitlePrefetchRefresh logs refresh failures from timer callback'
setActiveParsedSubtitleMediaPath: () => {},
subtitleProcessingController: {
consumeCachedSubtitle: () => null,
onSubtitleChange: () => {},
refreshCurrentSubtitle: () => {},
onSubtitleChange: () => true,
refreshCurrentSubtitle: () => true,
notePlainSubtitleEmitted: () => {},
},
emitSubtitlePayload: () => {},
getSubtitlePrefetchService: () => null,
@@ -93,13 +96,25 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su
setActiveParsedSubtitleMediaPath: () => {},
subtitleProcessingController: {
consumeCachedSubtitle: () => null,
onSubtitleChange: (text) => calls.push(`change:${text}`),
refreshCurrentSubtitle: (text) => calls.push(`refresh:${text ?? ''}`),
onSubtitleChange: (text) => {
calls.push(`change:${text}`);
return true;
},
refreshCurrentSubtitle: (text) => {
calls.push(`refresh:${text ?? ''}`);
return true;
},
notePlainSubtitleEmitted: () => {},
},
emitSubtitlePayload: (payload) => calls.push(`emit:${payload.text}`),
emitSubtitlePayload: (payload, options) =>
calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`),
getSubtitlePrefetchService: () => ({
pause: () => calls.push('prefetch:pause'),
onSeek: (timePos) => calls.push(`prefetch:seek:${timePos}`),
pause: () => {
calls.push('prefetch:pause');
},
resume: () => {
calls.push('prefetch:resume');
},
}),
getLastObservedTimePos: () => 12,
getVisibleOverlayVisible: () => true,
@@ -120,8 +135,10 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su
'request:time-pos',
'set:起動字幕',
'prefetch:pause',
'emit:起動字幕',
'change:起動字幕',
'emit:起動字幕:resume=false',
// Uncached priming refreshes rather than announcing a change, so an
// invalidated-but-unchanged line is still re-tokenized.
'refresh:起動字幕',
]);
});
@@ -151,13 +168,25 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before
setActiveParsedSubtitleMediaPath: () => {},
subtitleProcessingController: {
consumeCachedSubtitle: () => null,
onSubtitleChange: (text) => calls.push(`change:${text}`),
refreshCurrentSubtitle: (text) => calls.push(`refresh:${text ?? ''}`),
onSubtitleChange: (text) => {
calls.push(`change:${text}`);
return true;
},
refreshCurrentSubtitle: (text) => {
calls.push(`refresh:${text ?? ''}`);
return true;
},
notePlainSubtitleEmitted: () => {},
},
emitSubtitlePayload: (payload) => calls.push(`emit:${payload.text}`),
emitSubtitlePayload: (payload, options) =>
calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`),
getSubtitlePrefetchService: () => ({
pause: () => calls.push('prefetch:pause'),
onSeek: (timePos) => calls.push(`prefetch:seek:${timePos}`),
pause: () => {
calls.push('prefetch:pause');
},
resume: () => {
calls.push('prefetch:resume');
},
}),
getLastObservedTimePos: () => 12,
getVisibleOverlayVisible: () => true,
@@ -175,7 +204,228 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before
'request:sub-text',
'set:起動字幕',
'prefetch:pause',
'emit:起動字幕',
'change:起動字幕',
'emit:起動字幕:resume=false',
// Uncached priming refreshes rather than announcing a change, so an
// invalidated-but-unchanged line is still re-tokenized.
'refresh:起動字幕',
]);
});
// Driven by the real processing controller rather than a stub: the failure this
// covers is a disagreement between the priming path and the controller's own
// staleness rules, which a hand-written stub cannot reproduce.
function createPrimingRuntimeWithRealController(options: {
text: string;
calls: string[];
onTokenize: () => void;
tokenize?: (text: string) => SubtitleData | null | Promise<SubtitleData | null>;
cacheLimit?: number;
}) {
const { text, calls } = options;
let currentSubText = '';
let currentSubtitleData: SubtitleData | null = null;
const mediaPath = '/media/video.mkv';
const prefetchService = {
pause: () => calls.push('prefetch:pause'),
resume: () => calls.push('prefetch:resume'),
};
// Mirrors main.ts emitSubtitlePayload: an emit resumes prefetching unless it
// is explicitly marked as not the end of the work for the line, and every
// controller emit is so marked.
const emitSubtitlePayload = (
payload: SubtitleData,
emitOptions?: { resumePrefetch?: boolean },
): void => {
currentSubtitleData = payload;
calls.push(
emitOptions?.resumePrefetch === false
? `emit-raw:${payload.text}`
: `emit-direct:${payload.text}`,
);
if (emitOptions?.resumePrefetch !== false) {
prefetchService.resume();
}
};
const subtitleProcessingController = createSubtitleProcessingController({
tokenizeSubtitle: async (subtitleText) => {
options.onTokenize();
return options.tokenize ? options.tokenize(subtitleText) : { text: subtitleText, tokens: [] };
},
// main.ts routes controller emits through emitSubtitlePayload with
// resumePrefetch: false, so they never release the pause on their own.
emitSubtitle: (payload) => {
currentSubtitleData = payload;
calls.push(`emit:${payload.text}:tokens=${payload.tokens === null ? 'none' : 'yes'}`);
},
onProcessingSettled: () => {
prefetchService.resume();
},
...(options.cacheLimit === undefined ? {} : { cacheLimit: options.cacheLimit }),
});
const runtime = createAutoplaySubtitlePrimingRuntime({
getCurrentMediaPath: () => mediaPath,
getMpvClient: () => ({
connected: true,
currentVideoPath: mediaPath,
requestProperty: async (name) => (name === 'sub-text' ? text : null),
}),
setCurrentSubText: (value) => {
currentSubText = value;
},
getCurrentSubText: () => currentSubText,
getCurrentSubtitleData: () => currentSubtitleData,
getActiveParsedSubtitleCues: () => [],
setActiveParsedSubtitleMediaPath: () => {},
subtitleProcessingController,
emitSubtitlePayload,
getSubtitlePrefetchService: () => prefetchService,
getLastObservedTimePos: () => 12,
getVisibleOverlayVisible: () => true,
emitSecondarySubtitle: () => {},
initSubtitlePrefetch: async () => {},
refreshSubtitlePrefetchFromActiveTrack: async () => {},
logDebug: () => {},
});
return { runtime, subtitleProcessingController, mediaPath };
}
test('primeCurrentSubtitleForAutoplay re-tokenizes text whose cached annotation was invalidated', async () => {
const calls: string[] = [];
let tokenizations = 0;
const text = '起動字幕';
const { runtime, subtitleProcessingController, mediaPath } =
createPrimingRuntimeWithRealController({
text,
calls,
onTokenize: () => {
tokenizations += 1;
},
});
// The line was already tokenized and cached during normal playback.
subtitleProcessingController.onSubtitleChange(text);
await new Promise((resolve) => setTimeout(resolve, 0));
const tokenizationsBeforeInvalidation = tokenizations;
// Mining a card drops every cached tokenization.
subtitleProcessingController.invalidateTokenizationCache();
calls.length = 0;
await runtime.primeCurrentSubtitleForAutoplay(mediaPath);
await new Promise((resolve) => setTimeout(resolve, 0));
// The cache miss must schedule fresh work, or the line stays unannotated for
// as long as it is on screen.
assert.equal(
tokenizations,
tokenizationsBeforeInvalidation + 1,
'expected the invalidated subtitle to be tokenized again',
);
assert.ok(
calls.includes(`emit:${text}:tokens=yes`),
`expected an annotated emit, saw ${JSON.stringify(calls)}`,
);
});
test('primeCurrentSubtitleForAutoplay releases the prefetch pause when nothing is scheduled', async () => {
const calls: string[] = [];
const text = '起動字幕';
const { runtime, subtitleProcessingController, mediaPath } =
createPrimingRuntimeWithRealController({
text,
calls,
onTokenize: () => {},
cacheLimit: 1,
});
// Emitted at the current cache generation, then evicted from the one-entry
// cache: priming misses the cache but the controller has nothing to redo, so
// no emit is coming and the pause must be released here.
subtitleProcessingController.onSubtitleChange(text);
await new Promise((resolve) => setTimeout(resolve, 0));
subtitleProcessingController.preCacheTokenization('別の字幕', {
text: '別の字幕',
tokens: [],
});
calls.length = 0;
await runtime.primeCurrentSubtitleForAutoplay(mediaPath);
await new Promise((resolve) => setTimeout(resolve, 0));
assert.deepEqual(calls, ['prefetch:pause', `emit-raw:${text}`, 'prefetch:resume']);
});
test('primeCurrentSubtitleForAutoplay releases the prefetch pause when tokenization emits nothing', async () => {
const calls: string[] = [];
const text = '起動字幕';
const { runtime, subtitleProcessingController, mediaPath } =
createPrimingRuntimeWithRealController({
text,
calls,
onTokenize: () => {},
// Transient tokenizer failure: the controller falls back to plain text it
// has already shown, so it suppresses the emit entirely.
tokenize: () => null,
});
subtitleProcessingController.onSubtitleChange(text);
await new Promise((resolve) => setTimeout(resolve, 0));
subtitleProcessingController.invalidateTokenizationCache();
calls.length = 0;
await runtime.primeCurrentSubtitleForAutoplay(mediaPath);
await new Promise((resolve) => setTimeout(resolve, 0));
assert.ok(
!calls.some((call) => call.startsWith('emit:')),
`expected no controller emit, saw ${JSON.stringify(calls)}`,
);
assert.equal(
calls.filter((call) => call === 'prefetch:resume').length,
1,
`expected the prefetch pause to be released, saw ${JSON.stringify(calls)}`,
);
});
test('prefetch stays paused until tokenization of an uncached line completes', async () => {
const calls: string[] = [];
const text = '起動字幕';
let finishTokenization = (): void => {};
const tokenizationGate = new Promise<void>((resolve) => {
finishTokenization = resolve;
});
const { runtime, mediaPath } = createPrimingRuntimeWithRealController({
text,
calls,
onTokenize: () => {},
tokenize: async (subtitleText) => {
await tokenizationGate;
return { text: subtitleText, tokens: [] };
},
});
// Driven through the priming path, which is what takes the pause out.
await runtime.primeCurrentSubtitleForAutoplay(mediaPath);
await new Promise((resolve) => setTimeout(resolve, 0));
// Neither the priming emit nor the controller's provisional plain emit may
// release the pause: the expensive scan is still ahead of them and would
// compete with prefetching for the parser.
// One plain payload, not two: priming paints it and tells the controller, so
// the controller goes straight for the tokenized one. And it does not resume
// prefetch, because the expensive scan is still ahead of it.
assert.deepEqual(calls, ['prefetch:pause', `emit-raw:${text}`]);
finishTokenization();
await new Promise((resolve) => setTimeout(resolve, 0));
assert.deepEqual(calls, [
'prefetch:pause',
`emit-raw:${text}`,
`emit:${text}:tokens=yes`,
'prefetch:resume',
]);
});
@@ -17,7 +17,7 @@ type AutoplaySubtitlePrimingMpvClient = {
type AutoplaySubtitlePrimingPrefetchService = {
pause: () => void;
onSeek: (timePos: number) => void;
resume: () => void;
};
export interface AutoplaySubtitlePrimingRuntimeDeps {
@@ -30,10 +30,12 @@ export interface AutoplaySubtitlePrimingRuntimeDeps {
setActiveParsedSubtitleMediaPath: (mediaPath: string | null) => void;
subtitleProcessingController: {
consumeCachedSubtitle: (text: string) => SubtitleData | null;
onSubtitleChange: (text: string) => void;
refreshCurrentSubtitle: (text: string) => void;
// Both report whether processing is pending; see pausePrefetchUntilProcessed.
onSubtitleChange: (text: string) => boolean;
refreshCurrentSubtitle: (text: string) => boolean;
notePlainSubtitleEmitted: (text: string) => void;
};
emitSubtitlePayload: (payload: SubtitleData) => void;
emitSubtitlePayload: (payload: SubtitleData, options?: { resumePrefetch?: boolean }) => void;
getSubtitlePrefetchService: () => AutoplaySubtitlePrimingPrefetchService | null;
getLastObservedTimePos: () => number;
getVisibleOverlayVisible: () => boolean;
@@ -64,6 +66,19 @@ export function setMpvCurrentSecondarySubText(
export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimingRuntimeDeps) {
const { subtitleProcessingController, emitSubtitlePayload } = deps;
// Prefetching is paused so the on-screen line gets the parser to itself; the
// resume rides on the controller settling (see onProcessingSettled), not on
// an emit, which a suppressed duplicate or a failed tokenization never sends.
// When the controller reports it has nothing scheduled, no settle is coming
// either, so release the pause here or prefetching idles indefinitely.
function pausePrefetchUntilProcessed(scheduleTokenization: () => boolean): void {
const prefetch = deps.getSubtitlePrefetchService();
prefetch?.pause();
if (!scheduleTokenization()) {
prefetch?.resume();
}
}
let subtitlePrefetchRefreshTimer: ReturnType<typeof setTimeout> | null = null;
let autoplaySubtitlePrimedMediaPath: string | null = null;
let visibleOverlaySubtitleRefreshAfterFirstPaintTimer: ReturnType<typeof setTimeout> | null =
@@ -104,12 +119,25 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
const cachedPayload = subtitleProcessingController.consumeCachedSubtitle(text);
if (cachedPayload) {
subtitleProcessingController.onSubtitleChange(text);
// This emit resumes prefetching, so no pause is left outstanding.
emitSubtitlePayload(cachedPayload);
return true;
}
emitSubtitlePayload({ text, tokens: null });
subtitleProcessingController.onSubtitleChange(text);
// Provisional raw emit: keep prefetch paused until the processing
// controller is done with this line, and tell it this line has already been
// painted plain so it does not broadcast the same payload again.
emitSubtitlePayload({ text, tokens: null }, { resumePrefetch: false });
subtitleProcessingController.notePlainSubtitleEmitted(text);
// refreshCurrentSubtitle, not onSubtitleChange: the cache miss above can be
// an invalidation (mining a card) on text the controller still holds, and
// onSubtitleChange treats unchanged text as nothing to do, which would
// leave this line permanently unannotated. refreshCurrentSubtitle also
// re-tokenizes for a new cache generation.
if (!subtitleProcessingController.refreshCurrentSubtitle(text)) {
// Nothing scheduled, so no settle is coming to release the pause.
deps.getSubtitlePrefetchService()?.resume();
}
return true;
}
@@ -153,14 +181,12 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
getCurrentSubtitleData: () => deps.getCurrentSubtitleData(),
consumeCachedSubtitle: (text) => subtitleProcessingController.consumeCachedSubtitle(text),
onSubtitleChange: (text) => {
deps.getSubtitlePrefetchService()?.pause();
deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
subtitleProcessingController.onSubtitleChange(text);
pausePrefetchUntilProcessed(() => subtitleProcessingController.onSubtitleChange(text));
},
refreshCurrentSubtitle: (text) => {
deps.getSubtitlePrefetchService()?.pause();
deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
subtitleProcessingController.refreshCurrentSubtitle(text);
pausePrefetchUntilProcessed(() =>
subtitleProcessingController.refreshCurrentSubtitle(text),
);
},
deferUncachedRefresh: true,
emitSubtitle: (payload) => emitSubtitlePayload(payload),
@@ -204,9 +230,7 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
if (!text.trim()) {
return;
}
deps.getSubtitlePrefetchService()?.pause();
deps.getSubtitlePrefetchService()?.onSeek(deps.getLastObservedTimePos());
subtitleProcessingController.refreshCurrentSubtitle(text);
pausePrefetchUntilProcessed(() => subtitleProcessingController.refreshCurrentSubtitle(text));
}, VISIBLE_OVERLAY_SUBTITLE_REFRESH_AFTER_FIRST_PAINT_DELAY_MS);
visibleOverlaySubtitleRefreshAfterFirstPaintTimer.unref?.();
}
+48
View File
@@ -0,0 +1,48 @@
import type { OverlayHostedModal } from '../../shared/ipc/contracts';
import { IPC_CHANNELS } from '../../shared/ipc/contracts';
import { openOverlayHostedModal, retryOverlayModalOpen } from './overlay-hosted-modal-open';
const CHANGELOG_MODAL: OverlayHostedModal = 'changelog';
const CHANGELOG_OPEN_TIMEOUT_MS = 1500;
export async function openChangelogModal(deps: {
ensureOverlayStartupPrereqs: () => void;
ensureOverlayWindowsReadyForVisibilityActions: () => void;
sendToActiveOverlayWindow: (
channel: string,
payload?: unknown,
runtimeOptions?: {
restoreOnModalClose?: OverlayHostedModal;
preferModalWindow?: boolean;
},
) => boolean;
waitForModalOpen: (modal: OverlayHostedModal, timeoutMs: number) => Promise<boolean>;
logWarn: (message: string) => void;
}): Promise<boolean> {
return await retryOverlayModalOpen(
{
waitForModalOpen: deps.waitForModalOpen,
logWarn: deps.logWarn,
},
{
modal: CHANGELOG_MODAL,
timeoutMs: CHANGELOG_OPEN_TIMEOUT_MS,
retryWarning:
'Changelog modal did not acknowledge modal open on first attempt; retrying dedicated modal window.',
sendOpen: () =>
openOverlayHostedModal(
{
ensureOverlayStartupPrereqs: deps.ensureOverlayStartupPrereqs,
ensureOverlayWindowsReadyForVisibilityActions:
deps.ensureOverlayWindowsReadyForVisibilityActions,
sendToActiveOverlayWindow: deps.sendToActiveOverlayWindow,
},
{
channel: IPC_CHANNELS.event.changelogOpen,
modal: CHANGELOG_MODAL,
preferModalWindow: true,
},
),
},
);
}
@@ -0,0 +1,55 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { readBundledChangelog, resolveBundledChangelogPath } from './bundled-changelog';
test('bundled changelog path prefers the packaged resources copy', () => {
const resolved = resolveBundledChangelogPath({
resourcesPath: '/res',
appPath: '/app',
dirname: '/app/dist/main',
joinPath: (...parts) => parts.join('/'),
fileExists: (candidate) => candidate === '/res/CHANGELOG.md',
});
assert.equal(resolved, '/res/CHANGELOG.md');
});
test('bundled changelog path falls back to the repo root during development', () => {
const resolved = resolveBundledChangelogPath({
resourcesPath: '/res',
appPath: '/app',
dirname: '/repo/dist/main',
joinPath: (...parts) => parts.join('/'),
fileExists: (candidate) => candidate === '/repo/dist/main/../../CHANGELOG.md',
});
assert.equal(resolved, '/repo/dist/main/../../CHANGELOG.md');
});
test('bundled changelog returns null when no copy is installed', () => {
const result = readBundledChangelog({
resolvePath: () => null,
readFile: () => {
throw new Error('should not read');
},
logWarn: () => {},
});
assert.equal(result, null);
});
test('bundled changelog logs and returns null when the file cannot be read', () => {
const warnings: string[] = [];
const result = readBundledChangelog({
resolvePath: () => '/res/CHANGELOG.md',
readFile: () => {
throw new Error('EACCES');
},
logWarn: (message) => warnings.push(message),
});
assert.equal(result, null);
assert.equal(warnings.length, 1);
assert.match(warnings[0] ?? '', /EACCES/);
});
@@ -0,0 +1,35 @@
export function resolveBundledChangelogPath(deps: {
resourcesPath: string;
appPath: string;
dirname: string;
joinPath: (...parts: string[]) => string;
fileExists: (path: string) => boolean;
}): string | null {
const candidates = [
deps.joinPath(deps.resourcesPath, 'CHANGELOG.md'),
deps.joinPath(deps.appPath, 'CHANGELOG.md'),
deps.joinPath(deps.dirname, '..', 'CHANGELOG.md'),
deps.joinPath(deps.dirname, '..', '..', 'CHANGELOG.md'),
];
return candidates.find((candidate) => deps.fileExists(candidate)) ?? null;
}
export function readBundledChangelog(deps: {
resolvePath: () => string | null;
readFile: (path: string) => string;
logWarn: (message: string) => void;
}): string | null {
const changelogPath = deps.resolvePath();
if (!changelogPath) return null;
try {
return deps.readFile(changelogPath);
} catch (error) {
deps.logWarn(
`Failed to read bundled changelog at ${changelogPath}: ${
error instanceof Error ? error.message : String(error)
}`,
);
return null;
}
}
@@ -0,0 +1,122 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
CHANGELOG_REQUEST_TIMEOUT_MS,
createChangelogRuntime,
withRequestTimeout,
} from './changelog-runtime';
import type { FetchLike, FetchResponseLike } from '../update/release-assets';
import { createCurlFetch } from '../update/fetch-adapter';
function okResponse(body: string): FetchResponseLike {
return {
ok: true,
status: 200,
json: async () => JSON.parse(body),
text: async () => body,
arrayBuffer: async () => new ArrayBuffer(0),
};
}
test('request timeout wrapper attaches an abort signal to every request', async () => {
const seen: Array<{ url: string; init?: Record<string, unknown> }> = [];
const wrapped = withRequestTimeout(async (url, init) => {
seen.push({ url, init });
return okResponse('body');
}, 1_234);
await wrapped('https://example.test/a');
await wrapped('https://example.test/b', { headers: { 'User-Agent': 'SubMiner' } });
assert.equal(seen.length, 2);
for (const request of seen) {
assert.ok(request.init?.signal instanceof AbortSignal, 'each request carries a signal');
assert.equal((request.init?.signal as AbortSignal).aborted, false);
}
// Existing init is preserved rather than replaced.
assert.deepEqual(seen[1]?.init?.headers, { 'User-Agent': 'SubMiner' });
});
test('request timeout wrapper aborts a request that never settles', async () => {
let observed: AbortSignal | undefined;
const wrapped = withRequestTimeout((_url, init) => {
observed = init?.signal as AbortSignal;
return new Promise<FetchResponseLike>(() => {
// Never resolves, standing in for a stalled connection.
});
}, 10);
void wrapped('https://example.test/stalled');
await new Promise((resolve) => setTimeout(resolve, 40));
assert.equal(observed?.aborted, true);
});
test('changelog runtime times out both requests instead of hanging the modal', async () => {
const signals: AbortSignal[] = [];
// Stays pending until the deadline fires, standing in for a stalled server.
const stalling: FetchLike = (_url, init) => {
const signal = init?.signal as AbortSignal;
signals.push(signal);
return new Promise((_resolve, reject) => {
signal.addEventListener('abort', () => reject(signal.reason), { once: true });
});
};
const runtime = createChangelogRuntime({
getInstalledVersion: () => '0.19.2',
getUpdateChannel: () => 'stable',
resourcesPath: '/res',
appPath: '/app',
dirname: '/app/dist/main',
joinPath: (...parts) => parts.join('/'),
fileExists: () => false,
readFile: () => '',
logWarn: () => {},
createFetch: () => withRequestTimeout(stalling, 10),
});
const snapshot = await runtime.getChangelogSnapshot();
// Release lookup and changelog download both hit the deadline rather than hang.
assert.equal(signals.length, 2);
assert.ok(
signals.every((signal) => signal.aborted),
'every stalled request was aborted',
);
// No bundled copy is readable here, so the timeout surfaces as an error state
// that does not claim a bundled changelog is on screen.
assert.match(snapshot.error ?? '', /^Changelog unavailable: /);
assert.match(snapshot.error ?? '', /timed out|timeout|abort/i);
assert.deepEqual(snapshot.entries, []);
});
test('changelog timeout reaches the curl transport, not just global fetch', async () => {
// The POSIX transport is curl, whose own --max-time is 60s; the changelog
// deadline is shorter, so the signal has to actually terminate the process.
let killed: string | undefined;
const curlFetch = createCurlFetch({
execFile: ((
_file: string,
_args: readonly string[],
_options: unknown,
_callback: unknown,
) => ({
kill: (signal?: string) => {
killed = signal;
return true;
},
})) as never,
});
const wrapped = withRequestTimeout(curlFetch, 10);
await assert.rejects(wrapped('https://example.test/stalled'));
assert.equal(killed, 'SIGKILL', 'the stalled curl process is killed at the changelog deadline');
});
test('changelog request timeout is finite', () => {
assert.ok(Number.isFinite(CHANGELOG_REQUEST_TIMEOUT_MS));
assert.ok(CHANGELOG_REQUEST_TIMEOUT_MS > 0);
});
@@ -0,0 +1,85 @@
import type { ChangelogSnapshot } from '../../../types/changelog';
import type { UpdateChannel } from '../../../types/config';
import { createCurlFetch, createGlobalFetch } from '../update/fetch-adapter';
import { fetchLatestStableRelease, type FetchLike } from '../update/release-assets';
import { readBundledChangelog, resolveBundledChangelogPath } from './bundled-changelog';
import { createChangelogSource } from './changelog-source';
export interface ChangelogRuntimeDeps {
getInstalledVersion: () => string;
getUpdateChannel: () => UpdateChannel;
resourcesPath: string;
appPath: string;
dirname: string;
joinPath: (...parts: string[]) => string;
fileExists: (path: string) => boolean;
readFile: (path: string) => string;
logWarn: (message: string) => void;
/** Injected in tests; production picks curl on POSIX and global fetch on Windows. */
createFetch?: () => FetchLike;
}
/**
* curl enforces its own `--max-time`, but the global-fetch transport has no
* deadline: without this a stalled connection leaves the modal on "Loading
* changelog..." with no way back except closing it.
*/
export const CHANGELOG_REQUEST_TIMEOUT_MS = 30_000;
export function withRequestTimeout(fetchImpl: FetchLike, timeoutMs: number): FetchLike {
return (url, init) => {
if (typeof AbortSignal?.timeout !== 'function') return fetchImpl(url, init);
return fetchImpl(url, { ...init, signal: AbortSignal.timeout(timeoutMs) });
};
}
export function createChangelogRuntime(deps: ChangelogRuntimeDeps): {
getChangelogSnapshot: (options?: { refresh?: boolean }) => Promise<ChangelogSnapshot>;
} {
// curl matches the updater's transport choice: Electron's global fetch is
// unreliable for GitHub on some Linux builds.
const fetchImpl = withRequestTimeout(
deps.createFetch?.() ??
(process.platform === 'win32' ? createGlobalFetch() : createCurlFetch()),
CHANGELOG_REQUEST_TIMEOUT_MS,
);
const source = createChangelogSource({
fetchLatestReleaseTag: async () => {
const release = await fetchLatestStableRelease({
fetch: fetchImpl,
channel: deps.getUpdateChannel(),
});
return release?.tag_name ?? null;
},
fetchText: async (url) => {
const response = await fetchImpl(url, {
headers: { 'User-Agent': 'SubMiner changelog' },
});
if (!response.ok) {
throw new Error(`Changelog request failed with ${response.status}`);
}
return await response.text();
},
readBundledChangelog: () =>
readBundledChangelog({
resolvePath: () =>
resolveBundledChangelogPath({
resourcesPath: deps.resourcesPath,
appPath: deps.appPath,
dirname: deps.dirname,
joinPath: deps.joinPath,
fileExists: deps.fileExists,
}),
readFile: deps.readFile,
logWarn: deps.logWarn,
}),
getInstalledVersion: deps.getInstalledVersion,
now: () => Date.now(),
logWarn: deps.logWarn,
});
return {
getChangelogSnapshot: (options?: { refresh?: boolean }) => source.getSnapshot(options),
};
}
@@ -0,0 +1,45 @@
import type { ChangelogSnapshot, ChangelogSourceKind } from '../../../types/changelog';
import { parseChangelog } from '../../../core/utils/changelog-parse';
import { compareSemverLike } from '../update/release-assets';
export function buildChangelogSnapshot(
markdown: string,
options: {
installedVersion: string;
source: ChangelogSourceKind;
releaseTag?: string;
warning?: string;
},
): ChangelogSnapshot {
const entries = parseChangelog(markdown);
const latest = entries.reduce<string | null>(
(best, entry) =>
best === null || compareSemverLike(entry.version, best) > 0 ? entry.version : best,
null,
);
const latestEntry = entries.find((entry) => entry.version === latest) ?? entries[0] ?? null;
return {
entries,
installedVersion: options.installedVersion,
latestVersion: latest,
expandedGroupKey: latestEntry?.groupKey ?? null,
source: options.source,
...(options.releaseTag ? { releaseTag: options.releaseTag } : {}),
...(options.warning ? { warning: options.warning } : {}),
};
}
export function buildEmptyChangelogSnapshot(options: {
installedVersion: string;
error: string;
}): ChangelogSnapshot {
return {
entries: [],
installedVersion: options.installedVersion,
latestVersion: null,
expandedGroupKey: null,
source: 'bundled',
error: options.error,
};
}
@@ -0,0 +1,204 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { buildRawChangelogUrl, createChangelogSource } from './changelog-source';
import { buildChangelogSnapshot } from './changelog-snapshot';
const REMOTE = `# Changelog
## v0.20.0 (2026-09-01)
### Added
- Remote only entry.
## v0.19.2 (2026-08-04)
### Fixed
- Installed entry.
`;
const BUNDLED = `# Changelog
## v0.19.2 (2026-08-04)
### Fixed
- Installed entry.
`;
function createDeps(overrides: Partial<Parameters<typeof createChangelogSource>[0]> = {}) {
return {
fetchLatestReleaseTag: async () => 'v0.20.0',
fetchText: async () => REMOTE,
readBundledChangelog: () => BUNDLED,
getInstalledVersion: () => '0.19.2',
now: () => 1_000,
logWarn: () => {},
...overrides,
};
}
test('changelog source reads the changelog at the latest release tag', async () => {
const urls: string[] = [];
const source = createChangelogSource(
createDeps({
fetchText: async (url: string) => {
urls.push(url);
return REMOTE;
},
}),
);
const snapshot = await source.getSnapshot();
assert.deepEqual(urls, [
'https://raw.githubusercontent.com/ksyasuda/SubMiner/v0.20.0/CHANGELOG.md',
]);
assert.equal(snapshot.source, 'remote');
assert.equal(snapshot.releaseTag, 'v0.20.0');
assert.equal(snapshot.latestVersion, '0.20.0');
assert.equal(snapshot.installedVersion, '0.19.2');
assert.equal(snapshot.expandedGroupKey, '0.20');
});
test('changelog source falls back to the default branch when no release tag resolves', async () => {
const urls: string[] = [];
const source = createChangelogSource(
createDeps({
fetchLatestReleaseTag: async () => null,
fetchText: async (url: string) => {
urls.push(url);
return REMOTE;
},
}),
);
const snapshot = await source.getSnapshot();
assert.deepEqual(urls, ['https://raw.githubusercontent.com/ksyasuda/SubMiner/main/CHANGELOG.md']);
assert.equal(snapshot.source, 'remote');
assert.equal(snapshot.releaseTag, undefined);
});
test('changelog source falls back to the bundled changelog when the download fails', async () => {
const warnings: string[] = [];
const source = createChangelogSource(
createDeps({
fetchText: async () => {
throw new Error('offline');
},
logWarn: (message: string) => warnings.push(message),
}),
);
const snapshot = await source.getSnapshot();
assert.equal(snapshot.source, 'bundled');
assert.match(snapshot.warning ?? '', /offline/);
assert.equal(snapshot.latestVersion, '0.19.2');
assert.equal(warnings.length, 1);
});
test('changelog source reports an error when no changelog can be loaded', async () => {
const source = createChangelogSource(
createDeps({
fetchText: async () => {
throw new Error('offline');
},
readBundledChangelog: () => null,
}),
);
const snapshot = await source.getSnapshot();
assert.deepEqual(snapshot.entries, []);
assert.match(snapshot.error ?? '', /offline/);
// Nothing is rendered, so the message must not promise a bundled changelog.
assert.match(snapshot.error ?? '', /^Changelog unavailable: /);
assert.doesNotMatch(snapshot.error ?? '', /Showing the bundled changelog/);
assert.equal(snapshot.installedVersion, '0.19.2');
});
test('changelog source caches remote results and refreshes on demand', async () => {
let fetches = 0;
let clock = 0;
const source = createChangelogSource(
createDeps({
now: () => clock,
fetchText: async () => {
fetches += 1;
return REMOTE;
},
}),
);
await source.getSnapshot();
await source.getSnapshot();
assert.equal(fetches, 1);
await source.getSnapshot({ refresh: true });
assert.equal(fetches, 2);
clock = 11 * 60 * 1000;
await source.getSnapshot();
assert.equal(fetches, 3);
});
test('changelog source retries the network after a bundled fallback', async () => {
let fetches = 0;
const source = createChangelogSource(
createDeps({
fetchText: async () => {
fetches += 1;
throw new Error('offline');
},
}),
);
await source.getSnapshot();
await source.getSnapshot();
assert.equal(fetches, 2);
});
test('changelog source treats an empty remote changelog as a failure', async () => {
const source = createChangelogSource(createDeps({ fetchText: async () => ' ' }));
const snapshot = await source.getSnapshot();
assert.equal(snapshot.source, 'bundled');
});
test('changelog source falls back when the remote body parses to no releases', () => {
const warnings: string[] = [];
const source = createChangelogSource(
createDeps({
// A 200 that is not a changelog, e.g. a redirect landing page.
fetchText: async () => '<!doctype html><html><body>Moved</body></html>',
logWarn: (message: string) => warnings.push(message),
}),
);
return source.getSnapshot().then((snapshot) => {
assert.equal(snapshot.source, 'bundled');
assert.equal(snapshot.entries.length, 1);
assert.match(snapshot.warning ?? '', /no releases/);
assert.equal(warnings.length, 1);
});
});
test('raw changelog urls encode the release ref', () => {
assert.equal(
buildRawChangelogUrl('v1.0.0', 'owner', 'repo'),
'https://raw.githubusercontent.com/owner/repo/v1.0.0/CHANGELOG.md',
);
});
test('snapshot expansion uses the newest version even when file order is unsorted', () => {
const snapshot = buildChangelogSnapshot(
'## v0.18.0 (2026-01-01)\n\n### Fixed\n- Old.\n\n## v0.19.0 (2026-02-01)\n\n### Fixed\n- New.\n',
{ installedVersion: '0.18.0', source: 'bundled' },
);
assert.equal(snapshot.latestVersion, '0.19.0');
assert.equal(snapshot.expandedGroupKey, '0.19');
});
@@ -0,0 +1,117 @@
import type { ChangelogSnapshot } from '../../../types/changelog';
import { buildChangelogSnapshot, buildEmptyChangelogSnapshot } from './changelog-snapshot';
const DEFAULT_OWNER = 'ksyasuda';
const DEFAULT_REPO = 'SubMiner';
const DEFAULT_CACHE_TTL_MS = 10 * 60 * 1000;
export interface ChangelogSourceDeps {
/** Resolves the release the changelog should be read from, or null when unknown. */
fetchLatestReleaseTag: () => Promise<string | null>;
fetchText: (url: string) => Promise<string>;
/** Reads the CHANGELOG.md shipped with the install; null when unavailable. */
readBundledChangelog: () => string | null;
getInstalledVersion: () => string;
now: () => number;
logWarn: (message: string) => void;
owner?: string;
repo?: string;
cacheTtlMs?: number;
}
export function buildRawChangelogUrl(ref: string, owner: string, repo: string): string {
return `https://raw.githubusercontent.com/${owner}/${repo}/${encodeURIComponent(ref)}/CHANGELOG.md`;
}
function summarize(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
export function createChangelogSource(deps: ChangelogSourceDeps): {
getSnapshot: (options?: { refresh?: boolean }) => Promise<ChangelogSnapshot>;
} {
const owner = deps.owner ?? DEFAULT_OWNER;
const repo = deps.repo ?? DEFAULT_REPO;
const cacheTtlMs = deps.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS;
let cached: { snapshot: ChangelogSnapshot; fetchedAt: number } | null = null;
let inFlight: Promise<ChangelogSnapshot> | null = null;
/** `reason` is the raw failure; each branch phrases it for what it can show. */
function fallbackToBundled(reason: string): ChangelogSnapshot {
const bundled = deps.readBundledChangelog();
if (bundled === null) {
// Nothing is on screen, so promising a bundled changelog would be a lie.
return buildEmptyChangelogSnapshot({
installedVersion: deps.getInstalledVersion(),
error: `Changelog unavailable: ${reason}`,
});
}
return buildChangelogSnapshot(bundled, {
installedVersion: deps.getInstalledVersion(),
source: 'bundled',
warning: `Showing the bundled changelog: ${reason}`,
});
}
async function loadSnapshot(): Promise<ChangelogSnapshot> {
let releaseTag: string | null = null;
try {
releaseTag = await deps.fetchLatestReleaseTag();
} catch (error) {
deps.logWarn(`Changelog release lookup failed: ${summarize(error)}`);
}
// Without a release tag the default branch still gives the newest published
// changelog, so try it before falling back to the bundled copy.
const ref = releaseTag ?? 'main';
try {
const markdown = await deps.fetchText(buildRawChangelogUrl(ref, owner, repo));
if (markdown.trim().length === 0) {
throw new Error('Remote changelog was empty.');
}
const snapshot = buildChangelogSnapshot(markdown, {
installedVersion: deps.getInstalledVersion(),
source: 'remote',
...(releaseTag ? { releaseTag } : {}),
});
// A 200 that isn't a changelog (a redirect landing page, a renamed repo)
// parses to nothing; the bundled copy beats showing an empty modal.
if (snapshot.entries.length === 0) {
throw new Error('Remote changelog contained no releases.');
}
return snapshot;
} catch (error) {
const message = summarize(error);
deps.logWarn(`Changelog download failed (${ref}): ${message}`);
return fallbackToBundled(message);
}
}
return {
async getSnapshot(options?: { refresh?: boolean }): Promise<ChangelogSnapshot> {
const refresh = options?.refresh === true;
if (!refresh && cached && deps.now() - cached.fetchedAt < cacheTtlMs) {
return cached.snapshot;
}
if (inFlight) return await inFlight;
inFlight = loadSnapshot()
.then((snapshot) => {
// Only a successful remote read is worth caching; a bundled fallback
// should retry the network on the next open.
if (snapshot.source === 'remote') {
cached = { snapshot, fetchedAt: deps.now() };
} else {
cached = null;
}
return snapshot;
})
.finally(() => {
inFlight = null;
});
return await inFlight;
},
};
}
@@ -53,3 +53,52 @@ test('character dictionary sync completion refreshes subtitle state when diction
'log:[dictionary:auto-sync] refreshed current subtitle after sync (AniList 1, changed=yes, title=Frieren)',
]);
});
test('character dictionary sync completion drops cached dictionary reads before refreshing', () => {
const calls: string[] = [];
handleCharacterDictionaryAutoSyncComplete(
{
mediaId: 1,
mediaTitle: 'Frieren',
changed: true,
},
{
hasParserWindow: () => true,
invalidateCharacterDictionaryLookups: () => calls.push('invalidate-dictionary-lookups'),
clearParserCaches: () => calls.push('clear-parser'),
invalidateTokenizationCache: () => calls.push('invalidate'),
refreshSubtitlePrefetch: () => calls.push('prefetch'),
refreshCurrentSubtitle: () => calls.push('refresh-subtitle'),
logInfo: () => {},
},
);
// Must run before the refreshes, or they re-tokenize against the character
// names and images from the previous dictionary build.
assert.equal(calls[0], 'invalidate-dictionary-lookups');
assert.ok(calls.indexOf('invalidate-dictionary-lookups') < calls.indexOf('refresh-subtitle'));
});
test('character dictionary sync completion leaves cached dictionary reads alone when unchanged', () => {
const calls: string[] = [];
handleCharacterDictionaryAutoSyncComplete(
{
mediaId: 1,
mediaTitle: 'Frieren',
changed: false,
},
{
hasParserWindow: () => true,
invalidateCharacterDictionaryLookups: () => calls.push('invalidate-dictionary-lookups'),
clearParserCaches: () => calls.push('clear-parser'),
invalidateTokenizationCache: () => calls.push('invalidate'),
refreshSubtitlePrefetch: () => calls.push('prefetch'),
refreshCurrentSubtitle: () => calls.push('refresh-subtitle'),
logInfo: () => {},
},
);
assert.deepEqual(calls, []);
});

Some files were not shown because too many files have changed in this diff Show More