Compare commits

..
Author SHA1 Message Date
sudacode 88b73656a7 chore(release): prepare v0.20.1-beta.2 2026-10-03 21:22:19 -07:00
sudacode e77aa5c3b3 fix(stats): count mined cards once and keep lexical stats exact after deletes (#280) 2026-10-03 21:16:12 -07:00
sudacode 121b688afb fix(deps): bump http-cache-semantics to 4.3.0 for GHSA-ch52-4w7c-c8xp (#283) 2026-10-03 21:13:32 -07:00
sudacode 727be5683a fix(youtube): keep mpv queue when advancing to a queued video on Windows (#281) 2026-10-02 22:13:35 -07:00
sudacode 138bfe9759 fix(subtitles): allow Whisper generation during Windows YouTube stream playback (#282) 2026-10-02 22:13:15 -07:00
sudacode 0aba17d95b chore(release): generate concise release notes per feature or area
- Rewrite polish prompt to default to inline items, nesting only for several distinct changes
- Add length/density example and drop internal details from release bodies
- Tell prerelease mode to shorten long baseline bullets
- Update changes/README.md and prompt assertions in tests
2026-10-02 19:52:16 -07:00
sudacode a5d0b63792 fix(storage): protect Yomitan profiles across Electron runtime changes (#211) 2026-10-01 21:44:53 -07:00
sudacode 15cb9fe918 test(launcher): allow Windows bootstrap integration startup time
Cold and cached launches each have a 15-second command timeout. Give the integration test time for both launches and executable copies instead of the default five-second budget on hosted Windows runners.
2026-10-01 01:42:10 -07:00
sudacode db45a70ec2 chore(release): prepare v0.20.1-beta.1 2026-10-01 01:21:59 -07:00
198 changed files with 2546 additions and 6216 deletions

No files matched your search

-2
View File
@@ -20,8 +20,6 @@ coverage/
/main-entry.js
/main-entry.js.map
/build/hachidori/
# Logs
*.log
npm-debug.log*
-4
View File
@@ -8,7 +8,3 @@
[submodule "vendor/subminer-yomitan"]
path = vendor/subminer-yomitan
url = https://github.com/ksyasuda/subminer-yomitan
[submodule "vendor/hachidori"]
path = vendor/hachidori
url = https://github.com/ksyasuda/hachidori.git
branch = subminer
+2 -5
View File
@@ -4,7 +4,7 @@
# SubMiner
Look up words with Yomitan or Hachidori, mine them to Anki, and track your immersion without leaving mpv
Look up words with Yomitan, mine them to Anki, and track your immersion without leaving mpv
[Installation](#quick-start) · [Requirements](#requirements) · [Usage](https://docs.subminer.moe/usage) · [Documentation](https://docs.subminer.moe)
@@ -25,8 +25,6 @@ Look up words with Yomitan or Hachidori, mine them to Anki, and track your immer
Hover over any word in the subtitles to open the full Yomitan popup with definitions, pitch accent, and frequency data. SubMiner bundles its own Yomitan, separate from any browser install.
Yomitan remains the default. Select the bundled Hachidori backend with `dictionaryBackend: "hachidori"` and restart SubMiner. The tray opens the selected backend's settings. See [dictionary setup](https://docs.subminer.moe/usage#hachidori-setup) for importing dictionaries, linking an external Hachidori host, and configuring Anki.
<div align="center">
<img src="docs-site/public/screenshots/yomitan-lookup.png" width="800" alt="Yomitan dictionary popup over annotated subtitles in mpv">
</div>
@@ -267,8 +265,7 @@ SubMiner builds on the work of these open-source projects:
| [jellyfin-mpv-shim](https://github.com/jellyfin/jellyfin-mpv-shim) | Jellyfin integration |
| [Jimaku.cc](https://jimaku.cc) | Japanese subtitle search and downloads |
| [Renji's Texthooker Page](https://github.com/Renji-XD/texthooker-ui) | Base for the WebSocket texthooker integration |
| [Yomitan](https://github.com/yomidevs/yomitan) | Default dictionary engine and morphological parser |
| [Hachidori](https://github.com/bee-san/hachidori) | Alternative dictionary backend, powered by HoshiDicts |
| [Yomitan](https://github.com/yomidevs/yomitan) | Dictionary engine powering all lookups and the morphological parser |
| [yomitan-jlpt-vocab](https://github.com/stephenmk/yomitan-jlpt-vocab) | JLPT level tags for vocabulary |
## License
+3 -3
View File
@@ -21,7 +21,7 @@
"@electron/asar": "3.4.1",
"@types/node": "^24.10.0",
"@types/ws": "^8.18.1",
"electron": "42.10.0",
"electron": "43.7.7",
"electron-builder": "26.16.1",
"esbuild": "^0.25.12",
"eslint": "^10.8.0",
@@ -347,7 +347,7 @@
"ejs": ["ejs@3.1.10", "", { "dependencies": { "jake": "^10.8.5" }, "bin": { "ejs": "bin/cli.js" } }, "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA=="],
"electron": ["electron@42.10.0", "", { "dependencies": { "@electron-internal/extract-zip": "^1.0.1", "@electron/get": "^5.0.0", "@types/node": "^24.9.0" }, "bin": { "electron": "cli.js", "install-electron": "install.js" } }, "sha512-yBWnZO8tKId0ADvGa4+imPmm3+8Kfs8MtkUPhQQJz1AjTnUUDwCTWkQEU0CrArVVL97Jo4Q/0mVcLHciicBDGQ=="],
"electron": ["electron@43.7.7", "", { "dependencies": { "@electron-internal/extract-zip": "^1.0.1", "@electron/get": "^5.0.0", "@types/node": "^24.9.0" }, "bin": { "electron": "cli.js", "install-electron": "install.js" } }, "sha512-y0pg+KhWLAkW2Ts2njkFFNFl3KtB1FmbUp9Te9i+R2Og2v1fcI/I6MADJ7jvnzB5UKdt+U+0EUoCnVL79xMPMA=="],
"electron-builder": ["electron-builder@26.16.1", "", { "dependencies": { "app-builder-lib": "26.16.1", "builder-util": "26.16.0", "builder-util-runtime": "9.7.0", "chalk": "^4.1.2", "ci-info": "^4.2.0", "dmg-builder": "26.16.1", "fs-extra": "^10.1.0", "lazy-val": "^1.0.5", "simple-update-notifier": "2.0.0", "yargs": "^17.6.2" }, "bin": { "electron-builder": "./cli.js", "install-app-deps": "./install-app-deps.js" } }, "sha512-LrLK65QX5PUYYODXqp23FKrV7CILTtVY7mrJckNknO9jLNSMiqFkKbSMiDRw4CjOADMPVDdWLxY4mezOZWswxg=="],
@@ -467,7 +467,7 @@
"hosted-git-info": ["hosted-git-info@4.1.0", "", { "dependencies": { "lru-cache": "^6.0.0" } }, "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA=="],
"http-cache-semantics": ["http-cache-semantics@4.2.0", "", {}, "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ=="],
"http-cache-semantics": ["http-cache-semantics@4.3.0", "", {}, "sha512-M5t5LlJpS1UHMjvwRQVdFHvPISGeLAxNcrWuJkeGh0KxsqCHZ1O3NXZU/8x7cD0BDcGW8kapxMKTvwlqrNkHkA=="],
"http-proxy-agent": ["http-proxy-agent@7.0.2", "", { "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" } }, "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig=="],
+1 -1
View File
@@ -42,7 +42,7 @@ How fragments turn into a release:
- At release time, `bun run changelog:build` (and `bun run changelog:prerelease-notes`) pipes every pending fragment through `claude -p` to merge related items, drop noise, and rewrite into a clean user-facing release body. Write fragments as raw, informative notes — don't worry about polished prose, deduping across PRs, or line-by-line phrasing. The polish step handles all of that.
- The polish step treats pending fragments as the final release outcome, not prerelease history. If a feature is added and then renamed or fixed before the stable cut, ship the final feature bullet instead of separate prerelease-only breaking/fix entries.
- `CHANGELOG.md`, GitHub release notes, and prerelease notes all use short top-level items with one nested bullet per distinct change, instead of packing a release's worth of detail into a single paragraph bullet. An item with only one thing to say stays inline on the top-level bullet. Release notes and prerelease notes additionally cover user benefit and any useful action note in their nested bullets.
- `CHANGELOG.md`, GitHub release notes, and prerelease notes all use one item per feature or area, inline by default: `- **Name**: one or two short sentences.` Items with several separate changes may use a few short nested bullets instead of a run-on sentence. Items say what changed and how to reach it; implementation details, edge cases, and step-by-step usage belong in the docs, not the release body.
- `internal` fragments stay in `CHANGELOG.md` (inside a collapsed `<details>` block) but are dropped from the GitHub release notes entirely.
- The polished `CHANGELOG.md` and `release/release-notes.md` are committed and reviewed before tagging — edit the Markdown by hand if Claude misses something.
+4
View File
@@ -0,0 +1,4 @@
type: internal
area: release
- Release notes, prerelease notes, and `CHANGELOG.md` are now generated as short items per feature or area, nesting only when an item has several separate changes.
@@ -0,0 +1,6 @@
type: fixed
area: dictionary
- Upgraded the desktop runtime to Electron 43.7.7 and added profile guards that block unsupported runtimes and Electron major or minor downgrades before Yomitan storage is loaded. Patch downgrades within the same major and minor version are allowed.
- Development launches now use a separate `SubMiner-dev` profile unless production-profile access is explicitly requested.
- Automatic character-dictionary changes now stop when a previously non-empty Yomitan profile suddenly reports zero dictionaries.
@@ -1,4 +0,0 @@
type: fixed
area: anki
- Media timing review now keeps playback paused while it is open when the dictionary popup or subtitle hover pause ends, as with Hachidori popups closing when the review appears. Playback resumes after the review closes unless a dictionary popup is still open.
-14
View File
@@ -1,14 +0,0 @@
type: added
area: dictionary
- Added a bundled Hachidori dictionary backend alongside the default Yomitan backend. Select it with `dictionaryBackend` and restart SubMiner.
- The tray and dictionary-settings shortcut follow the selected backend. `--hachidori` opens Hachidori settings, while `--yomitan` opens Yomitan settings unless a read-only external Yomitan profile is configured. This restriction also applies with Hachidori active, without blocking Hachidori settings.
- Hachidori integrates with subtitle scanning, popup controls, lookup tracking, character dictionaries, and Anki media enrichment, with separate dictionaries and settings for each backend. Linked Docker hosts receive character dictionary uploads at the linked machine's management port, with `hachidori.externalHostManagementUrl` as an optional override, retry busy imports, and replace the previous dictionary only after a successful import. Browser and app hosts report where to import the dictionary manually.
- Hachidori auto-populates its first Anki template from SubMiner's deck, tags, and field mappings, detects an unambiguous matching note type, and preserves existing custom templates apart from the deck, which follows `ankiConnect.deck` so polling mode enriches Hachidori cards. Anki discovery retries after an unavailable connection.
- Hachidori saves downloadable word audio before sending a note through SubMiner's Anki proxy, so freshly mined animated cards include the word-audio delay.
- Both bundled dictionary backends reload their background code on startup so extension updates take effect while preserving installed dictionaries and settings. Failed Yomitan connection-setting updates can be retried without manually resetting the managed Anki endpoint.
- First-run setup remembers each backend that finished it, including when reopened for legacy plugin cleanup, so switching back does not repeat setup, and the launcher gates playback on the backend the running app started with. Current incomplete or cancelled setup takes precedence over stale completion history. A running Yomitan session keeps using its external profile until it restarts into Hachidori.
- Stats dashboard mining and deck lookup use the selected backend, with the Anki proxy enabled or disabled. Dashboard cards retain the selected history line and media instead of being processed again with the current mpv subtitle, with both Yomitan and Hachidori. Stats cards carry a `SubMiner::Stats` tag so polling also preserves their context. Hachidori word mining supplies dictionary aliases, IDs, and frequency metadata to its native Anki renderer. Settings labels for popup pause and the dictionary deck no longer name Yomitan, and the backend selector sits with the other dictionary settings.
- Hachidori scans, dictionary counts, and settings reads wait for the dictionary engine to finish loading or importing instead of caching empty results.
- First-run setup can link an external Hachidori dictionary host in an app, browser, or Docker container, verify its library, and unlink back to local dictionaries. The optional host controls are collapsed by default and explain which apps or containers must stay running. Unresponsive connection checks time out so setup remains usable. Anki mining, media enrichment, and custom toolbar buttons stay in SubMiner, including when editing buttons while linked.
- Hachidori frequency highlighting reuses dictionary-entry ranks and fills missing ranks through its existing API. Entries without a matching definition may remain unranked. The bundled integration is maintained in a pinned fork submodule using upstream HoshiDicts and WASM binaries.
-4
View File
@@ -1,4 +0,0 @@
type: docs
area: dictionary
- Documented backend selection, separate Hachidori dictionary and Anki setup, the settings flags and shortcuts, and external host linking with local Anki mining and the running requirements for each host type.
-4
View File
@@ -1,4 +0,0 @@
type: fixed
area: setup
- Fixed the command-line launcher showing "Not on PATH" on macOS when SubMiner was opened from Finder or the Dock, even though the launcher directory was already on your terminal PATH.
-4
View File
@@ -1,4 +0,0 @@
type: fixed
area: overlay
- Overlay modals such as the media timing review open on the first attempt when their dedicated window is created on demand, instead of after a four-second retry.
-5
View File
@@ -1,5 +0,0 @@
type: fixed
area: annotations
- Subtitle lines shown while two cues overlap (common in Jimaku and Jellyfin SRTs, and usually the long lines) are now prefetched, so their annotations appear immediately instead of after a delay.
- Subtitle prefetching no longer restarts each time the same subtitle file is re-selected during startup, and a line tokenized before a restart or seek is kept instead of discarded.
@@ -0,0 +1,7 @@
type: fixed
area: stats
- Fixed sentence mining (single and multi-line) counting each created card twice in session and lifetime stats.
- Fixed word and kanji dates after session deletion. Historical "first seen" dates and counts survive retention pruning; new occurrences record observation time so deletion can update dates exactly. Legacy earlier dates are preserved when the remaining history cannot prove their origin. Stale "last seen" dates are repaired once in the background on next launch.
- Fixed re-seen words that the vocabulary filter hides still being counted in the new-words charts, so charts and the vocabulary summary now agree. Existing mismatches are repaired by the same background pass.
- Fixed Overview totals, charts, activity calendar, and vocabulary summary staying stale after deleting sessions. Failed refreshes keep the dashboard visible and offer a retry.
+5
View File
@@ -0,0 +1,5 @@
type: docs
area: stats
- Documented the automatic Overview refresh after deleting sessions or session groups and retrying failed refreshes.
- Documented discovery-date preservation after retention pruning and the handling of older write-time timestamps.
@@ -0,0 +1,4 @@
type: fixed
area: youtube
- The subtitle generation modal now works during Windows YouTube playback, where mpv plays a resolved stream URL instead of the YouTube page; it no longer asks you to open a local file or YouTube video, and generated subtitles load into the playing video.
@@ -0,0 +1,4 @@
type: fixed
area: youtube
- Fixed Windows YouTube playback dropping the rest of mpv's queue when it advanced to a video queued from the YouTube browser window. The direct stream now replaces only that playlist entry.
-16
View File
@@ -6,22 +6,6 @@
*/
{
// ==========================================
// Dictionary Backend
// Select the dictionary lookup backend: yomitan or hachidori.
// Restart SubMiner after changing the backend. Each backend keeps separate settings and dictionaries.
// ==========================================
"dictionaryBackend": "yomitan", // Dictionary lookup backend. Restart SubMiner after changing this setting. Values: yomitan | hachidori
// ==========================================
// Hachidori External Dictionary Imports
// Override the linked Docker host management URL only for a non-default port or a reverse proxy.
// Used only while Hachidori is linked to an external host.
// ==========================================
"hachidori": {
"externalHostManagementUrl": "" // Management URL override for character dictionary uploads to a linked Hachidori Docker host. Empty uses the linked host on the Docker default management port.
}, // Override the linked Docker host management URL only for a non-default port or a reverse proxy.
// ==========================================
// Subtitle Selection
// Select primary and secondary mpv subtitle tracks from the overlay.
+2 -22
View File
@@ -15,9 +15,9 @@ If you changed AnkiConnect's port, set `ankiConnect.url` to match.
## How cards get filled
When Yomitan or Hachidori adds a note, SubMiner fills the sentence, audio, image, and MiscInfo fields. It finds new notes in one of two ways:
When Yomitan adds a note, SubMiner fills the sentence, audio, image, and MiscInfo fields. It finds new notes in one of two ways:
- **Proxy (default).** SubMiner runs a local AnkiConnect-compatible server. The dictionary sends notes through it, and SubMiner fills each one right after Anki accepts it.
- **Proxy (default).** SubMiner runs a local AnkiConnect-compatible server. Yomitan sends notes through it, and SubMiner fills each one right after Anki accepts it.
- **Polling.** With `ankiConnect.proxy.enabled` set to `false`, SubMiner asks AnkiConnect for recently added notes every `ankiConnect.pollingRate` milliseconds.
Set `ankiConnect.behavior.autoUpdateNewCards` to `false` to stop automatic filling and update cards by hand with `Ctrl/Cmd+V` instead.
@@ -42,20 +42,6 @@ Clients must send notes to the proxy (`http://127.0.0.1:8766` here), not to Anki
- **Bundled Yomitan.** SubMiner sets the active Yomitan profile's Anki server for you. With the proxy on, it always points the profile at the proxy. With the proxy off, it sets `ankiConnect.url`, but only if the profile's server is blank or the stock `http://127.0.0.1:8765`.
- **Browser Yomitan or other clients.** Set the Anki server to the proxy URL yourself. To leave your main profile alone, create a separate Yomitan profile for SubMiner, set its Anki server (Settings, Anki) to the proxy URL, and make it active while you mine.
- **Hachidori.** SubMiner routes Hachidori to the proxy while it is active. Keep the proxy on for screenshots and sentence audio.
### Hachidori settings from SubMiner
With the [Hachidori backend](/usage#hachidori-setup), SubMiner fills Hachidori's first Anki template from your `ankiConnect` settings on startup and whenever you open Hachidori Settings:
- The deck always follows `ankiConnect.deck`, because polling only looks for new cards in that deck.
- Configured tags go into untouched defaults.
- Missing word, sentence, pronunciation-audio, and picture mappings are filled with fields that exist in Anki. Pronunciation uses `fields.wordAudio`, or `fields.audio` when no word-audio field is set.
- If the note type is unset, SubMiner picks the one note type that has your word and sentence fields. Enabled Lapis, Kiku, or Senren narrows the search. A fresh mapping also gets Hachidori's matching preset for readings, definitions, and other known fields.
Apart from the deck, SubMiner only fills missing settings. Custom tags, field mappings, advanced templates, and extra templates stay as you set them. If several note types match, pick one in Hachidori Settings. If Anki was closed, start it and open Hachidori Settings again to retry.
Sentence audio, image timing, translation, metadata, and field grouping stay under SubMiner's control. Pronunciation sources are set in Hachidori. Linking an external dictionary host does not change any of this.
### Proxy troubleshooting
@@ -101,8 +87,6 @@ If cards are not getting filled:
Field names are matched case-insensitively. A mapped field that is missing from the note type is skipped.
Hachidori prepares downloadable word audio before it saves a note through the proxy, so the animated image delay works on the first mine. Set a downloadable pronunciation source in Hachidori's Audio settings. Browser speech cannot be saved to Anki. Without word audio, Hachidori shows a warning and the card gets no word-audio hold.
`fields.audio` gets sentence audio, not word audio. Yomitan writes its own dictionary audio into your note, so point `fields.audio` at a separate field such as `SentenceAudio`. The default, `ExpressionAudio`, is the field many note types use for Yomitan's word audio, so leaving it would overwrite that audio.
`ankiConnect.tags` adds tags to every mined or updated card. Set it to `[]` to add none.
@@ -138,8 +122,6 @@ Media settings apply to the next card without a restart.
With `media.reviewTiming` on, SubMiner pauses before making media for word, sentence, and audio cards and opens a review dialog. You can also toggle it for the current session with **Review Media Timing** in the runtime options palette (`Ctrl/Cmd+Shift+O`). Clipboard updates and stats-dashboard mining skip the review.
Playback stays paused while the dialog is open, even if the popup or hover that paused it goes away. When the dialog closes, playback resumes if it was playing before, or if the popup closed in the meantime. A popup that is still open keeps it paused.
The dialog shows the clip over a speech waveform. When the waveform loads, an untouched clip end moves back to just after the last speech in the line. The orange bars still mark the subtitle's original start and end.
During preview, the yellow cursor follows the audio player's position through the selected range and stays at the end when playback finishes. Buffering pauses the cursor too. If the player stops reporting progress for 15 seconds, the preview stops so you can retry.
@@ -226,8 +208,6 @@ For Senren, use the same keys under `isSenren`.
| `auto` | Merge into the existing card. With `deleteDuplicateInAuto`, delete the new card |
| `manual` | Show both cards, let you choose which to keep and preview the merge |
Grouping runs when a new note is added while duplicates exist. With Hachidori, that is the popup's **Add anyway** choice. **Overwrite** updates the existing note in place and only gets media.
The manual dialog cancels itself after 90 seconds. Identical entries are not deduplicated. Press `Ctrl/Cmd+G` to run the duplicate check on the last card yourself.
| Key | Action |
-8
View File
@@ -67,14 +67,6 @@ Everything else needs a restart.
## Core settings
### Dictionary backend
`dictionaryBackend` accepts `"yomitan"` or `"hachidori"`. The default is defined in the [generated configuration example](/config.example.jsonc). Restart SubMiner after changing it. The tray shows the selected backend's settings, and the global dictionary-settings shortcut follows the same selection.
Each backend stores its own dictionaries and mining settings. `yomitan.externalProfilePath` applies only to Yomitan. See [Hachidori setup](./usage.md#hachidori-setup) before switching an existing installation.
`hachidori.externalHostManagementUrl` overrides the management origin that character dictionaries upload to when Hachidori is linked to a Docker host. Leave it empty to use the linked host's machine on the Docker management port. Set it only for a non-default `ADMIN_PORT` or a reverse proxy, and use the management port, not the sharing or dictionary API port. See [Hachidori setup](./usage.md#hachidori-setup).
### Logging
Log files are named by date (`app-YYYY-MM-DD.log`, `launcher-...`, `mpv-...`). Log export writes a sanitized copy and leaves the originals alone.
+4
View File
@@ -47,6 +47,10 @@ electron . --start --dev --log-level debug # verbose launch of an existing bui
electron . --background # tray/background mode
```
Development and debug launches use a separate `SubMiner-dev` profile so runtime experiments cannot modify the installed app's configuration or Yomitan dictionaries. To intentionally use the production profile for a development launch, set `SUBMINER_USE_PRODUCTION_PROFILE=1`. Use that override only after backing up the profile.
Launch source builds through `bun run dev` or `bun run electron`. SubMiner refuses to load its profile when the running Electron major differs from the version pinned by the repository.
To test through the mpv plugin without exporting `SUBMINER_BINARY_PATH` each time, point `mpv.subminerBinaryPath` in your config at the dev script. The launcher passes it to the plugin at runtime:
```json
+6 -4
View File
@@ -42,6 +42,8 @@ To share stats and watch history between machines, use [`subminer sync <host>`](
Recent sessions, a streak calendar, watch-time history, and totals for completed episodes and shows.
Deleting a session or session group refreshes the Overview totals, charts, calendar, and vocabulary summary automatically. If a refresh fails, Overview keeps the previous totals visible and offers **Retry**.
![Stats Overview](/screenshots/stats-overview.png)
### Library
@@ -89,13 +91,11 @@ Searches the primary subtitle lines and titles in your history. **Search by head
Search results and the Vocabulary word panel can create cards from past lines, as long as the source video file is still available:
- **Mine Word**: looks up the word with the selected dictionary backend (Yomitan or Hachidori), plus sentence, audio, and image. The history line you picked is the card's sentence, even while mpv plays another line. Hachidori uses its own Anki template, dictionary aliases, and frequency data.
- **Mine Word**: full Yomitan lookup for the word, plus sentence, audio, and image.
- **Mine Sentence**: a sentence card with `IsSentenceCard` set, for Lapis and Kiku note types.
- **Mine Audio**: an audio card with `IsAudioCard` set.
Word and audio mining appear only when the word occurs in the sentence. All three use your `ankiConnect` deck, note type, fields, and media settings. Anki must be running, and Mine Word needs dictionaries in the selected backend.
Dashboard cards also get the `SubMiner::Stats` tag. With the Anki proxy off, SubMiner uses it to keep the chosen history line when polling picks up the card.
Word and audio mining appear only when the word occurs in the sentence. All three use your `ankiConnect` deck, note type, fields, and media settings. Anki must be running, and Mine Word needs Yomitan dictionaries.
## Repeated lines
@@ -124,4 +124,6 @@ Leave out `--lookback-days` to scan all history. Word and kanji counts are corre
By default SubMiner keeps everything. To limit history, set `immersionTracking.retentionPreset` to `minimal`, `balanced`, or `deep-history`, or set the `immersionTracking.retention.*Days` values yourself (`0` keeps all). Lifetime totals and vocabulary counts are stored separately and stay exact when old sessions are pruned.
Deleting retained sessions preserves earlier word and kanji discovery dates from pruned history. New occurrences record observation time so deletion can update their dates exactly. Older history used write time, so an earlier discovery date is preserved when its origin cannot be determined.
See [Immersion tracking](/configuration#immersion-tracking) and [Stats dashboard](/configuration#stats-dashboard) in the config reference for every option and default.
-16
View File
@@ -6,22 +6,6 @@
*/
{
// ==========================================
// Dictionary Backend
// Select the dictionary lookup backend: yomitan or hachidori.
// Restart SubMiner after changing the backend. Each backend keeps separate settings and dictionaries.
// ==========================================
"dictionaryBackend": "yomitan", // Dictionary lookup backend. Restart SubMiner after changing this setting. Values: yomitan | hachidori
// ==========================================
// Hachidori External Dictionary Imports
// Override the linked Docker host management URL only for a non-default port or a reverse proxy.
// Used only while Hachidori is linked to an external host.
// ==========================================
"hachidori": {
"externalHostManagementUrl": "" // Management URL override for character dictionary uploads to a linked Hachidori Docker host. Empty uses the linked host on the Docker default management port.
}, // Override the linked Docker host management URL only for a non-default port or a reverse proxy.
// ==========================================
// Subtitle Selection
// Select primary and secondary mpv subtitle tracks from the overlay.
+16 -16
View File
@@ -6,12 +6,12 @@ Shortcuts work when the overlay has focus. With the [mpv plugin](/mpv-plugin), `
## Global
| Shortcut | Action | Config key |
| ------------- | ------------------------------- | -------------------------------------- |
| `Alt+Shift+O` | Toggle visible overlay | `shortcuts.toggleVisibleOverlayGlobal` |
| `Alt+Shift+Y` | Open active dictionary settings | Fixed |
| Shortcut | Action | Config key |
| ------------- | ---------------------- | -------------------------------------- |
| `Alt+Shift+O` | Toggle visible overlay | `shortcuts.toggleVisibleOverlayGlobal` |
| `Alt+Shift+Y` | Open Yomitan settings | Fixed |
`Alt+Shift+Y` opens Yomitan or Hachidori settings, whichever backend is running. It is registered with the OS and works from any app. If another app already uses it, SubMiner cannot take it and you cannot rebind it.
`Alt+Shift+Y` is registered with the OS and works from any app. If another app already uses it, SubMiner cannot take it and you cannot rebind it.
## Mining
@@ -98,17 +98,17 @@ The subtitle picker (`g` then `s`) is off until you turn it on in **Settings, Be
Press `y`, then the second key.
| Keys | Action |
| ----- | ------------------------------- |
| `y-y` | Open the SubMiner menu |
| `y-s` | Start the overlay |
| `y-S` | Stop the overlay |
| `y-t` | Toggle the visible overlay |
| `y-o` | Open active dictionary settings |
| `y-r` | Restart the overlay |
| `y-c` | Show overlay status |
| `y-h` | Open session help |
| `v` | Cycle primary subtitle bar |
| Keys | Action |
| ----- | -------------------------- |
| `y-y` | Open the SubMiner menu |
| `y-s` | Start the overlay |
| `y-S` | Stop the overlay |
| `y-t` | Toggle the visible overlay |
| `y-o` | Open Yomitan settings |
| `y-r` | Restart the overlay |
| `y-c` | Show overlay status |
| `y-h` | Open session help |
| `v` | Cycle primary subtitle bar |
The plugin's `v` replaces mpv's own subtitle visibility toggle. When the overlay has focus, `y` then `d` toggles DevTools.
+10
View File
@@ -67,6 +67,16 @@ If you have not set up dictionaries yet, start with [Yomitan setup](/usage#yomit
Word boundaries come from Yomitan's parser. Some splits will be wrong, since Japanese has no spaces.
## "Electron downgrade blocked" or "Unsupported Electron runtime"
SubMiner refuses to load Yomitan storage when the current Electron major does not match the app build, or when the profile was previously opened by a newer Electron major or minor version. Patch downgrades within the same major and minor version are allowed, such as `43.7.3` to `43.7.2`. Downgrades from `43.7.x` to `43.6.x`, or from Electron 43 to 42, are blocked. Launch the packaged app or use the repository's `bun run dev` command. Do not delete the runtime safety record to force an older major or minor version to open the profile.
Older SubMiner builds without this guard cannot enforce the safety record. When testing Electron 43 before returning to an older Electron 42 build, use the separate `SubMiner-dev` profile described in [development](/development#run-locally) so the older build never opens the upgraded profile.
## "Yomitan reported zero dictionaries after previously reporting ..."
SubMiner detected that a previously non-empty Yomitan profile suddenly appears empty. Automatic character-dictionary changes are blocked so they cannot overwrite the suspicious state. Close SubMiner, preserve the profile directory, and restore a known-good backup before importing or deleting dictionaries.
## "Yomitan extension not found in any search path"
The bundled Yomitan is missing. Re-download the AppImage, or place an unpacked Yomitan extension in `~/.config/SubMiner/yomitan`. Source builds must run `bun run build` first to produce `build/yomitan`.
+13 -40
View File
@@ -16,41 +16,10 @@ Run `subminer` with no file to pick one from the current directory instead. See
### Yomitan setup
Lookups need at least one dictionary in SubMiner's bundled Yomitan (or in Hachidori, if you [switched backends](#hachidori-setup)). First-run setup asks you to import one. To add more later, open Yomitan settings with `Alt+Shift+Y` or `subminer app --yomitan`.
Lookups need at least one dictionary in SubMiner's bundled Yomitan. First-run setup asks you to import one. To add more later, open Yomitan settings with `Alt+Shift+Y` or `subminer app --yomitan`.
The bundled Yomitan is separate from any Yomitan in your browser. It has its own dictionaries and settings.
### Hachidori setup
Hachidori is an alternative lookup backend. Set `dictionaryBackend` to `"hachidori"` in settings or `config.jsonc`, then restart SubMiner. Set it back to `"yomitan"` and restart to switch back.
Open Hachidori settings with `Alt+Shift+Y`, the tray's **Open Hachidori Settings**, or `subminer app --hachidori`. Import dictionary ZIPs or use Hachidori's recommended dictionary installer, then set up its Anki template (SubMiner [fills in what it can](/anki-integration#hachidori-settings-from-subminer)). Yomitan and Hachidori keep separate dictionaries and settings. Yomitan profiles, custom Handlebars templates, and `yomitan.externalProfilePath` do not carry over.
Hachidori uses SubMiner's subtitle scanning, popup pause, controller commands, character dictionaries, and Anki media. Keep the [Anki proxy](/anki-integration#proxy-mode-setup-yomitan-texthooker) on for screenshots and sentence audio. Hachidori's own screen recorder and screenshot capture are off inside SubMiner. `startupWarmups.yomitanExtension` and `subtitleStyle.autoPauseVideoOnYomitanPopup` apply to whichever backend is selected.
Switching backends:
- First-run setup asks for dictionaries the first time you switch to a backend. Switching back to a backend that already finished setup skips it.
- Until you restart, SubMiner keeps running the backend it started with. The launcher waits for that backend before playback and logs a restart reminder.
- `--yomitan` and `--hachidori` both work whichever backend is selected. Opening settings does not switch backends.
- When `yomitan.externalProfilePath` is set, `--yomitan` is disabled to keep the external profile read-only. Hachidori settings still open.
#### External dictionary host
First-run setup can link a Hachidori host instead of using local dictionaries: **Dictionary source → Use an external dictionary host → Link host**. Turn on sharing in the other Hachidori app or browser, or start a compatible Docker host, then enter its sharing address, for example `127.0.0.1:8771` or `ws://host:8771/link`. Use the WebSocket sharing port, not the management page or HTTP API port.
| Host | Must be running |
| -------- | ------------------------------------------- |
| Browser | The browser, the Hachidori extension, relay |
| Electron | The host app and any relay it needs |
| Docker | The container only |
Setup checks the connection and the host's dictionaries before **Finish** unlocks, so import at least one dictionary on the host and refresh. The link survives restarts. **Unlink and use local dictionaries** goes back to local.
While linked, dictionaries and dictionary settings come from the host. Anki templates, pronunciation sources, custom buttons, and SubMiner's audio and image processing stay local. Frequency annotations use ranks returned with dictionary entries, and SubMiner asks the host for missing ones. Words with no matching definition entry may stay unranked even if a frequency dictionary lists them.
When linked to a Docker host, SubMiner uploads [character dictionaries](/character-dictionary) to the linked machine's management port (8780) and replaces the previous dictionary once the import succeeds, retrying while the host is busy. If the host uses a different `ADMIN_PORT` or sits behind a reverse proxy, set `hachidori.externalHostManagementUrl` to its management origin, for example `"http://pve-main:9000"`. This is not the WebSocket sharing address. Browser and app hosts cannot receive uploads over the link, so sync reports where the merged ZIP is and you import it from that host's Hachidori settings. Local Hachidori does not need this setting.
## Picking files
```bash
@@ -64,15 +33,15 @@ See [Launcher script](/launcher-script#video-picker) for picker and history deta
## Overlay basics
| Key | Action |
| ------------- | ---------------------------------------------------------------------------- |
| `Alt+Shift+O` | Show or hide the overlay (works while the overlay or mpv has focus) |
| `Alt+Shift+Y` | Open Yomitan or Hachidori settings (works from any window, not configurable) |
| `V` | Cycle the subtitle bar through hidden, visible, and hover-only |
| `Ctrl+Alt+P` | Open the playlist browser to queue, reorder, or jump between episodes |
| `Ctrl/Cmd+/` | Show every overlay and mpv keybinding for this session |
| Key | Action |
| ------------- | --------------------------------------------------------------------- |
| `Alt+Shift+O` | Show or hide the overlay (works while the overlay or mpv has focus) |
| `Alt+Shift+Y` | Open Yomitan settings (works from any window, not configurable) |
| `V` | Cycle the subtitle bar through hidden, visible, and hover-only |
| `Ctrl+Alt+P` | Open the playlist browser to queue, reorder, or jump between episodes |
| `Ctrl/Cmd+/` | Show every overlay and mpv keybinding for this session |
Hovering subtitle text pauses mpv, and moving away resumes it. An open dictionary popup also keeps playback paused. Turn these off with `subtitleStyle.autoPauseVideoOnHover` and `subtitleStyle.autoPauseVideoOnYomitanPopup`.
Hovering subtitle text pauses mpv, and moving away resumes it. An open Yomitan popup also keeps playback paused. Turn these off with `subtitleStyle.autoPauseVideoOnHover` and `subtitleStyle.autoPauseVideoOnYomitanPopup`.
You can drop files onto the overlay:
@@ -176,6 +145,10 @@ Press `Alt+C` to choose a controller and remap buttons. Click an action's **Lear
On controllers that report the W3C standard layout, the default quit button lands on `L2` instead of `Select`. Remap it with `Alt+C`. All options are in [Configuration](/configuration#controller-support).
## Development profiles
`--dev` and `--debug` use a separate `SubMiner-dev` profile. They do not read or modify dictionaries and configuration from the installed app unless `SUBMINER_USE_PRODUCTION_PROFILE=1` is explicitly set. See [Development](/development#run-locally).
## Changing settings while you watch
SubMiner watches your config file and applies most changes without a restart, including subtitle style, keybindings, and most Anki settings. If a change needs a restart, SubMiner tells you. If the file has an error, it keeps the last working config and shows a notification. See [Configuration](/configuration).
-6
View File
@@ -46,12 +46,6 @@ Update checks and startup launcher migration share a serialized update-state sto
## Architecture Intent
The dictionary backend is selected once at startup by `dictionaryBackend`. Yomitan keeps its existing session and external-profile policy. Hachidori uses `persist:hachidori`; overlay windows select that session before extension loading, including deferred startup. Named settings flags can open either backend without injecting a second reader into the active overlay. The detached stats word helper reads the same config key so dashboard mining uses the active backend.
`setup-state.json` records one backend's status at a time plus `completedDictionaryBackends`, the backends that finished setup before. The app projects the file onto its active backend on startup and stamps that backend into the file. The launcher gates playback on the stamped backend when an app is already running, since a config edit takes effect only after restart.
`vendor/hachidori/` is a submodule of `ksyasuda/hachidori`, tracking the `subminer` branch and pinned to a tested commit. Its nested HoshiDicts submodule and WASM binaries remain upstream versions. Initialize sources with `git submodule update --init --recursive`; merge upstream updates in the fork, test them, then update SubMiner's submodule commit. `SOURCE.json` records the upstream base and artifact checksums; the submodule commit identifies the integrated version. `build:hachidori` verifies recorded artifact checksums and stages the extension for development and packaging. It enables overlay mode, disables custom JavaScript, keeps the lookup highlight on in the overlay first-install options (SubMiner captures media from mpv, not the overlay viewport), removes the unsupported `userScripts` permission, and sets a fixed manifest `key` so the extension ID (and the storage origin holding its dictionaries and settings) does not depend on the userData path, only in that staged copy; the fork keeps upstream browser defaults. Before loading the extension, its session clears service worker registrations so Electron uses the current bundled code; dictionary databases and settings remain intact. First-run setup uses Hachidori sharing messages to link or unlink external dictionary hosts and checks their live inventory. Linked dictionaries and dictionary edits use the host, while Anki configuration, pronunciation sources, custom buttons, and mining stay local to SubMiner. The parser bridge adapts its runtime messages to the existing subtitle scanner and dictionary automation. Scanning retains term-entry frequencies, and only tokens without ranks need further frequency lookups through the existing term-entry API. This requires a matching definition entry and does not preserve the frequency source's reading provenance. SubMiner consumes native `hachidori-popup-shown` and `hachidori-popup-hidden` attention events for mouse handling, keyboard focus, and the subtitle sidebar. Attention also covers a left press anywhere on the overlay that may start a selection, and the host element only exists after the first lookup, so once a Hachidori event has been seen popup auto-pause requires an unhidden popup pane in the host's shadow root and rechecks after each successful lookup. The fork retains host attributes, hover and successful-lookup notifications, and commands that need private reader state. The Anki proxy strips local duplicate/overwrite metadata before forwarding requests and enriches only confirmed writes.
- Small units, explicit boundaries
- Composition over monoliths
- Pure helpers where possible
-1
View File
@@ -63,7 +63,6 @@ bun run docs:build
Content checks run inside the electron-builder afterPack hook. See the
[release guide](../RELEASING.md#package-contents-checks) for the
installed-app verification checklist.
- Dictionary backend windows: after a full build, run `xvfb-run -a bun run test:dictionary:electron` on headless Linux. It uses temporary profiles to check both named settings commands, backend session isolation, the overlay's external-link bridge, and Hachidori's native dictionary parser.
- Coverage for the maintained source lane: `bun run test:coverage:src`
- Deep/local full gate: default handoff gate above
+1 -7
View File
@@ -26,10 +26,7 @@ import {
readSetupState,
} from '../../src/shared/setup-state.js';
import { detectInstalledFirstRunPluginCandidates } from '../../src/main/runtime/first-run-setup-plugin.js';
import {
hasLauncherExternalYomitanProfileConfig,
loadLauncherDictionaryBackend,
} from '../config.js';
import { hasLauncherExternalYomitanProfileConfig } from '../config.js';
const SETUP_WAIT_TIMEOUT_MS = 10 * 60 * 1000;
const SETUP_POLL_INTERVAL_MS = 500;
@@ -118,9 +115,6 @@ async function ensurePlaybackSetupReady(context: LauncherCommandContext): Promis
const configDir = getLauncherConfigDir();
const statePath = getSetupStatePath(configDir);
const ready = await ensureLauncherSetupReady({
dictionaryBackend: loadLauncherDictionaryBackend(),
isAppRunning: () => isRunningAppControlServerAvailable(args.logLevel, configDir),
warn: (message) => log('warn', args.logLevel, message),
readSetupState: () => readSetupState(statePath),
isExternalYomitanConfigured: () => hasLauncherExternalYomitanProfileConfig(),
hasLegacyMpvPlugin: () =>
-7
View File
@@ -1,5 +1,4 @@
import { fail } from './log.js';
import type { DictionaryBackend } from '../src/types/config.js';
import type {
Args,
LauncherLoggingConfig,
@@ -101,12 +100,6 @@ export function loadLauncherLoggingConfig(): LauncherLoggingConfig {
};
}
export function loadLauncherDictionaryBackend(): DictionaryBackend {
return readLauncherMainConfigObject()?.dictionaryBackend === 'hachidori'
? 'hachidori'
: 'yomitan';
}
export function hasLauncherExternalYomitanProfileConfig(): boolean {
return readExternalYomitanProfilePath(readLauncherMainConfigObject()) !== null;
}
-4
View File
@@ -214,10 +214,6 @@ export function applyRootOptionsToArgs(
if (options.update === true) parsed.update = true;
if (options.version === true) parsed.version = true;
if (options.settings === true) parsed.settings = true;
if (options.yomitan === true || options.hachidori === true) {
parsed.appPassthrough = true;
parsed.appArgs = [options.yomitan === true ? '--yomitan' : '--hachidori'];
}
if (options.startOverlay === true) parsed.autoStartOverlay = true;
if (options.texthooker === false) parsed.useTexthooker = false;
if (typeof options.args === 'string') parsed.mpvArgs = options.args;
-2
View File
@@ -87,8 +87,6 @@ function applyRootOptions(program: Command): void {
.option('--log-level <level>', 'Log level')
.option('-v, --version', 'Show SubMiner version')
.option('--settings', 'Open settings window')
.option('--yomitan', 'Open Yomitan settings window')
.option('--hachidori', 'Open Hachidori settings window')
.option('-u, --update', 'Check for updates')
.option('-R, --rofi', 'Use rofi picker')
.option('-H, --history', 'Browse local watch history')
+20 -73
View File
@@ -282,31 +282,29 @@ test('doctor refresh-known-words forwards app refresh command without requiring
});
});
for (const flag of ['--settings', '--yomitan', '--hachidori']) {
test(`launcher ${flag} forwards the matching app settings command`, () => {
withTempDir((root) => {
const homeDir = path.join(root, 'home');
const xdgConfigHome = path.join(root, 'xdg');
const appPath = path.join(root, 'fake-subminer.sh');
const capturePath = path.join(root, 'captured-args.txt');
fs.writeFileSync(
appPath,
`#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}if [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n`,
);
fs.chmodSync(appPath, 0o755);
test('launcher settings option forwards app settings window command', () => {
withTempDir((root) => {
const homeDir = path.join(root, 'home');
const xdgConfigHome = path.join(root, 'xdg');
const appPath = path.join(root, 'fake-subminer.sh');
const capturePath = path.join(root, 'captured-args.txt');
fs.writeFileSync(
appPath,
`#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}if [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n`,
);
fs.chmodSync(appPath, 0o755);
const env = {
...makeTestEnv(homeDir, xdgConfigHome),
SUBMINER_APPIMAGE_PATH: appPath,
SUBMINER_TEST_CAPTURE: capturePath,
};
const result = runLauncher([flag], env);
const env = {
...makeTestEnv(homeDir, xdgConfigHome),
SUBMINER_APPIMAGE_PATH: appPath,
SUBMINER_TEST_CAPTURE: capturePath,
};
const result = runLauncher(['--settings'], env);
assert.equal(result.status, 0);
assert.equal(fs.readFileSync(capturePath, 'utf8'), `${flag}\n`);
});
assert.equal(result.status, 0);
assert.equal(fs.readFileSync(capturePath, 'utf8'), '--settings\n');
});
}
});
test('launcher settings command forwards app settings window command', () => {
withTempDir((root) => {
@@ -1133,54 +1131,3 @@ test('classifyJellyfinChildSelection keeps container drilldown state instead of
id: 'season-2',
});
});
test('external Yomitan profile remains available while a running app awaits a backend switch', () => {
withTempDir((dir) => {
const env = makeTestEnv(dir, path.join(dir, 'config'));
const configPath = resolveConfigFilePath({
appDataDir: env.APPDATA,
xdgConfigHome: env.XDG_CONFIG_HOME,
homeDir: dir,
existsSync: () => false,
});
fs.mkdirSync(path.dirname(configPath), { recursive: true });
fs.writeFileSync(
configPath,
JSON.stringify({
dictionaryBackend: 'hachidori',
yomitan: { externalProfilePath: '/external/yomitan-profile' },
}),
);
const result = spawnSync(
process.execPath,
[
'--eval',
`
import assert from 'node:assert/strict';
import { hasLauncherExternalYomitanProfileConfig } from './launcher/config.ts';
import { ensureLauncherSetupReady } from './launcher/setup-gate.ts';
import { createDefaultSetupState } from './src/shared/setup-state.ts';
for (const running of [true, false]) {
let launches = 0;
let tick = 0;
const ready = await ensureLauncherSetupReady({
dictionaryBackend: 'hachidori',
isAppRunning: async () => running,
readSetupState: () => ({ ...createDefaultSetupState(), dictionaryBackend: 'yomitan' }),
isExternalYomitanConfigured: hasLauncherExternalYomitanProfileConfig,
launchSetupApp: () => { launches += 1; },
sleep: async () => {},
now: () => tick++,
timeoutMs: 2,
pollIntervalMs: 1,
});
assert.equal(ready, running);
assert.equal(launches, running ? 0 : 1);
}
`,
],
{ cwd: process.cwd(), env, encoding: 'utf8', timeout: LAUNCHER_RUN_TIMEOUT_MS },
);
assert.equal(result.status, 0, result.stderr);
});
});
-9
View File
@@ -353,12 +353,3 @@ test('parseArgs requires an explicit logs action', () => {
assert.equal(exit.code, 1);
assert.match(exit.stderr, /Logs command requires -e or --export/);
});
for (const flag of ['--yomitan', '--hachidori']) {
test(`parseArgs forwards ${flag} to the app`, () => {
const parsed = parseArgs([flag], 'subminer', {});
assert.equal(parsed.appPassthrough, true);
assert.deepEqual(parsed.appArgs, [flag]);
assert.equal(parsed.settings, false);
});
}
+2 -104
View File
@@ -1,11 +1,7 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
ensureLauncherSetupReady,
resolveLauncherGateBackend,
waitForSetupCompletion,
} from './setup-gate';
import { createDefaultSetupState, type SetupState } from '../src/shared/setup-state';
import { ensureLauncherSetupReady, waitForSetupCompletion } from './setup-gate';
import type { SetupState } from '../src/shared/setup-state';
const commandLineSetupDefaults = {
bunInstallStatus: 'unknown',
@@ -299,101 +295,3 @@ test('ensureLauncherSetupReady ignores stale cancelled state after launching set
assert.equal(result, true);
});
test('Hachidori setup ignores completed Yomitan state and external Yomitan profiles', async () => {
let state: SetupState = { ...createDefaultSetupState(), status: 'completed' };
let launched = 0;
let polls = 0;
const ready = await ensureLauncherSetupReady({
dictionaryBackend: 'hachidori',
readSetupState: () => state,
isExternalYomitanConfigured: () => true,
launchSetupApp: () => {
launched += 1;
},
sleep: async () => {
polls += 1;
state = { ...state, dictionaryBackend: 'hachidori', lastSeenYomitanDictionaryCount: 1 };
},
now: () => polls,
timeoutMs: 5,
pollIntervalMs: 1,
});
assert.equal(ready, true);
assert.equal(launched, 1);
assert.equal(polls, 1);
});
test('matching Hachidori completion resumes playback without launching setup', async () => {
const ready = await ensureLauncherSetupReady({
dictionaryBackend: 'hachidori',
readSetupState: () => ({
...createDefaultSetupState(),
dictionaryBackend: 'hachidori',
status: 'completed',
lastSeenYomitanDictionaryCount: 1,
}),
launchSetupApp: () => assert.fail('setup should not open'),
sleep: async () => undefined,
now: () => 0,
timeoutMs: 5,
pollIntervalMs: 1,
});
assert.equal(ready, true);
});
test('a backend that finished setup earlier passes the gate after switching back', async () => {
const ready = await ensureLauncherSetupReady({
dictionaryBackend: 'yomitan',
readSetupState: () => ({
...createDefaultSetupState(),
dictionaryBackend: 'hachidori',
status: 'incomplete',
completedDictionaryBackends: ['yomitan'],
}),
launchSetupApp: () => assert.fail('setup should not open'),
sleep: async () => undefined,
now: () => 0,
timeoutMs: 5,
pollIntervalMs: 1,
});
assert.equal(ready, true);
});
test('gate follows the running app backend until it restarts into the configured one', async () => {
const warnings: string[] = [];
const state = {
...createDefaultSetupState(),
dictionaryBackend: 'yomitan' as const,
status: 'completed' as const,
};
assert.equal(
await resolveLauncherGateBackend({
configuredBackend: 'hachidori',
state,
isAppRunning: async () => true,
warn: (message) => warnings.push(message),
}),
'yomitan',
);
assert.match(warnings[0] ?? '', /restart it to switch to hachidori/);
assert.equal(
await resolveLauncherGateBackend({
configuredBackend: 'hachidori',
state,
isAppRunning: async () => false,
}),
'hachidori',
);
const ready = await ensureLauncherSetupReady({
dictionaryBackend: 'hachidori',
isAppRunning: async () => true,
readSetupState: () => state,
launchSetupApp: () => assert.fail('the running Yomitan app already completed setup'),
sleep: async () => undefined,
now: () => 0,
timeoutMs: 5,
pollIntervalMs: 1,
});
assert.equal(ready, true);
});
+6 -47
View File
@@ -1,13 +1,7 @@
import type { DictionaryBackend } from '../src/types/config.js';
import {
getSetupStateDictionaryBackend,
isSetupCompleted,
type SetupState,
} from '../src/shared/setup-state.js';
import { isSetupCompleted, type SetupState } from '../src/shared/setup-state.js';
export async function waitForSetupCompletion(deps: {
readSetupState: () => SetupState | null;
dictionaryBackend?: DictionaryBackend;
sleep: (ms: number) => Promise<void>;
now: () => number;
timeoutMs: number;
@@ -19,7 +13,7 @@ export async function waitForSetupCompletion(deps: {
while (deps.now() <= deadline) {
const state = deps.readSetupState();
if (isSetupCompleted(state, deps.dictionaryBackend)) {
if (isSetupCompleted(state)) {
return 'completed';
}
if (ignoringCancelled && state != null && state.status !== 'cancelled') {
@@ -40,7 +34,6 @@ export async function waitForSetupCompletion(deps: {
export async function waitForLegacyMpvPluginPromptResolution(deps: {
readSetupState: () => SetupState | null;
dictionaryBackend?: DictionaryBackend;
sleep: (ms: number) => Promise<void>;
now: () => number;
timeoutMs: number;
@@ -48,13 +41,13 @@ export async function waitForLegacyMpvPluginPromptResolution(deps: {
initialState?: SetupState | null;
}): Promise<'acknowledged' | 'cancelled' | 'timeout'> {
const deadline = deps.now() + deps.timeoutMs;
const initialCompleted = isSetupCompleted(deps.initialState, deps.dictionaryBackend);
const initialCompleted = isSetupCompleted(deps.initialState);
const initialCompletedAt = deps.initialState?.completedAt ?? null;
while (deps.now() <= deadline) {
const state = deps.readSetupState();
if (
isSetupCompleted(state, deps.dictionaryBackend) &&
isSetupCompleted(state) &&
(!initialCompleted || state?.completedAt !== initialCompletedAt)
) {
return 'acknowledged';
@@ -69,34 +62,8 @@ export async function waitForLegacyMpvPluginPromptResolution(deps: {
return 'timeout';
}
/**
* The app pins its dictionary backend at startup while the config file can change
* underneath it. When an app is already running, gate on the backend it recorded
* in the setup state rather than the config value it has not restarted into.
*/
export async function resolveLauncherGateBackend(deps: {
configuredBackend: DictionaryBackend;
state: SetupState | null;
isAppRunning?: () => Promise<boolean>;
warn?: (message: string) => void;
}): Promise<DictionaryBackend> {
const runningBackend = deps.state
? getSetupStateDictionaryBackend(deps.state)
: deps.configuredBackend;
if (runningBackend === deps.configuredBackend || !(await deps.isAppRunning?.())) {
return deps.configuredBackend;
}
deps.warn?.(
`SubMiner is running with the ${runningBackend} dictionary backend; restart it to switch to ${deps.configuredBackend}.`,
);
return runningBackend;
}
export async function ensureLauncherSetupReady(deps: {
readSetupState: () => SetupState | null;
dictionaryBackend?: DictionaryBackend;
isAppRunning?: () => Promise<boolean>;
warn?: (message: string) => void;
isExternalYomitanConfigured?: () => boolean;
hasLegacyMpvPlugin?: () => boolean;
launchSetupApp: () => void;
@@ -106,12 +73,6 @@ export async function ensureLauncherSetupReady(deps: {
pollIntervalMs: number;
}): Promise<boolean> {
const initialState = deps.readSetupState();
const dictionaryBackend = await resolveLauncherGateBackend({
configuredBackend: deps.dictionaryBackend ?? 'yomitan',
state: initialState,
isAppRunning: deps.isAppRunning,
warn: deps.warn,
});
let setupLaunched = false;
const launchSetupApp = () => {
if (setupLaunched) return;
@@ -123,7 +84,6 @@ export async function ensureLauncherSetupReady(deps: {
launchSetupApp();
const result = await waitForLegacyMpvPluginPromptResolution({
readSetupState: deps.readSetupState,
dictionaryBackend,
sleep: deps.sleep,
now: deps.now,
timeoutMs: deps.timeoutMs,
@@ -135,18 +95,17 @@ export async function ensureLauncherSetupReady(deps: {
}
}
if (dictionaryBackend !== 'hachidori' && deps.isExternalYomitanConfigured?.()) {
if (deps.isExternalYomitanConfigured?.()) {
return true;
}
const stateAfterLegacyPrompt = deps.readSetupState();
if (isSetupCompleted(stateAfterLegacyPrompt, dictionaryBackend)) {
if (isSetupCompleted(stateAfterLegacyPrompt)) {
return true;
}
launchSetupApp();
const result = await waitForSetupCompletion({
...deps,
dictionaryBackend,
ignoreInitialCancelledState: stateAfterLegacyPrompt?.status === 'cancelled',
});
return result === 'completed';
+4 -10
View File
@@ -2,7 +2,7 @@
"name": "subminer",
"productName": "SubMiner",
"desktopName": "SubMiner.desktop",
"version": "0.20.0",
"version": "0.20.1-beta.2",
"description": "All-in-one sentence mining overlay with AnkiConnect and dictionary integration",
"packageManager": "bun@1.3.5",
"main": "dist/main-entry.js",
@@ -12,7 +12,6 @@
"get-frequency": "bun run scripts/get_frequency.ts --pretty --color-top-x 10000 --yomitan-user-data ~/.config/SubMiner --colorized-line",
"get-frequency:electron": "bun run build:yomitan && bun build scripts/get_frequency.ts --format=cjs --target=node --outfile dist/scripts/get_frequency.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/get_frequency.js --pretty --color-top-x 10000 --yomitan-user-data ~/.config/SubMiner --colorized-line",
"test-yomitan-parser": "bun run scripts/test-yomitan-parser.ts",
"test:dictionary:electron": "bun scripts/run-dictionary-settings-smoke.mjs && bun scripts/run-dictionary-settings-smoke.mjs --backend=yomitan && env -u ELECTRON_RUN_AS_NODE electron --no-sandbox --ozone-platform=x11 scripts/check-hachidori-parser.cjs",
"test-yomitan-parser:electron": "bun run build:yomitan && bun build scripts/test-yomitan-parser.ts --format=cjs --target=node --outfile dist/scripts/test-yomitan-parser.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/test-yomitan-parser.js",
"verify-known-word-highlights:electron": "bun run build:yomitan && bun build scripts/verify-known-word-highlights.ts --format=cjs --target=node --outfile dist/scripts/verify-known-word-highlights.js --packages=external && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/verify-known-word-highlights.js",
"record-tokenizer-fixture:electron": "bun run build:yomitan && bun build scripts/record-tokenizer-fixture.ts --format=cjs --target=node --outfile dist/scripts/record-tokenizer-fixture.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/record-tokenizer-fixture.js",
@@ -22,7 +21,7 @@
"build:launcher": "bun run scripts/build-launcher.ts",
"build:stats": "cd stats && bun run build",
"dev:stats": "cd stats && bun run dev",
"build": "bun run build:yomitan && bun run build:hachidori && bun run build:stats && tsc -p tsconfig.json && bun run build:renderer && bun run build:settings && bun run build:syncui && bun run build:launcher && bun run build:assets",
"build": "bun run build:yomitan && bun run build:stats && tsc -p tsconfig.json && bun run build:renderer && bun run build:settings && bun run build:syncui && bun run build:launcher && bun run build:assets",
"build:renderer": "esbuild src/renderer/renderer.ts --bundle --platform=browser --format=esm --target=es2022 --outfile=dist/renderer/renderer.js --sourcemap",
"build:settings": "esbuild src/settings/settings.ts --bundle --platform=browser --format=esm --target=es2022 --outfile=dist/settings/settings.js --sourcemap",
"build:syncui": "esbuild src/syncui/syncui.ts --bundle --platform=browser --format=esm --target=es2022 --outfile=dist/syncui/syncui.js --sourcemap && esbuild src/preload-syncui.ts --bundle --platform=node --format=cjs --target=node20 --external:electron --outfile=dist/preload-syncui.js --sourcemap",
@@ -82,8 +81,7 @@
"build:mac:zip": "bun run build && electron-builder --mac zip --publish never",
"build:win": "bun run build && electron-builder --win nsis zip --publish never",
"build:win:unsigned": "bun run build && node scripts/build-win-unsigned.mjs",
"test:package": "bun scripts/run-package-smoke.mjs",
"build:hachidori": "bun scripts/build-hachidori.mjs"
"test:package": "bun scripts/run-package-smoke.mjs"
},
"overrides": {
"@xmldom/xmldom": "0.8.15",
@@ -130,7 +128,7 @@
"@electron/asar": "3.4.1",
"@types/node": "^24.10.0",
"@types/ws": "^8.18.1",
"electron": "42.10.0",
"electron": "43.7.7",
"electron-builder": "26.16.1",
"esbuild": "^0.25.12",
"eslint": "^10.8.0",
@@ -234,10 +232,6 @@
"from": "build/yomitan",
"to": "yomitan"
},
{
"from": "build/hachidori",
"to": "hachidori"
},
{
"from": "vendor/yomitan-jlpt-vocab",
"to": "yomitan-jlpt-vocab"
+66 -103
View File
@@ -1,125 +1,88 @@
> This is a prerelease build for testing. Stable changelog and docs-site updates remain pending until the final stable release.
<!-- prerelease-version: 0.20.0-beta.2; since: v0.20.0-beta.1 -->
<!-- prerelease-version: 0.20.1-beta.2; since: v0.20.1-beta.1 -->
## Changes since v0.20.0-beta.1
## Changes since v0.20.1-beta.1
- Added an optional subtitle selection modal for primary and secondary mpv subtitle tracks. Enable it in Settings > Behavior, then press `g` followed by `s`. Disabling it restores mpv's subtitle selection binding.
- Single-key shortcut actions now take priority over configured multi-key sequence prefixes, conflicting sequences are disabled with a warning, and existing `y` commands remain reserved.
- Jellyfin casting and playback now honor a configured mpv executable path, allowing playback when mpv is installed outside the system PATH, and portable plugins located beside that executable are now detected.
- Fixed word-card sentence furigana falling out of sync with full stats-search context and expanded timing-review selections; stale furigana is now cleared if regeneration fails.
- Upgraded the desktop runtime to Electron 43.7.7.
- SubMiner now refuses to start on an unsupported Electron runtime, or after an Electron major or minor downgrade, so your Yomitan profile data stays safe; patch downgrades are still allowed.
- Automatic character-dictionary updates now stop if a Yomitan profile that had dictionaries suddenly reports none.
- Development launches now use a separate `SubMiner-dev` profile unless you explicitly ask for the production profile.
- Sentence mining (single and multi-line) no longer counts each card twice in session and lifetime stats.
- Deleting sessions now updates word and kanji "first seen" and "last seen" dates correctly, and retention pruning keeps historical discovery dates and counts.
- Words hidden by the vocabulary filter are no longer counted in the new-words charts, so the charts now match the vocabulary summary.
- A one-time background repair on next launch fixes stale "last seen" dates and existing chart mismatches.
- Overview totals, charts, the activity calendar, and the vocabulary summary now refresh after you delete sessions, and a failed refresh keeps the dashboard visible with a retry option.
- Subtitle generation now works during Windows YouTube playback and loads the generated subtitles into the playing video.
- On Windows, moving on to a video queued from the YouTube browser window no longer drops the rest of mpv's queue.
- The stats docs now cover the automatic Overview refresh after deletions and how discovery dates are kept through retention pruning.
## Highlights
### Added
- **Japanese Subtitle Generation**:
- Generate Japanese subtitles locally with whisper.cpp, right from a modal (`Ctrl+Shift+G`), the subtitle sidebar's generation button when no subtitles are loaded, or `subminer generate-subs`, with progress, cancellation, and automatic loading into mpv when it's done.
- Pick and download an official multilingual Whisper model in-app (including smaller quantized variants), or point Settings at one you already have. SubMiner recommends `large-v3-turbo` when CUDA is available and `small` otherwise, and tells you up front if `whisper-cli`, `ffmpeg`, or `ffprobe` can't be found.
- An optional "Focus on spoken dialogue" mode uses a Silero VAD model to keep quiet or music-covered dialogue that would otherwise get dropped.
- Long passages split near natural speech pauses, guided by an existing subtitle track when one is loaded, giving tighter timing and fewer repeated-word glitches.
- **Media Timing Review Frame Picker**:
- The screenshot used for a mined card can now be chosen independently of the audio clip, with its own live preview, time slider, and frame-by-frame stepping.
- Works for local video and for seekable remote streams like Jellyfin.
- **Overlay Keybinding Pickup**: The overlay now recognizes your mpv keybindings (from mpv's defaults, `input.conf`, and loaded scripts) as long as they don't conflict with SubMiner's own controls. Picked-up bindings work for the session but won't show up in the help menu.
- **Subtitle Selection Modal**:
- An optional subtitle selection modal lets you pick primary and secondary mpv subtitle tracks without leaving the overlay.
- Enable it in Settings under Behavior, then trigger it with `g` followed by `s`; turning it off restores mpv's normal subtitle selection binding.
- Single-key shortcuts always take priority over multi-key sequences, and any conflicting sequence is disabled with a warning instead of misbehaving.
- **Subtitle Sidebar Selection & Copy**: You can now select dialogue across multiple subtitle sidebar rows and copy it, without timestamps, using Ctrl/Cmd+C or the Copy button, without seeking or mining a card.
- **Jimaku Live Action Search**: The Jimaku modal has separate Anime and Live Action tabs (switch with Arrow Left/Right) so you can search Jimaku's live-action subtitle catalogue directly.
- **Live-Action TMDB Library**:
- Live-action dramas and movies in the stats Library now get posters, synopses, and titles from TMDB.
- Titles AniList can't match are looked up on TMDB automatically when the parsed filename matches a title exactly; otherwise use the new **Link to TMDB** action. Entries linked to the same TMDB title merge into one card, and the Library kind selector gained a Live Action option.
- Release builds already include a TMDB key; if you run from source, set `tmdb.apiKey` (or `tmdb.apiKeyCommand`) yourself.
- **YouTube Library Kind**:
- YouTube channels are now their own Library media kind, with new All Titles, Anime, and YouTube filters. Existing channel entries migrate automatically with viewing history and manual video assignments intact.
- Channels stay out of AniList matching, season repair, and duplicate recommendations, and can't be merged or moved into an anime entry.
- **YouTube Browser Window**:
- Open it with `subminer youtube` (or `subminer yt`), or pick **Browse YouTube** from the tray. It stays logged in across restarts.
- Clicking a video plays it in mpv. Middle-click, Shift-click, Ctrl-click, or the right-click menu adds it to mpv's playlist.
- Queued videos get the same Japanese subtitle setup when they start.
- Closing the window quits SubMiner if you started it with `subminer youtube`. If a video is still playing, SubMiner waits until mpv closes.
- **Whisper Subtitles for YouTube**:
- Set `youtube.subtitleSource` to `whisper` to make subtitles with Whisper instead of downloading YouTube's captions.
- The video stays paused while Whisper works, and progress shows in the subtitle generation modal. You can close the modal to keep watching, or cancel.
- The subtitle generation modal (`Ctrl+Shift+G`) now works on YouTube videos, including on Windows.
- **Mouse and Scroll Wheel Bindings**:
- You can now use scroll wheel keys (`WHEEL_UP`, `WHEEL_DOWN`, `WHEEL_LEFT`, `WHEEL_RIGHT`) in `keybindings` and record them in the settings key editor.
- mpv's mouse button and wheel bindings now work while the cursor is over the overlay. This matters most on Hyprland, where these bindings did nothing before.
- When a binding conflicts, SubMiner's own bindings and right-click pause take priority over mpv's.
- **Session Help Commands**: During playback, press `Enter` or double-click a row in session help to run that command. Hovering over a row makes it the target for `Enter`.
### Changed
- **Bundled Bun Runtime**: Every SubMiner launcher, installed or downloaded, now runs on the Bun runtime bundled with the app instead of a system-wide Bun install. Recognized legacy launchers migrate automatically, Windows users get a new `subminer.cmd` download, and first-run setup now shows a single optional launcher control with runtime repair guidance only when something actually needs it.
- **Compressed Incremental Sync**: Cross-machine sync between compatible macOS/Linux machines now transfers only what changed, compressed, using a cached snapshot from the last sync to cut traffic further. Machines without a compatible rsync (including Windows) fall back to compressed scp automatically, older peers keep working, and transfers now time out after 30 minutes instead of hanging indefinitely.
- **Smaller Install Size**: Installers and the unpacked app are smaller after dropping demo media, source maps, TypeScript sources, test fixtures, and unused binaries, and sharing one Japanese UI font across windows. Release builds now publish a package-size comparison against the previous release.
- **Stats Server Request Safety**: The stats server, including the in-app stats overlay which now loads through it, only accepts requests from the local machine and requires a JSON content type for anything that changes data. If you were exposing the dashboard through a reverse proxy or Tailscale Serve, that's no longer supported, and any script posting to the stats API needs to send `Content-Type: application/json`.
- **Yomitan Updated**: Bundled Yomitan is updated to upstream 26.9.8, adding historical Japanese kana transformations and Ukrainian language support, plus improvements to Anki duplicate search and audio retrieval.
- **Release Assets**: Releases no longer include package-size JSON reports.
### Fixed
- **Jellyfin**:
- Playback, subtitles, artwork, and remote control now authenticate with an `ApiKey` parameter instead of legacy headers, so Jellyfin 12 works correctly even with legacy authorization disabled.
- "Play on SubMiner" no longer silently drops the connection after about a minute on Jellyfin 12.
- Casting now honors your configured mpv executable path, so playback works and portable plugins are detected correctly when mpv isn't on PATH.
- The "now playing" bar clears when you close or finish a cast video instead of running to the end of the episode.
- Anki cards mined from Jellyfin now get the real episode title in the misc info field instead of "Unknown media".
- Jellyfin streams no longer leak URL-derived titles or credential-bearing URLs into metadata, Anki fields, Discord presence, stats, or AniList lookups; previously cached data that had credentials in it is cleaned up automatically.
- **Jellyfin Subtitles**:
- The Japanese track is selected, and annotations start, as soon as it downloads. SubMiner no longer waits for the other tracks.
- Japanese and English tracks are now picked automatically on episodes with image-based subtitles (PGS, DVD, DVB).
- If one track fails to download, the other tracks are still selected.
- **Jellyfin Audio Track**: On direct-play streams, timing review previews and mined card audio now use the audio track you're playing.
- **Media Timing Review**:
- Trim handles are now thin lines centered on the clip edges.
- Original subtitle boundaries are now plain orange bars with no labels.
- The preview cursor no longer stutters on macOS with Bluetooth headphones such as AirPods.
- **Hyprland Fullscreen**: The overlay stays visible when mpv enters or leaves fullscreen. With `mpv.backend: x11`, subtitles and the sidebar stay clickable after going fullscreen.
- **mpv Key Bindings in the Overlay**: mpv key bindings, such as `9`/`0` for volume, now work while the overlay has focus, even when the overlay loaded before mpv connected.
- **YouTube Japanese Subtitles**: Some videos also list a machine-translated Japanese track. On those, Japanese subtitles no longer fail with "HTTP 429". If YouTube refuses a direct download, SubMiner retries with yt-dlp.
- **Stats Accuracy**:
- Sentence mining no longer counts each card twice in session and lifetime stats.
- Overview totals, charts, the activity calendar, and the vocabulary summary now update after you delete sessions.
- Word and kanji dates stay correct after deleting sessions. New-word charts now match the vocabulary summary.
- **Dictionary Safety**: Automatic character dictionary changes now stop if a Yomitan profile that had dictionaries suddenly reports none. SubMiner also refuses to load Yomitan data after a downgrade to an older Electron version.
- **AppImage in Sandboxes**: The AppImage now opens its window in sandboxes that mount the image themselves, such as `firejail --appimage`.
- **Ctrl+C Crash**: Stopping the terminal command that started SubMiner with Ctrl+C no longer crashes it with a "write EPIPE" error.
- **Security Updates**: Electron is updated to 43.7.7 and undici to 7.29.1, which fixes known security advisories.
- **Anki & Mining**:
- Word audio now reads from its own configured field (`ankiConnect.fields.wordAudio`) instead of the sentence-audio field, fixing animated word images that started moving immediately instead of on demand.
- Setting `ankiConnect.media.maxMediaDuration` to `0` for unlimited duration now also applies when mining from the stats dashboard, matching overlay mining.
- Closing the overlay while a media timing review is still loading now properly cancels setup, restores playback if the review had paused it, and cleans up the hidden preview player.
- Word-card sentence furigana now stays in sync with the full stats-search context and expanded timing-review selections, and clears stale readings automatically if regeneration fails.
### Docs
- **Settings**:
- AnkiConnect, Kiku, and Senren settings are now validated before use, with a warning and a safe default for anything invalid instead of a bad value reaching runtime.
- Settings marked as applying live now correctly avoid showing a restart warning, and mixed saves apply the live parts immediately while listing only the sections that actually need a restart.
- **Overlay**:
- Clicking a subtitle sidebar cue no longer leaves Space bound to seeking back to it; Enter still seeks the focused cue, and Space keeps whatever playback action you've configured.
- Hyprland recovery dialogs now stay above SubMiner windows instead of being covered by overlay placement updates.
- Fixed a rare case on Linux where a delayed window-close callback could reopen the overlay after it was torn down.
- **Stats**:
- Malformed or partly invalid resource IDs are now rejected before they can affect library mutations or cover-art backfills.
- Stats server port conflicts now surface as a status notification instead of crashing SubMiner, and startup/shutdown are more robust: concurrent startup requests share one attempt, stopping a background instance no longer disconnects an open dashboard, and shutdown no longer waits indefinitely on active requests.
- **Subtitle Sidebar Gap Follow**: The subtitle sidebar now stays near actual playback position during gaps in files where a cue starts at time zero.
- **First Launch on macOS**: Fixed first launch exiting immediately when the SubMiner config directory didn't exist yet.
- **Docs Site Rewrite**: Pages are shorter and easier to scan. The configuration reference now has a short explanation and a table of keys and defaults for each config block. Pages that no longer matched current behavior are fixed.
- **Arch Linux Install**: The MeCab install command now uses the AUR package `mecab-git`. The README now matches the installation guide.
- **Stats**: The docs now cover the Overview refreshing after you delete sessions, and how first-seen dates are kept when old sessions are cleaned up.
## What's Changed
- feat(sidebar): add dialogue selection and copying by @ksyasuda in #238
- feat(subtitles): add local Japanese subtitle generation by @ksyasuda in #240
- perf(stats): use compressed incremental snapshot transfers by @ksyasuda in #241
- fix(startup): create config directory before singleton lock by @ksyasuda in #242
- feat(launcher): bundle Bun and use it across all launchers by @ksyasuda in #243
- build(release): reduce package size and report release sizes by @ksyasuda in #244
- fix(overlay): keep Hyprland recovery dialogs above overlays by @ksyasuda in #245
- feat(overlay): discover unclaimed mpv key bindings by @ksyasuda in #246
- fix(sidebar): preserve Space playback after cue seeking by @ksyasuda in #247
- fix(jellyfin): fix jellyfin media metadata by @ksyasuda in #250
- feat(jimaku): add live-action subtitle search by @ksyasuda in #251
- feat(stats): add TMDB metadata for live-action dramas in the Library by @ksyasuda in #252
- feat(stats): separate YouTube channels in the Library by @ksyasuda in #253
- feat(mining): add a screenshot frame picker to media review by @aalhendi in #254
- fix(config): align live save feedback with hot reload policy by @ksyasuda in #255
- fix(anki): separate word audio mapping for animation sync by @ksyasuda in #256
- fix(config): validate AnkiConnect and field grouping settings by @ksyasuda in #257
- fix(anki): honor unlimited duration in stats mining by @ksyasuda in #258
- fix(stats): reject malformed resource IDs before mutations by @ksyasuda in #259
- fix(stats): harden server lifecycle and verify compiled runtime by @ksyasuda in #261
- fix(overlay): cancel pending window transitions and timing reviews by @ksyasuda in #262
- fix(stats): restrict local requests and serve the dashboard over HTTP by @ksyasuda in #263
- fix(jellyfin): support modern authentication by @ksyasuda in #264
- feat(overlay): add optional subtitle selection modal by @ksyasuda in #265
- fix(jellyfin): respect Windows mpv configuration when casting by @aalhendi in #267
- fix(anki): regenerate sentence furigana from the final sentence by @ksyasuda in #268
## New Contributors
- @aalhendi made their first contribution in #254
- fix(storage): protect Yomitan profiles across Electron runtime changes by @ksyasuda in #211
- fix(jellyfin): select subtitles faster and survive failed track downloads by @ksyasuda in #270
- fix(jellyfin): use the current audio track for media generation by @ksyasuda in #271
- feat(youtube): add YouTube browser window and Whisper subtitle source by @ksyasuda in #273
- fix(overlay): thin centered trim handles and label-free subtitle boundaries in timing review by @ksyasuda in #274
- feat(overlay): run session help commands with Enter or double-click by @ksyasuda in #275
- fix(overlay): keep overlay mapped during Hyprland/Sway fullscreen by @ksyasuda in #276
- fix(linux): skip AppImage mount keepalive on sandbox squashfs mounts by @ksyasuda in #277
- fix(overlay): reload imported mpv keys after mpv connects by @ksyasuda in #278
- feat(overlay): forward mpv mouse button and wheel bindings by @ksyasuda in #279
- fix(stats): count mined cards once and keep lexical stats exact after deletes by @ksyasuda in #280
- fix(youtube): keep mpv queue when advancing to a queued video on Windows by @ksyasuda in #281
- fix(subtitles): allow Whisper generation during Windows YouTube stream playback by @ksyasuda in #282
## Installation
+12 -17
View File
@@ -44,20 +44,12 @@ function fragmentTypesInPrompt(input: string): string[] {
.map((line) => line.slice('type: '.length).trim());
}
function assertPromptRequestsNestedBullets(input: string): void {
assert.match(input, /In both modes, split every item into one nested bullet per distinct change/);
assert.match(input, /Never stack several distinct changes into one long paragraph-shaped bullet/);
assert.match(input, /Keep nested bullets short, concrete, and readable by non-technical users/);
assert.match(input, /Avoid paragraph-style release-note bullets/);
}
function assertReleaseNotesPromptRequestsNestedBullets(input: string): void {
assertPromptRequestsNestedBullets(input);
assert.match(
input,
/In MODE: release-notes, nested bullets should also cover user benefit and any user action/,
);
assert.match(input, /Do not require the exact nested labels/);
function assertPromptRequestsConciseItems(input: string): void {
assert.match(input, /Default to inline/);
assert.match(input, /Use nested bullets only when an item has several distinct changes/);
assert.match(input, /Never write paragraph-length bullets/);
assert.match(input, /Merge aggressively: one item per feature or area/);
assert.match(input, /Release notes summarize; the docs hold the details/);
}
function defaultPolishedBody(input: string): string {
@@ -454,14 +446,13 @@ test('writeChangelogArtifacts prompts Claude to summarize the final stable outco
prompt,
/Multiple fixes within the same prerelease cycle should collapse into one current-state bullet/,
);
assertPromptRequestsNestedBullets(prompt);
assertPromptRequestsConciseItems(prompt);
}
const releaseNotesPrompt = stub.calls.find(
(call) => modeFromPrompt(call.input) === 'release-notes',
);
assert.ok(releaseNotesPrompt, 'expected a release-notes Claude invocation');
assertReleaseNotesPromptRequestsNestedBullets(releaseNotesPrompt.input);
} finally {
fs.rmSync(workspace, { recursive: true, force: true });
}
@@ -817,7 +808,11 @@ test('writePrereleaseNotesForVersion prompts Claude to revise stale prerelease b
prompt,
/Multiple fixes within the same prerelease cycle should collapse into one current-state bullet/,
);
assertReleaseNotesPromptRequestsNestedBullets(prompt);
assert.match(
prompt,
/shorten long baseline bullets and collapse nested lists that do not need to be nested/,
);
assertPromptRequestsConciseItems(prompt);
} finally {
fs.rmSync(workspace, { recursive: true, force: true });
}
+24 -16
View File
@@ -476,21 +476,29 @@ You will receive a list of FRAGMENT entries below. Each fragment has metadata (t
</details>
Do not include the Internal section at all in MODE: release-notes; internal fragments will not be present in the input for that mode.
4. Each top-level change item should:
- Lead with a short feature/area name in title case. Pick the name from the fragment's bullet content, not the raw 'area:' slug.
- Be written in user-facing language. Drop implementation jargon, internal class names, file paths, and PR numbers.
- Be merged with related bullets when possible. If five fragments all touch Windows overlay z-order/focus/restore, write one or two bullets that summarize the overall improvement instead of five.
4. Start every item with a short bold feature/area name in title case and a colon. Pick the name from the fragment's bullet content, not the raw 'area:' slug.
- Default to inline: put one or two short sentences right after the colon. If the important details fit in two short sentences, stay inline.
- Use nested bullets only when an item has several distinct changes a user would care about separately and inline text would turn into a long run-on sentence. Then leave the top-level bullet as just the name and indent two to four nested bullets by two spaces.
- Each nested bullet is a single short sentence, roughly 20 words or fewer. Never write paragraph-length bullets. Cut minor details rather than adding sentences.
5. Keep items brief. Release notes summarize; the docs hold the details.
- Say what changed for the user and how to reach it (shortcut, command, setting, or menu). Stop there.
- Leave out how it works internally, edge cases, precedence rules, defaults, fallbacks, tool discovery, migration mechanics, and step-by-step usage.
- Name a config key only when users must set it to use the change or to keep their current behavior.
- Write in user-facing language. Drop implementation jargon, internal class names, file paths, and PR numbers.
- Merge aggressively: one item per feature or area. If five fragments touch Jellyfin subtitles, audio, and auth, write one Jellyfin item, not five. Group small unrelated fixes for the same area into one item.
- Drop bullets that only describe PR housekeeping, CodeRabbit follow-ups, or test-only changes that don't affect users.
- Preserve the substance of breaking changes that remain breaking after applying the Release Outcome Rules. Do not soften or omit them.
5. In both modes, split every item into one nested bullet per distinct change. Write a short bold name on the top-level bullet, then indent the details two spaces:
- **Playlist Browser**:
- Saved shows now open without rescanning the library.
- The picker remembers the last folder you browsed between launches.
Each nested bullet covers exactly one change, behavior, or user-visible outcome. Never stack several distinct changes into one long paragraph-shaped bullet.
Aim for two to five nested bullets per item. When an item genuinely has only one thing to say, put it inline on the top-level bullet ("- **Playlist Browser**: Saved shows now open without rescanning the library.") instead of emitting a single nested bullet.
Keep nested bullets short, concrete, and readable by non-technical users. Avoid paragraph-style release-note bullets.
Bullets inside the Internal section may stay single-level.
6. In MODE: release-notes, nested bullets should also cover user benefit and any user action or compatibility note when useful. Do not require the exact nested labels; natural phrasing is fine. Omit the action bullet when no action is needed.
- Preserve the substance of breaking changes that remain breaking after applying the Release Outcome Rules, including any action users must take. Do not soften or omit them.
6. Match the length and density of this example:
### Added
- **Subtitle Selection Modal**: Optional modal for picking primary and secondary subtitle tracks. Enable it in Settings under Behavior, then press g followed by s.
### Changed
- **Faster Sync**: Sync between macOS and Linux machines now uses compressed, incremental rsync transfers. Windows and machines without rsync fall back to compressed scp.
### Fixed
- **Subtitle Sidebar**: Space keeps its playback action after clicking a cue, and the sidebar no longer jumps away from the current position during subtitle gaps.
- **Jellyfin 12**:
- Playback, subtitles, artwork, and "Play on SubMiner" now work on Jellyfin 12.
- Casting no longer drops after about a minute.
- Mined cards get the episode title again.
7. Do not invent features. Every bullet must be grounded in the input fragments.
8. Do not include the version heading (## v...) — that wrapper is added by the caller.
@@ -791,7 +799,7 @@ function polishFragmentsWithClaude(
? [
'## Existing Prerelease Notes',
'',
'The input includes EXISTING PRERELEASE NOTES before the fragment list. Existing prerelease notes are a baseline, not an immutable changelog. Reuse reviewed highlight bullets when they still describe the current outcome, but replace stale beta or RC wording when new fragments supersede it. Merge in only new or changed fragment material, and deduplicate instead of restating existing bullets. Output only the final highlights body using the section headings above; do not include the prerelease disclaimer, any "Changes since" section, or the Installation or Assets sections.',
'The input includes EXISTING PRERELEASE NOTES before the fragment list. Existing prerelease notes are a baseline, not an immutable changelog. Reuse reviewed highlight bullets when they still describe the current outcome, but replace stale beta or RC wording when new fragments supersede it. Merge in only new or changed fragment material, and deduplicate instead of restating existing bullets. The output rules above win over the baseline format: shorten long baseline bullets and collapse nested lists that do not need to be nested. Output only the final highlights body using the section headings above; do not include the prerelease disclaimer, any "Changes since" section, or the Installation or Assets sections.',
'',
].join('\n')
: '';
@@ -814,7 +822,7 @@ Rules:
3. ADDED fragments describe changes that are new in this build; summarize them.
4. MODIFIED fragments include BEFORE and AFTER content. Describe only the behavioral difference between them. If the edit is editorial (rewording, deduplication, reformatting, reconciling stale phrasing) with no user-visible behavior change, omit it entirely.
5. DELETED fragments mean the described change was removed or reverted before this build; say so explicitly.
6. Keep bullets short and concrete. Use nested bullets sparingly.
6. Keep each bullet to one short, concrete sentence. Use nested bullets sparingly.
7. Do not invent changes. Every bullet must be grounded in the diffs.
8. If no bullet survives rules 2-5, output exactly this single line:
- No user-facing changes since PREVIOUS_TAG.
-69
View File
@@ -1,69 +0,0 @@
import fs from 'node:fs';
import path from 'node:path';
import { createHash } from 'node:crypto';
import { fileURLToPath } from 'node:url';
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const source = path.join(repoRoot, 'vendor', 'hachidori');
const output = path.join(repoRoot, 'build', 'hachidori');
const provenance = JSON.parse(fs.readFileSync(path.join(source, 'SOURCE.json'), 'utf8'));
// Upstream commits the engine binaries. Verify the pinned bytes before staging
// so ordinary app builds need neither Emscripten nor network access.
for (const [file, expected] of Object.entries(provenance.artifacts)) {
const actual = createHash('sha256')
.update(fs.readFileSync(path.join(source, file)))
.digest('hex');
if (actual !== expected) throw new Error(`Hachidori artifact checksum mismatch: ${file}`);
}
const extension = path.join(source, 'extension');
const manifest = JSON.parse(fs.readFileSync(path.join(extension, 'manifest.json'), 'utf8'));
for (const file of [
manifest.options_page,
manifest.background.service_worker,
'offscreen.html',
...manifest.content_scripts.flatMap(({ js, css }) => [...js, ...css]),
]) {
if (!fs.existsSync(path.join(extension, file)))
throw new Error(`Missing Hachidori asset: ${file}`);
}
fs.rmSync(output, { recursive: true, force: true });
fs.mkdirSync(output, { recursive: true });
fs.cpSync(extension, output, { recursive: true });
// Host configuration belongs in the staged copy, leaving the fork usable in Chrome.
const hostConfiguration = {
'overlay-mode.js': [
['export const OVERLAY_MODE = false;', 'export const OVERLAY_MODE = true;'],
['customJavaScript: true,', 'customJavaScript: false,'],
],
// Overlay hosts seed the lookup highlight off because their Anki screenshot is
// the see-through viewport. SubMiner captures media from mpv, and without the
// highlight the sidebar shows nothing for the word being looked up.
'setup-state.js': [
[
'lookupMode: "hover",\n sourceHighlightEnabled: false,',
'lookupMode: "hover",\n sourceHighlightEnabled: true,',
],
],
};
for (const [file, replacements] of Object.entries(hostConfiguration)) {
const filePath = path.join(output, file);
let text = fs.readFileSync(filePath, 'utf8');
for (const [original, replacement] of replacements) {
if (!text.includes(original))
throw new Error(`Hachidori host configuration changed upstream: ${original}`);
text = text.replace(original, replacement);
}
fs.writeFileSync(filePath, text);
}
manifest.permissions = manifest.permissions.filter((permission) => permission !== 'userScripts');
// A fixed key pins the extension ID, so Hachidori's storage origin (dictionaries,
// settings, setup state) survives userData path changes. Electron otherwise
// derives the ID from the unpacked directory path.
manifest.key =
'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA40zTTbhCbrKnDOqj17tosDChP60VXuuMjR0tFFR/kAet21jrYn0JKPzqqNpXMGaJsx1chnuMOgaP0zuOT+gv9DNVV83wGLhQL7cC4LCoGN648WqnE2jYqBPj6hdD5A/N3uPeKFRAHkiC9HKoj2Et6sYZFTqag79QqQkkUz17X2VpjuTGmTP3lvhyEwhJWQKzOU0k6dTn6fKB2ZSqTebpw/JS2G8jf58vHlQOcfLhIiolKCG9DfMuo11aoG1s3AsFmQmw9VVyp4wgyQ80UI4lyBdqboanJxEiZ211+HKWPEgtG6VML1+Uo59qWS3uBU1apxE1+7E51UBEHaTsxajFIQIDAQAB';
fs.writeFileSync(path.join(output, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
for (const file of ['LICENSE', 'SOURCE.json', 'README.md']) {
fs.copyFileSync(path.join(source, file), path.join(output, file));
}
process.stdout.write(`Hachidori ${provenance.revision} staged in ${output}\n`);
-53
View File
@@ -1,53 +0,0 @@
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import test from 'node:test';
test('Hachidori staging configures Electron without changing the fork source', async () => {
const files = ['overlay-mode.js', 'setup-state.js', 'manifest.json'];
const source = (file: string) =>
new URL(`../vendor/hachidori/extension/${file}`, import.meta.url);
const before = files.map((file) => readFileSync(source(file), 'utf8'));
execFileSync(process.execPath, [
fileURLToPath(new URL('./build-hachidori.mjs', import.meta.url)),
]);
assert.deepEqual(
files.map((file) => readFileSync(source(file), 'utf8')),
before,
);
const staged = await import(new URL('../build/hachidori/overlay-mode.js', import.meta.url).href);
assert.equal(staged.OVERLAY_MODE, true);
assert.equal(staged.HOST_CAPABILITIES.customJavaScript, false);
const original = await import(source('overlay-mode.js').href);
assert.equal(original.OVERLAY_MODE, false);
assert.equal(original.HOST_CAPABILITIES.customJavaScript, true);
// SubMiner mines media from mpv, not the overlay viewport, so the lookup
// highlight stays on for a first install.
const stagedSetup = await import(
new URL('../build/hachidori/setup-state.js', import.meta.url).href
);
assert.equal(stagedSetup.OVERLAY_MODE_OPTIONS.sourceHighlightEnabled, true);
const originalSetup = await import(source('setup-state.js').href);
assert.equal(originalSetup.OVERLAY_MODE_OPTIONS.sourceHighlightEnabled, false);
const manifest = JSON.parse(
readFileSync(new URL('../build/hachidori/manifest.json', import.meta.url), 'utf8'),
);
const originalManifest = JSON.parse(readFileSync(source('manifest.json'), 'utf8'));
assert.deepEqual(manifest, {
...originalManifest,
permissions: originalManifest.permissions.filter(
(permission: string) => permission !== 'userScripts',
),
key: manifest.key,
});
// The ID keys Hachidori's stored dictionaries and settings; changing the key orphans them.
const digest = createHash('sha256').update(Buffer.from(manifest.key, 'base64')).digest('hex');
const extensionId = [...digest.slice(0, 32)]
.map((digit) => String.fromCharCode(97 + parseInt(digit, 16)))
.join('');
assert.equal(extensionId, 'jpfgmfknblendgepdejlfhocdcnjjelj');
});
-139
View File
@@ -1,139 +0,0 @@
// Exercises the real entry point with disposable Linux config and data directories.
const { app, BrowserWindow, session, shell } = require('electron');
const fs = require('node:fs');
const path = require('node:path');
const assert = require('node:assert/strict');
if (process.platform !== 'linux')
throw new Error('This app-entry smoke requires Linux XDG isolation.');
const root = process.cwd();
const backend = process.argv.includes('--backend=yomitan') ? 'yomitan' : 'hachidori';
const externalYomitanProfile =
backend === 'hachidori' && !process.argv.includes('--bundled-yomitan');
const profile = process.env.SUBMINER_DICTIONARY_SMOKE_DATA;
assert(
profile && fs.existsSync(profile),
'Use bun run test:dictionary:electron for profile isolation',
);
process.env.XDG_CONFIG_HOME = profile;
process.env.XDG_DATA_HOME = path.join(profile, 'data');
fs.mkdirSync(path.join(profile, 'SubMiner'));
fs.writeFileSync(
path.join(profile, 'SubMiner', 'config.json'),
JSON.stringify({
dictionaryBackend: backend,
// External Yomitan profiles must not block Hachidori settings.
yomitan: {
externalProfilePath: externalYomitanProfile ? path.join(profile, 'external-yomitan') : '',
},
mpv: { socketPath: path.join(profile, 'missing-mpv.sock') },
ankiConnect: { enabled: false },
startupWarmups: { lowPowerMode: true },
discordPresence: { enabled: false },
updates: { enabled: false },
}),
);
const openedLinks = [];
shell.openExternal = async (url) => {
openedLinks.push(url);
};
app.setAppPath(root);
app.getVersion = () => require(path.join(root, 'package.json')).version;
process.env.SUBMINER_APP_LOG = path.join(profile, 'app.log');
app.commandLine.appendSwitch('ozone-platform', 'x11');
app.commandLine.appendSwitch('disable-gpu');
app.commandLine.appendSwitch('disable-dev-shm-usage');
process.argv = [process.execPath, root, '--hachidori', '--log-level', 'debug'];
require(path.join(root, 'dist/main-entry.js'));
const deadline = setTimeout(() => {
console.error(
'FAIL timeout',
BrowserWindow.getAllWindows().map((w) => w.webContents.getURL()),
);
finish(1);
}, 60000);
function finish(exitCode) {
clearTimeout(deadline);
app.exit(exitCode);
}
(async () => {
await app.whenReady();
let window;
for (let i = 0; i < 300; i++) {
window = BrowserWindow.getAllWindows().find(
(w) => w.getTitle().includes('Hachidori') && w.isVisible(),
);
if (window) break;
await new Promise((r) => setTimeout(r, 100));
}
assert.ok(window, 'Hachidori settings window opens from actual app flag');
assert.equal(window.webContents.session, session.fromPartition('persist:hachidori'));
assert.match(window.webContents.getURL(), /settings.html/);
const status = await window.webContents.executeJavaScript(
`chrome.runtime.sendMessage({target:'hoshidicts-offscreen',type:'hd_status',requestId:'app-settings-smoke'})`,
);
assert.equal(status.ok, true);
assert.equal(status.ready, true);
console.log(
'PASS actual --hachidori startup, visible settings, isolated backend session, native engine ready',
);
app.emit('second-instance', {}, [process.execPath, root, '--yomitan'], root);
if (externalYomitanProfile) {
const blockedMessage =
'Yomitan settings unavailable while using read-only external-profile mode.';
for (let i = 0; i < 100; i++) {
if (fs.readFileSync(process.env.SUBMINER_APP_LOG, 'utf8').includes(blockedMessage)) break;
await new Promise((r) => setTimeout(r, 100));
}
assert.ok(
fs.readFileSync(process.env.SUBMINER_APP_LOG, 'utf8').includes(blockedMessage),
'--yomitan is rejected when an external Yomitan profile is configured',
);
assert.equal(session.defaultSession.extensions.getAllExtensions().length, 0);
assert.ok(!BrowserWindow.getAllWindows().some((w) => w.getTitle().includes('Yomitan')));
console.log('PASS external Yomitan settings blocked while Hachidori settings remain available');
} else {
let yomi;
for (let i = 0; i < 200; i++) {
yomi = BrowserWindow.getAllWindows().find(
(w) => w.getTitle().includes('Yomitan') && w.isVisible(),
);
if (yomi) break;
await new Promise((r) => setTimeout(r, 100));
}
assert.ok(yomi, 'inactive --yomitan settings opens');
assert.equal(yomi.webContents.session, session.defaultSession);
}
assert.equal(window.webContents.session, session.fromPartition('persist:hachidori'));
app.emit('second-instance', {}, [process.execPath, root, '--toggle-visible-overlay'], root);
let overlay;
for (let i = 0; i < 200; i++) {
overlay = BrowserWindow.getAllWindows().find((w) =>
w.webContents.getURL().includes('/renderer/index.html?'),
);
if (overlay && !overlay.webContents.isLoading()) break;
await new Promise((r) => setTimeout(r, 100));
}
assert.ok(overlay, 'actual overlay initialized');
assert.equal(
overlay.webContents.session,
backend === 'hachidori' ? session.fromPartition('persist:hachidori') : session.defaultSession,
);
const requestLink = (url) =>
overlay.webContents.executeJavaScript(`new Promise((resolve,reject)=>{
const timer=setTimeout(()=>reject(Error('No external link acknowledgment')),3000);
window.addEventListener('hachidori-open-external-result',e=>{clearTimeout(timer);resolve(e.detail);},{once:true});
window.dispatchEvent(new CustomEvent('hachidori-open-external',{detail:{requestId:'smoke-link',url:${JSON.stringify(url)}}}));
})`);
assert.equal((await requestLink('file:///tmp/private')).ok, false);
assert.equal((await requestLink('https://example.com/word')).ok, backend === 'hachidori');
assert.deepEqual(openedLinks, backend === 'hachidori' ? ['https://example.com/word'] : []);
console.log(
`PASS ${backend} overlay session, independent settings windows, real preload external link bridge`,
);
finish(0);
})().catch((error) => {
console.error(error);
finish(1);
});
-239
View File
@@ -1,239 +0,0 @@
// Run after bun run build. This uses a temporary profile and small local ZIPs.
const { app, BrowserWindow, protocol, session } = require('electron');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const http = require('node:http');
const root = path.resolve(__dirname, '..');
const parser = require(path.join(root, 'dist/core/services/tokenizer/yomitan-parser-runtime.js'));
const { writeStoredZip } = require(path.join(root, 'dist/shared/stored-zip.js'));
const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-hachidori-parser-'));
protocol.registerSchemesAsPrivileged([
{
scheme: 'chrome-extension',
privileges: {
standard: true,
secure: true,
supportFetchAPI: true,
corsEnabled: true,
bypassCSP: true,
},
},
]);
app.setPath('userData', profile);
app.setAppPath(root);
app.disableHardwareAcceleration();
app.on('window-all-closed', () => {});
const deadline = setTimeout(() => {
console.error('Hachidori parser verification timed out', profile);
app.exit(1);
}, 120_000);
function fixture(name, title, bankName, entries, frequencyMode = 'rank-based') {
const zipPath = path.join(profile, name + '.zip');
writeStoredZip(zipPath, [
{
name: 'index.json',
data: Buffer.from(JSON.stringify({ title, revision: '1', format: 3, frequencyMode })),
},
{ name: bankName, data: Buffer.from(JSON.stringify(entries)) },
]);
return zipPath;
}
app
.whenReady()
.then(async () => {
const targetSession = session.fromPartition('persist:hachidori-check');
const extension = await targetSession.extensions.loadExtension(
path.join(root, 'build/hachidori'),
{ allowFileAccess: true },
);
// Keep a host window alive while the dictionary importer opens/closes its
// temporary settings windows, matching the running app's window lifecycle.
const host = new BrowserWindow({
show: false,
webPreferences: { session: targetSession, contextIsolation: true, nodeIntegration: false },
});
await host.loadURL(`chrome-extension://${extension.id}/settings.html`);
let parserWindow = null;
let readyPromise = null;
let initPromise = null;
const deps = {
getYomitanExt: () => extension,
getYomitanSession: () => targetSession,
getYomitanParserWindow: () => parserWindow,
setYomitanParserWindow: (value) => {
parserWindow = value;
},
getYomitanParserReadyPromise: () => readyPromise,
setYomitanParserReadyPromise: (value) => {
readyPromise = value;
},
getYomitanParserInitPromise: () => initPromise,
setYomitanParserInitPromise: (value) => {
initPromise = value;
},
};
const errors = [];
const logger = {
error: (...args) => {
errors.push(args);
console.error(...args);
},
};
assert.deepEqual(await parser.getYomitanDictionaryInfo(deps, logger), []);
assert.equal(
await parser.syncYomitanDefaultAnkiServer('http://127.0.0.1:18766', deps, logger, {
forceOverride: true,
deck: 'Test Mining',
}),
true,
);
// Exercise discovery and revisioned options writes in the real extension,
// without creating a note or changing the user's Anki collection.
const fields = ['Term', 'Reading', 'Definition', 'Context', 'Pronunciation', 'Image'];
const metadataServer = http.createServer((request, response) => {
let body = '';
request.on('data', (chunk) => {
body += chunk;
});
request.on('end', () => {
const message = JSON.parse(body);
assert.equal(message.action, 'multi');
const result = message.params.actions.map(({ action }) => ({
result:
action === 'deckNames'
? ['Test Mining']
: action === 'modelNames'
? ['Custom Japanese']
: fields,
error: null,
}));
response.setHeader('Content-Type', 'application/json');
response.end(JSON.stringify({ result, error: null }));
});
});
await new Promise((resolve) => metadataServer.listen(0, '127.0.0.1', resolve));
try {
const url = `http://127.0.0.1:${metadataServer.address().port}`;
const ankiConfig = {
tags: ['SubMiner', 'Autofill'],
fields: {
word: 'Term',
sentence: 'Context',
wordAudio: 'Pronunciation',
image: 'Image',
},
};
assert.equal(
await parser.syncYomitanDefaultAnkiServer(url, deps, logger, {
forceOverride: true,
deck: 'Test Mining',
ankiConfig,
}),
true,
);
const anki = await parserWindow.webContents.executeJavaScript(
`(async () => (await chrome.storage.local.get('options')).options.anki)()`,
);
assert.equal(anki.model, 'Custom Japanese');
assert.equal(anki.deck, 'Test Mining');
assert.deepEqual(anki.tags, ankiConfig.tags);
assert.equal(anki.fieldTemplates.Term.value, '{expression}');
assert.equal(anki.fieldTemplates.Pronunciation.value, '{audio}');
assert.equal(anki.fieldTemplates.Context.value, '{sentence}');
assert.deepEqual(anki.templates[0].fieldTemplates, anki.fieldTemplates);
console.log('Hachidori Anki auto-population passed with an empty dictionary library');
} finally {
await new Promise((resolve) => metadataServer.close(resolve));
}
const archives = [
fixture('terms', 'SubMiner Test Terms', 'term_bank_1.json', [
['食べる', 'たべる', '', 'v1', 0, ['to eat'], 1, ''],
]),
fixture('names', 'SubMiner Character Dictionary (AniList 1)', 'term_bank_1.json', [
['ミナト', 'みなと', '', 'n', 0, ['name'], 1, ''],
]),
fixture('frequency', 'SubMiner Test Frequency', 'term_meta_bank_1.json', [
['食べる', 'freq', { reading: 'たべる', frequency: 42 }],
['頻度だけ', 'freq', { reading: 'ひんどだけ', frequency: 120 }],
['頻度だけ', 'freq', { reading: 'べつのよみ', frequency: 250 }],
['頻度だけ', 'freq', 17],
]),
fixture(
'occurrences',
'SubMiner Test Occurrences',
'term_meta_bank_1.json',
[['頻度だけ', 'freq', 9000]],
'occurrence-based',
),
];
for (const archive of archives)
assert.equal(await parser.importYomitanDictionaryFromZip(archive, deps, logger), true);
const dictionaries = await parser.getYomitanDictionaryInfo(deps, logger);
assert.equal(dictionaries.length, 4);
parser.clearYomitanParserCachesForWindow(parserWindow);
const tokens = await parser.requestYomitanScanTokens('ミナト 食べた', deps, logger, {
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
});
assert.equal(tokens[0].isNameMatch, true);
assert.equal(tokens[1].headword, '食べる');
assert.equal(tokens[1].reading, 'たべた');
assert.equal(tokens[1].startPos, 4);
assert.equal(tokens[1].endPos, 7);
assert.equal(tokens[1].frequencyRank, 42);
assert.deepEqual(tokens[1].wordClasses, ['v1']);
const exact = await parser.requestYomitanTermFrequencies(
[{ term: '食べる', reading: 'たべる' }],
deps,
logger,
);
assert.equal(exact.length, 1);
assert.equal(exact[0].frequency, 42);
assert.equal(exact[0].reading, 'たべる');
assert.equal(exact[0].hasReading, false);
const otherReading = await parser.requestYomitanTermFrequencies(
[{ term: '食べる', reading: 'べつのよみ' }],
deps,
logger,
);
// The shared frequency pipeline retries a missing reading as a term-only query.
assert.equal(otherReading[0]?.frequency, 42);
const unmatched = await parser.requestYomitanTermFrequencies(
[{ term: '頻度だけ', reading: null }],
deps,
logger,
);
assert.deepEqual(unmatched, []);
assert.equal(await parser.getYomitanCurrentAnkiDeckName(deps, logger), 'Test Mining');
assert.equal((await targetSession.extensions.getAllExtensions()).length, 1);
assert.equal(
await parser.syncYomitanDefaultAnkiServer('http://127.0.0.1:8765', deps, logger),
true,
);
const directSettings = await parser.getYomitanSettingsFull(deps, logger);
assert.equal(directSettings.profiles[0].options.anki.server, 'http://127.0.0.1:8765');
const proxyState = await host.webContents.executeJavaScript(
"chrome.storage.local.get('subminerAnkiProxyUrl')",
);
assert.equal(proxyState.subminerAnkiProxyUrl, null);
for (const entry of await parser.getYomitanDictionaryInfo(deps, logger)) {
assert.equal(await parser.deleteYomitanDictionaryByTitle(entry.title, deps, logger), true);
}
assert.deepEqual(await parser.getYomitanDictionaryInfo(deps, logger), []);
assert.equal(errors.length, 0);
console.log(
'PASS Hachidori native import, scanner, character names, term-entry API frequencies, settings and removal',
);
clearTimeout(deadline);
app.exit(0);
})
.catch((error) => {
console.error(error);
clearTimeout(deadline);
app.exit(1);
});
-137
View File
@@ -1,137 +0,0 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import '../vendor/hachidori/extension/reader-options.js';
import { createAnkiGateway } from '../vendor/hachidori/extension/anki.js';
import { createAnkiWorkerService } from '../vendor/hachidori/extension/anki-worker.js';
async function mine({ proxy = true, audioFails = false } = {}) {
const events: string[] = [];
const filename = `hachidori_${'a'.repeat(64)}.mp3`;
let fields: Record<string, string> = {};
let initialAudio = '';
let stored = false;
const gateway = createAnkiGateway({
readSubminerProxyUrl: async () => (proxy ? 'http://127.0.0.1:8766' : null),
fetch: async (_url: string, options: RequestInit) => {
const { action, params } = JSON.parse(String(options.body));
events.push(action);
let result: unknown;
switch (action) {
case 'multi':
result = [
{ result: ['Default'], error: null },
{ result: ['Basic'], error: null },
{ result: ['Expression', 'ExpressionAudio'], error: null },
];
break;
case 'canAddNotes':
result = [true];
break;
case 'canAddNotesWithErrorDetail':
result = [{ canAdd: true, error: null }];
break;
case 'getMediaFilesNames':
result = stored ? [filename] : [];
break;
case 'storeMediaFile':
stored = true;
result = filename;
break;
case 'addNote':
fields = { ...params.note.fields };
initialAudio = fields.ExpressionAudio ?? '';
if (initialAudio) assert.ok(stored, 'audio must exist before the note references it');
result = 123;
break;
case 'notesInfo':
result = [
{
noteId: 123,
fields: Object.fromEntries(
Object.entries(fields).map(([key, value]) => [key, { value }]),
),
},
];
break;
case 'updateNoteFields':
Object.assign(fields, params.note.fields);
result = null;
break;
default:
throw new Error(`Unexpected action: ${action}`);
}
return Response.json({ result, error: null });
},
});
const service = createAnkiWorkerService({
gateway,
readOptions: async () => ({
anki: {
url: 'http://127.0.0.1:8766',
apiKey: '',
templates: [
{
id: 'default',
name: 'Default',
deck: 'Default',
model: 'Basic',
tags: [],
fields: {},
duplicateScope: 'model',
duplicateBehavior: 'prevent',
captureScreenshot: false,
fieldTemplates: {
Expression: { value: '{expression}', overwriteMode: 'overwrite' },
ExpressionAudio: { value: '{audio}', overwriteMode: 'overwrite' },
},
},
],
},
audioSources: [
{ id: 'test', enabled: true, type: 'custom', url: 'https://example.test/{term}' },
],
experimental: {},
}),
readDictionaries: async () => [],
engine: async () => ({ ready: true, loading: false, generation: 1 }),
offscreen: async (message: { type: string; audio?: string }) => {
if (message.type === 'hd_anki_audio') {
events.push('pronunciation');
if (audioFails) throw new Error('No pronunciation available');
return { filename, data: 'YXVkaW8=' };
}
return { fields: { Expression: '猫', ExpressionAudio: message.audio ?? '' }, media: [] };
},
duplicateIndex: { source: async () => null, recordWrite: async () => {} },
});
const status = await service.status();
assert.equal(status.available, true, status.error);
const result = await service.submit({
configKey: status.configKey,
generation: 1,
term: { expression: '猫', reading: 'ねこ' },
});
return { result, initialAudio, fields, events, filename };
}
test('SubMiner receives pronunciation in the initial Hachidori note, before enrichment starts', async () => {
const value = await mine();
assert.equal(value.result.state, 'added');
assert.equal(value.initialAudio, `[sound:${value.filename}]`);
assert.equal(value.events.filter((event) => event === 'pronunciation').length, 1);
});
test('direct Hachidori keeps deferred pronunciation', async () => {
const value = await mine({ proxy: false });
assert.equal(value.initialAudio, '');
assert.equal(value.fields.ExpressionAudio, `[sound:${value.filename}]`);
});
test('unavailable pronunciation remains a warning without a late audio write', async () => {
const value = await mine({ audioFails: true });
assert.equal(value.result.state, 'added');
assert.match(value.result.warnings.join(' '), /No pronunciation available/);
assert.equal(value.initialAudio, '');
assert.equal(value.events.filter((event) => event === 'pronunciation').length, 1);
assert.equal(value.events.includes('updateNoteFields'), false);
});
-132
View File
@@ -1,132 +0,0 @@
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { runInNewContext } from 'node:vm';
import test from 'node:test';
import { CHARACTER_DICTIONARY_TITLE_PREFIX } from '../src/core/services/tokenizer/character-dictionary-title';
import { createAnkiGateway } from '../vendor/hachidori/extension/anki.js';
const bridge = readFileSync(
new URL('../vendor/hachidori/extension/subminer-host.js', import.meta.url),
'utf8',
);
function run(code: string) {
runInNewContext(`${bridge}\n${code}`, {
window: new EventTarget(),
EventTarget,
CustomEvent,
assert,
characterPrefix: CHARACTER_DICTIONARY_TITLE_PREFIX,
KeyboardEvent: class {
constructor(_type: string, options: KeyboardEventInit = {}) {
Object.assign(this, options);
}
},
});
}
test('Hachidori marks popup state independently from successful lookups', () => {
run(`
const events = [];
for (const name of ['yomitan-popup-shown', 'yomitan-popup-hidden', 'subminer-yomitan-lookup']) {
window.addEventListener(name, () => events.push(name));
}
const attributes = new Map();
const host = { setAttribute: (name, value) => attributes.set(name, value) };
SubMinerHachidori.markHost(host, true);
assert.equal(attributes.get('data-subminer-yomitan-popup-visible'), 'true');
assert.equal(events.join(','), '');
SubMinerHachidori.lookup();
SubMinerHachidori.markHost(host, false);
assert.equal(attributes.get('data-subminer-yomitan-popup-visible'), 'false');
assert.equal(events.join(','), 'subminer-yomitan-lookup');
`);
});
test('Hachidori promotes character glossaries and results without overriding linguistic rank', () => {
run(`
const result = (id, matched, dictionary, options = {}) => ({
matched, deinflected: matched, trace: [], preprocessorSteps: 0,
term: { expression: matched, reading: 'reading', glossaries: [{ dictionary }] }, ...options, id,
});
const character = characterPrefix + ' - Current show';
const longer = result('longer', '花子さん', 'General');
const normal = result('normal', '花子', 'General');
const person = result('person', '花子', character);
const shorter = result('shorter', '花', character);
const output = SubMinerHachidori.prioritizeCharacterResults([longer, normal, person, shorter]);
assert.equal(output.map(entry => entry.id).join(','), 'longer,person,normal,shorter');
const merged = result('merged', '花子', 'General');
merged.term.glossaries.push({ dictionary: character }, { dictionary: 'Second general' });
const [promoted] = SubMinerHachidori.prioritizeCharacterResults([merged]);
assert.equal(promoted.term.glossaries.map(g => g.dictionary).join(','), character + ',General,Second general');
assert.equal(merged.term.glossaries[0].dictionary, 'General');
for (const change of [{ preprocessorSteps: 1 }, { trace: [{}] }, { deinflected: '別の語' }]) {
const transformed = { ...person, ...change };
assert.equal(SubMinerHachidori.prioritizeCharacterResults([normal, transformed])[0].id, 'normal');
}
const preferredReading = { ...normal, term: { ...normal.term, reading: 'preferred' } };
assert.equal(SubMinerHachidori.prioritizeCharacterResults([person, preferredReading], {primaryReading: 'preferred'})[0].id, 'normal');
const aliased = result('aliased', '花子', 'Imported character data');
assert.equal(SubMinerHachidori.prioritizeCharacterResults([normal, aliased], {}, [{title: 'Imported character data', displayName: character}])[0].id, 'aliased');
`);
});
test('Hachidori sends private mining metadata only to the configured SubMiner proxy', async () => {
const requests: Array<{ url: string; body: string }> = [];
const proxy = 'http://127.0.0.1:8766';
const gateway = createAnkiGateway({
readSubminerProxyUrl: async () => proxy,
fetch: async (url: string, options: RequestInit) => {
assert.equal(typeof options.body, 'string');
requests.push({ url, body: String(options.body) });
return Response.json({ result: 123, error: null });
},
});
const params = {
note: { fields: { Expression: '花子' } },
subminerDuplicateNoteIds: [456],
subminerEnrich: true,
};
await gateway.invoke('addNote', params, '', 1000, proxy);
await gateway.invoke('addNote', params, '', 1000, 'http://127.0.0.1:8765');
assert.deepEqual(
requests.map((request) => JSON.parse(request.body).params),
[params, { note: params.note }],
);
assert.equal(params.subminerEnrich, true);
const direct = createAnkiGateway({
readSubminerProxyUrl: async () => null,
fetch: async (_url: string, options: RequestInit) => {
assert.deepEqual(JSON.parse(String(options.body)).params, { note: params.note });
return Response.json({ result: null, error: null });
},
});
await direct.invoke('updateNoteFields', params, '', 1000, proxy);
});
test('Hachidori routes host commands, validates keyboard input and disconnects', () => {
run(`
const calls = [];
const disconnect = SubMinerHachidori.connect({
hide: () => calls.push('hide'), clear: () => calls.push('clear'),
action: name => calls.push(name), cycleAudio: direction => calls.push(direction),
scroll: (x, y) => calls.push(x + ':' + y),
keydown: event => calls.push(event.key + ':' + event.ctrlKey + ':' + event.shiftKey),
});
const send = detail => window.dispatchEvent(new CustomEvent('subminer-yomitan-popup-command', {detail}));
send(null);
send({ type: 'forwardKeyDown', key: 12, modifiers: [] });
send({ type: 'mineSelected' });
send({ type: 'playCurrentAudio' });
send({ type: 'scanSelectedText' });
send({ type: 'cycleAudioSource', direction: -1 });
send({ type: 'scrollBy', deltaX: Infinity, deltaY: 40 });
send({ type: 'forwardKeyDown', key: 'j', code: 'KeyJ', modifiers: ['ctrl', 'shift'] });
send({ type: 'setVisible', visible: false });
send({ type: 'clearActiveTextSource' });
disconnect();
send({ type: 'mineSelected' });
assert.equal(calls.join(','), 'addNote,playAudio,scanSelectedText,-1,0:40,j:true:true,hide,clear');
`);
});
-8
View File
@@ -26,14 +26,6 @@ const REQUIRED_APP_FILES = [
]),
];
const REQUIRED_RESOURCES = [
'hachidori/manifest.json',
'hachidori/settings.html',
'hachidori/subminer-host.js',
'hachidori/vendor/hoshidicts.wasm',
'hachidori/vendor/hoshidicts-threaded.wasm',
'hachidori/vendor/hoshidicts-threaded-idbfs.wasm',
'hachidori/LICENSE',
'hachidori/SOURCE.json',
'yomitan/manifest.json',
'yomitan/data/fonts/kanji-stroke-orders.ttf',
'yomitan/fonts/NotoSansJP-Regular.ttf',
-26
View File
@@ -1,26 +0,0 @@
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawnSync } from 'node:child_process';
import { createRequire } from 'node:module';
import { fileURLToPath } from 'node:url';
const require = createRequire(import.meta.url);
const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-hachi-settings-'));
const env = { ...process.env, SUBMINER_DICTIONARY_SMOKE_DATA: profile };
delete env.ELECTRON_RUN_AS_NODE;
try {
const result = spawnSync(
require('electron'),
[
'--no-sandbox',
fileURLToPath(new URL('./check-dictionary-backends.cjs', import.meta.url)),
...process.argv.slice(2),
],
{ env, stdio: 'inherit', timeout: 75_000 },
);
if (result.error) throw result.error;
process.exitCode = result.status ?? 1;
} finally {
fs.rmSync(profile, { recursive: true, force: true, maxRetries: 3 });
}
+49 -45
View File
@@ -22,6 +22,39 @@ const timeout = setTimeout(() => {
app.exit(1);
}, 60_000);
const STATIC_TYPES = {
'.html': 'text/html',
'.js': 'text/javascript',
'.css': 'text/css',
'.png': 'image/png',
'.svg': 'image/svg+xml',
'.woff2': 'font/woff2',
'.ttf': 'font/ttf',
'.json': 'application/json',
};
// The stats dashboard is served by the stats HTTP server in the app, so load it
// over loopback HTTP from the packaged stats/dist and treat missing static
// assets as failures. API routes are not part of this smoke and may 404.
function serveStatsDist(root, failedRequests) {
const server = http.createServer((req, res) => {
const pathname = new URL(req.url, 'http://127.0.0.1').pathname;
const relative = pathname === '/' ? 'index.html' : pathname.slice(1);
try {
const body = fs.readFileSync(path.join(root, relative));
res.writeHead(200, {
'Content-Type': STATIC_TYPES[path.extname(relative)] ?? 'application/octet-stream',
});
res.end(body);
} catch {
if (!pathname.startsWith('/api/')) failedRequests.push(`${req.url}: missing static asset`);
res.writeHead(404).end();
}
});
server.listen(0, '127.0.0.1');
return server;
}
async function smoke() {
await app.whenReady();
const packagedRequire = createRequire(path.join(archive, 'package.json'));
@@ -50,10 +83,6 @@ async function smoke() {
{ allowFileAccess: true },
);
assert(extension.id, 'Yomitan extension failed to load');
const hachidori = await session
.fromPartition('persist:hachidori')
.extensions.loadExtension(path.join(resources, 'hachidori'), { allowFileAccess: true });
assert(hachidori.id, 'Hachidori extension failed to load');
const failedRequests = [];
session.defaultSession.webRequest.onErrorOccurred(
{ urls: ['file://*/*', 'http://127.0.0.1/*'] },
@@ -64,7 +93,9 @@ async function smoke() {
failedRequests.push(`${details.url}: ${details.error}`);
},
);
for (const ui of ['renderer', 'settings', 'syncui']) {
const statsServer = serveStatsDist(path.join(archive, 'stats', 'dist'), failedRequests);
await once(statsServer, 'listening');
for (const ui of ['renderer', 'settings', 'syncui', 'stats']) {
const win = new BrowserWindow({
show: false,
webPreferences: {
@@ -73,52 +104,25 @@ async function smoke() {
},
});
try {
await win.loadFile(path.join(archive, `dist/${ui}/index.html`));
const loaded = await win.webContents.executeJavaScript(
`document.fonts.load('400 16px "M PLUS 1"', '日本語').then(fonts => fonts.length > 0 && fonts.every(font => font.status === 'loaded'))`,
);
assert(loaded, `${ui}: shared Japanese font failed to load`);
if (ui === 'stats') {
await win.loadURL(`http://127.0.0.1:${statsServer.address().port}/`);
// Let in-flight font requests settle before the window goes away.
await win.webContents.executeJavaScript('document.fonts.ready.then(() => true)');
} else {
await win.loadFile(path.join(archive, `dist/${ui}/index.html`));
const loaded = await win.webContents.executeJavaScript(
`document.fonts.load('400 16px "M PLUS 1"', '日本語').then(fonts => fonts.length > 0 && fonts.every(font => font.status === 'loaded'))`,
);
assert(loaded, `${ui}: shared Japanese font failed to load`);
}
} finally {
win.destroy();
}
}
// The stats dashboard uses HTTP for both assets and API requests in the app.
const { ImmersionTrackerService } = packagedRequire(
'./dist/core/services/immersion-tracker-service.js',
);
const { createStatsApp, startNodeHttpServer } = packagedRequire(
'./dist/core/services/stats-server.js',
);
const tracker = new ImmersionTrackerService({ dbPath: path.join(isolatedData, 'stats.db') });
const statsConfig = { port: 0, staticDir: path.join(archive, 'stats/dist'), tracker };
let statsHttp;
const statsServer = await startNodeHttpServer(
createStatsApp(tracker, statsConfig),
statsConfig,
(listener) => (statsHttp = http.createServer(listener)),
);
const statsWindow = new BrowserWindow({ show: false });
try {
const url = `http://127.0.0.1:${statsHttp.address().port}`;
session.defaultSession.webRequest.onCompleted({ urls: [`${url}/*`] }, (details) => {
if (details.statusCode >= 400) failedRequests.push(`${details.url}: ${details.statusCode}`);
});
await statsWindow.loadURL(url);
// Let in-flight font requests settle before the window goes away.
await statsWindow.webContents.executeJavaScript('document.fonts.ready.then(() => true)');
for (const endpoint of ['overview', 'sessions']) {
const response = await fetch(`${url}/api/stats/${endpoint}`);
assert.equal(response.status, 200, `Stats ${endpoint} request failed`);
await response.json();
}
} finally {
statsWindow.destroy();
await statsServer.close();
tracker.destroy();
}
statsServer.close();
assert.deepEqual(failedRequests, [], 'Packaged UI resources failed to load');
console.log(
'Package smoke passed: SQLite, platform FFI, texthooker, both dictionary extensions, UI pages, stats HTTP, shared Japanese font.',
'Package smoke passed: SQLite, platform FFI, texthooker, Yomitan loading, UI pages, shared Japanese font.',
);
}
-3
View File
@@ -1807,9 +1807,6 @@ export class AnkiIntegration {
request: Omit<MediaTimingReviewRequest, 'audioPadding' | 'maxMediaDuration'>,
): Promise<MediaTimingReviewDecision> {
if (this.config.media?.reviewTiming !== true || !this.mediaTimingReviewCallback) {
log.debug(
`[media-timing] review skipped: reviewTiming=${String(this.config.media?.reviewTiming)} callback=${this.mediaTimingReviewCallback ? 'set' : 'missing'}`,
);
return { action: 'use-original' };
}
return await this.mediaTimingReviewCallback({
@@ -441,127 +441,6 @@ test('proxy strips SubMiner duplicate metadata before forwarding upstream addNot
}
});
test('proxy enriches confirmed Hachidori overwrites without counting a new card or forwarding metadata', async () => {
const received: unknown[] = [];
let upstreamError: string | null = null;
const upstream = http.createServer(async (req, res) => {
const chunks: Buffer[] = [];
for await (const chunk of req) chunks.push(Buffer.from(chunk));
received.push(JSON.parse(Buffer.concat(chunks).toString()));
res.setHeader('content-type', 'application/json');
res.end(JSON.stringify({ result: null, error: upstreamError }));
});
upstream.listen(0, '127.0.0.1');
await once(upstream, 'listening');
const address = upstream.address();
assert.ok(address && typeof address === 'object');
const processed: number[] = [];
const added: number[] = [];
const proxy = new AnkiConnectProxyServer({
shouldAutoUpdateNewCards: () => true,
processNewCard: async (id) => {
processed.push(id);
},
recordCardsAdded: (count) => {
added.push(count);
},
logInfo: () => {},
logWarn: () => {},
logError: () => {},
});
try {
proxy.start({ host: '127.0.0.1', port: 0, upstreamUrl: `http://127.0.0.1:${address.port}` });
await proxy.waitUntilReady();
const server: unknown = Reflect.get(proxy, 'server');
assert.ok(server instanceof http.Server);
const bound = server.address();
assert.ok(bound && typeof bound === 'object');
for (const [id, marked, error] of [
[51, true, null],
[52, false, null],
[53, true, 'failed'],
] satisfies Array<[number, boolean, string | null]>) {
upstreamError = error;
await fetch(`http://127.0.0.1:${bound.port}`, {
method: 'POST',
body: JSON.stringify({
action: 'updateNoteFields',
version: 6,
params: {
note: { id, fields: { Expression: '猫' } },
...(marked ? { subminerEnrich: true } : {}),
},
}),
});
}
await waitForCondition(() => processed.length > 0);
assert.deepEqual(processed, [51]);
assert.deepEqual(added, []);
assert.equal(JSON.stringify(received).includes('subminerEnrich'), false);
} finally {
proxy.stop();
upstream.close();
await once(upstream, 'close');
}
});
test('stats-owned notes bypass overlay enrichment while popup notes still enqueue', async () => {
const processed: number[] = [];
const added: number[] = [];
const received: unknown[] = [];
let noteId = 70;
const upstream = http.createServer(async (req, res) => {
const chunks: Buffer[] = [];
for await (const chunk of req) chunks.push(Buffer.from(chunk));
received.push(JSON.parse(Buffer.concat(chunks).toString()));
res.end(JSON.stringify({ result: ++noteId, error: null }));
});
upstream.listen(0, '127.0.0.1');
await once(upstream, 'listening');
const address = upstream.address();
assert.ok(address && typeof address === 'object');
const proxy = new AnkiConnectProxyServer({
shouldAutoUpdateNewCards: () => true,
processNewCard: async (id) => {
processed.push(id);
},
recordCardsAdded: (_count, ids) => {
added.push(...ids);
},
logInfo: () => {},
logWarn: () => {},
logError: () => {},
});
try {
proxy.start({ host: '127.0.0.1', port: 0, upstreamUrl: `http://127.0.0.1:${address.port}` });
await proxy.waitUntilReady();
const server: unknown = Reflect.get(proxy, 'server');
assert.ok(server instanceof http.Server);
const bound = server.address();
assert.ok(bound && typeof bound === 'object');
for (const metadata of [{ subminerEnrich: false }, {}]) {
const response: Response = await fetch(`http://127.0.0.1:${bound.port}`, {
method: 'POST',
body: JSON.stringify({
action: 'addNote',
version: 6,
params: { note: { fields: { Expression: '猫' } }, ...metadata },
}),
});
assert.equal(response.status, 200);
await response.json();
}
await waitForCondition(() => processed.includes(72));
assert.deepEqual(processed, [72]);
assert.deepEqual(added, [71, 72]);
assert.equal(JSON.stringify(received).includes('subminerEnrich'), false);
} finally {
proxy.stop();
upstream.close();
await once(upstream, 'close');
}
});
test('proxy returns addNote response without waiting for background enrichment', async () => {
const processed: number[] = [];
let releaseProcessing: (() => void) | undefined;
+6 -47
View File
@@ -224,31 +224,6 @@ export class AnkiConnectProxyServer {
typeof requestJson.action === 'string'
? requestJson.action
: String(requestJson.action ?? '');
if (action === 'updateNoteFields') {
const params = requestJson.params;
if (
!params ||
typeof params !== 'object' ||
!('subminerEnrich' in params) ||
params.subminerEnrich !== true
)
return;
const note = 'note' in params ? params.note : null;
if (!note || typeof note !== 'object' || !('id' in note)) return;
const response = this.tryParseJsonValue(responseBody);
// AnkiConnect confirms updates with {result:null,error:null}; failures must never enrich.
if (
!response ||
typeof response !== 'object' ||
!('error' in response) ||
response.error !== null ||
!('result' in response) ||
response.result !== null
)
return;
this.enqueueNotes(this.collectSingleResultId(note.id), false);
return;
}
if (action !== 'addNote' && action !== 'addNotes' && action !== 'multi') {
return;
}
@@ -264,23 +239,12 @@ export class AnkiConnectProxyServer {
return;
}
this.maybeTrackDuplicateNoteIds(requestJson, action, responseResult);
const noteIds =
action === 'multi'
? this.collectMultiResultIds(requestJson, responseResult)
: this.collectNoteIdsForAction(action, responseResult);
const params = requestJson.params;
if (
action === 'addNote' &&
params &&
typeof params === 'object' &&
'subminerEnrich' in params &&
params.subminerEnrich === false
) {
// Stats owns the saved sentence and media; the live mpv context is unrelated.
if (noteIds.length > 0) this.deps.recordCardsAdded?.(noteIds.length, noteIds);
return;
}
this.maybeTrackDuplicateNoteIds(requestJson, action, responseResult);
if (noteIds.length === 0 && shouldFallbackToLatestAdded) {
void this.enqueueMostRecentAddedNote();
return;
@@ -329,7 +293,7 @@ export class AnkiConnectProxyServer {
typeof requestJson.action === 'string'
? requestJson.action
: String(requestJson.action ?? '');
if (action !== 'addNote' && action !== 'updateNoteFields') {
if (action !== 'addNote') {
return requestJson;
}
@@ -337,17 +301,12 @@ export class AnkiConnectProxyServer {
requestJson.params && typeof requestJson.params === 'object'
? (requestJson.params as Record<string, unknown>)
: null;
if (
!params ||
(!Object.prototype.hasOwnProperty.call(params, 'subminerDuplicateNoteIds') &&
!Object.prototype.hasOwnProperty.call(params, 'subminerEnrich'))
) {
if (!params || !Object.prototype.hasOwnProperty.call(params, 'subminerDuplicateNoteIds')) {
return requestJson;
}
const nextParams = { ...params };
delete nextParams.subminerDuplicateNoteIds;
delete nextParams.subminerEnrich;
return {
...requestJson,
params: nextParams,
@@ -496,7 +455,7 @@ export class AnkiConnectProxyServer {
});
}
private enqueueNotes(noteIds: number[], recordAdded = true): void {
private enqueueNotes(noteIds: number[]): void {
let enqueuedCount = 0;
const acceptedIds: number[] = [];
for (const noteId of noteIds) {
@@ -513,7 +472,7 @@ export class AnkiConnectProxyServer {
return;
}
if (recordAdded) this.deps.recordCardsAdded?.(enqueuedCount, acceptedIds);
this.deps.recordCardsAdded?.(enqueuedCount, acceptedIds);
this.deps.logInfo(`[anki-proxy] Enqueued ${enqueuedCount} note(s) for enrichment`);
this.processQueue();
}
@@ -8,7 +8,6 @@ import {
import type { SubtitleMiningContext } from '../types/subtitle';
import type { CardKind } from '../types/anki';
import { applyCardKindFlagFields } from './card-kinds';
import { STATS_MINING_TAG } from '../shared/anki-source';
function setCardTypeFields(
updatedFields: Record<string, string>,
@@ -111,27 +110,6 @@ function createWorkflowHarness() {
};
}
test('NoteUpdateWorkflow preserves stats cards discovered by polling', async () => {
const { workflow, deps, updates } = createWorkflowHarness();
const note = {
noteId: 42,
tags: [STATS_MINING_TAG],
fields: { Expression: { value: '猫' }, Sentence: { value: '猫がいる。' } },
};
deps.client.notesInfo = async () => [note];
deps.captureSubtitleMediaContext = () => assert.fail('Must not capture current playback');
deps.findDuplicateNote = async () => assert.fail('Must not regroup a stats card');
let cachedNote: NoteUpdateWorkflowNoteInfo | undefined;
deps.appendKnownWordsFromNoteInfo = (value) => {
cachedNote = value;
};
await workflow.execute(42);
assert.deepEqual(updates, []);
assert.equal(cachedNote, note);
});
test('NoteUpdateWorkflow updates sentence field and emits notification', async () => {
const harness = createWorkflowHarness();
@@ -8,11 +8,9 @@ import type {
WordCardKind,
} from '../types/anki';
import { resolveWordCardKind } from './note-field-utils';
import { STATS_MINING_TAG } from '../shared/anki-source';
export interface NoteUpdateWorkflowNoteInfo {
noteId: number;
tags?: string[];
fields: Record<string, { value: string }>;
}
@@ -187,10 +185,6 @@ export class NoteUpdateWorkflow {
}
const noteInfo = notesInfo[0]!;
if (noteInfo.tags?.includes(STATS_MINING_TAG)) {
this.deps.appendKnownWordsFromNoteInfo(noteInfo);
return;
}
const fields = this.deps.extractFields(noteInfo.fields);
const config = this.deps.getConfig();
-19
View File
@@ -8,7 +8,6 @@ import {
isStandaloneTexthookerCommand,
parseArgs,
shouldRunYomitanOnlyStartup,
shouldRunDictionarySettingsOnlyStartup,
shouldStartApp,
} from './args';
@@ -477,21 +476,3 @@ test('hasExplicitCommand and shouldStartApp preserve command intent', () => {
assert.equal(hasExplicitCommand(setup), true);
assert.equal(shouldStartApp(setup), true);
});
test('Hachidori settings starts the app without overlay or mpv prerequisites', () => {
const args = parseArgs(['--hachidori']);
assert.equal(args.hachidori, true);
assert.equal(args.yomitan, false);
assert.equal(hasExplicitCommand(args), true);
assert.equal(shouldStartApp(args), true);
assert.equal(shouldRunDictionarySettingsOnlyStartup(args), true);
assert.equal(shouldRunYomitanOnlyStartup(args), false);
assert.equal(commandNeedsOverlayRuntime(args), false);
assert.equal(commandNeedsOverlayStartupPrereqs(args), false);
assert.equal(isStandaloneTexthookerCommand(parseArgs(['--texthooker', '--hachidori'])), false);
assert.equal(
shouldRunDictionarySettingsOnlyStartup(parseArgs(['--hachidori', '--start'])),
false,
);
assert.equal(shouldRunDictionarySettingsOnlyStartup(parseArgs(['--yomitan'])), true);
});
+1 -11
View File
@@ -12,7 +12,6 @@ export interface CliArgs {
toggleVisibleOverlay: boolean;
togglePrimarySubtitleBar: boolean;
yomitan: boolean;
hachidori: boolean;
settings: boolean;
syncWindow: boolean;
youtubeBrowser: boolean;
@@ -134,7 +133,6 @@ export function parseArgs(argv: string[]): CliArgs {
toggleVisibleOverlay: false,
togglePrimarySubtitleBar: false,
yomitan: false,
hachidori: false,
settings: false,
syncWindow: false,
youtubeBrowser: false,
@@ -281,7 +279,6 @@ export function parseArgs(argv: string[]): CliArgs {
else if (arg === '--toggle-visible-overlay') args.toggleVisibleOverlay = true;
else if (arg === '--toggle-primary-subtitle-bar') args.togglePrimarySubtitleBar = true;
else if (arg === '--yomitan') args.yomitan = true;
else if (arg === '--hachidori') args.hachidori = true;
else if (arg === '--settings') args.settings = true;
else if (arg === '--sync-window') args.syncWindow = true;
else if (arg === '--youtube-browser') args.youtubeBrowser = true;
@@ -566,7 +563,6 @@ export function hasExplicitCommand(args: CliArgs): boolean {
args.toggleVisibleOverlay ||
args.togglePrimarySubtitleBar ||
args.yomitan ||
args.hachidori ||
args.settings ||
args.syncWindow ||
args.youtubeBrowser ||
@@ -647,7 +643,6 @@ export function isStandaloneTexthookerCommand(args: CliArgs): boolean {
!args.toggleVisibleOverlay &&
!args.togglePrimarySubtitleBar &&
!args.yomitan &&
!args.hachidori &&
!args.settings &&
!args.syncWindow &&
!args.youtubeBrowser &&
@@ -721,7 +716,6 @@ export function shouldStartApp(args: CliArgs): boolean {
args.toggleVisibleOverlay ||
args.togglePrimarySubtitleBar ||
args.yomitan ||
args.hachidori ||
args.settings ||
args.syncWindow ||
args.youtubeBrowser ||
@@ -773,12 +767,8 @@ export function shouldStartApp(args: CliArgs): boolean {
}
export function shouldRunYomitanOnlyStartup(args: CliArgs): boolean {
return args.yomitan && !args.hachidori && shouldRunDictionarySettingsOnlyStartup(args);
}
export function shouldRunDictionarySettingsOnlyStartup(args: CliArgs): boolean {
return (
(args.yomitan || args.hachidori) &&
args.yomitan &&
!args.background &&
!args.start &&
!args.stop &&
-1
View File
@@ -24,7 +24,6 @@ test('printHelp includes configured texthooker port', () => {
assert.match(output, /--setup\s+Open first-run setup window/);
assert.match(output, /--settings\s+Open SubMiner settings window/);
assert.match(output, /--yomitan\s+Open Yomitan settings window/);
assert.match(output, /--hachidori\s+Open Hachidori settings window/);
assert.match(output, /--mark-watched\s+Mark current video watched and advance playlist/);
assert.match(output, /--anilist-status/);
assert.match(output, /--anilist-retry-queue/);
-1
View File
@@ -25,7 +25,6 @@ ${B}Overlay${R}
--show-visible-overlay Show subtitle overlay
--hide-visible-overlay Hide subtitle overlay
--yomitan Open Yomitan settings window
--hachidori Open Hachidori settings window
--settings Open SubMiner settings window
--setup Open first-run setup window
--auto-start-overlay Auto-hide mpv subs, show overlay on connect
-4
View File
@@ -22,8 +22,6 @@ export type {
} from './definitions/shared';
const {
dictionaryBackend,
hachidori,
subtitlePosition,
keybindings,
websocket,
@@ -59,8 +57,6 @@ const { stats } = STATS_DEFAULT_CONFIG;
export const DEFAULT_CONFIG: ResolvedConfig = {
subtitleSelection: { enabled: false },
subtitleGeneration: { ...DEFAULT_SUBTITLE_GENERATION_CONFIG },
dictionaryBackend,
hachidori,
subtitlePosition,
keybindings,
websocket,
-4
View File
@@ -2,8 +2,6 @@ import { ResolvedConfig } from '../../types/config';
export const CORE_DEFAULT_CONFIG: Pick<
ResolvedConfig,
| 'dictionaryBackend'
| 'hachidori'
| 'subtitlePosition'
| 'keybindings'
| 'websocket'
@@ -20,8 +18,6 @@ export const CORE_DEFAULT_CONFIG: Pick<
| 'notifications'
| 'auto_start_overlay'
> = {
dictionaryBackend: 'yomitan',
hachidori: { externalHostManagementUrl: '' },
subtitlePosition: { yPercent: 10 },
keybindings: [],
websocket: {
-14
View File
@@ -81,20 +81,6 @@ export function buildCoreConfigOptionRegistry(
] as const;
return [
{
path: 'hachidori.externalHostManagementUrl',
kind: 'string',
defaultValue: defaultConfig.hachidori.externalHostManagementUrl,
description:
'Management URL override for character dictionary uploads to a linked Hachidori Docker host. Empty uses the linked host on the Docker default management port.',
},
{
path: 'dictionaryBackend',
kind: 'enum',
enumValues: ['yomitan', 'hachidori'],
defaultValue: defaultConfig.dictionaryBackend,
description: 'Dictionary lookup backend. Restart SubMiner after changing this setting.',
},
{
path: 'logging.level',
kind: 'enum',
@@ -1,22 +1,6 @@
import { ConfigTemplateSection } from './shared';
const CORE_TEMPLATE_SECTIONS: ConfigTemplateSection[] = [
{
title: 'Dictionary Backend',
description: ['Select the dictionary lookup backend: yomitan or hachidori.'],
notes: [
'Restart SubMiner after changing the backend. Each backend keeps separate settings and dictionaries.',
],
key: 'dictionaryBackend',
},
{
title: 'Hachidori External Dictionary Imports',
description: [
'Override the linked Docker host management URL only for a non-default port or a reverse proxy.',
],
notes: ['Used only while Hachidori is linked to an external host.'],
key: 'hachidori',
},
{
title: 'Subtitle Selection',
description: ['Select primary and secondary mpv subtitle tracks from the overlay.'],
-27
View File
@@ -2,37 +2,10 @@ import { ResolveContext } from './context';
import { applyControllerConfig } from './controller';
import { isNotificationType, isOverlayNotificationPosition } from '../../types/notification';
import { asBoolean, asNumber, asString, isObject } from './shared';
import { parseHachidoriManagementUrl } from '../../shared/hachidori-sharing';
export function applyCoreDomainConfig(context: ResolveContext): void {
const { src, resolved, warn } = context;
if (isObject(src.hachidori) && src.hachidori.externalHostManagementUrl !== undefined) {
try {
resolved.hachidori.externalHostManagementUrl = parseHachidoriManagementUrl(
src.hachidori.externalHostManagementUrl,
);
} catch {
warn(
'hachidori.externalHostManagementUrl',
src.hachidori.externalHostManagementUrl,
resolved.hachidori.externalHostManagementUrl,
'Expected an HTTP(S) origin or an empty string.',
);
}
}
if (src.dictionaryBackend === 'yomitan' || src.dictionaryBackend === 'hachidori') {
resolved.dictionaryBackend = src.dictionaryBackend;
} else if (src.dictionaryBackend !== undefined) {
warn(
'dictionaryBackend',
src.dictionaryBackend,
resolved.dictionaryBackend,
"Expected 'yomitan' or 'hachidori'.",
);
}
if (isObject(src.subtitleSelection)) {
const enabled = asBoolean(src.subtitleSelection.enabled);
if (enabled !== undefined) resolved.subtitleSelection.enabled = enabled;
@@ -1,51 +0,0 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { resolveConfig } from '../resolve';
import { buildConfigSettingsRegistry } from '../settings/registry';
import { createResolveContext } from './context';
import { applyCoreDomainConfig } from './core-domains';
test('dictionary backend defaults to Yomitan and accepts Hachidori', () => {
assert.equal(resolveConfig({}).resolved.dictionaryBackend, 'yomitan');
const { resolved, warnings } = resolveConfig({ dictionaryBackend: 'hachidori' });
assert.equal(resolved.dictionaryBackend, 'hachidori');
assert.deepEqual(warnings, []);
const field = buildConfigSettingsRegistry(resolved).find(
(entry) => entry.configPath === 'dictionaryBackend',
);
assert.equal(field?.restartBehavior, 'restart');
assert.deepEqual(field?.enumValues, ['yomitan', 'hachidori']);
assert.equal(field?.category, 'integrations');
});
test('unknown dictionary backend values warn and preserve the default', () => {
for (const dictionaryBackend of ['unknown', '', null, true, {}]) {
const { context, warnings } = createResolveContext({});
context.src.dictionaryBackend = dictionaryBackend;
applyCoreDomainConfig(context);
assert.equal(context.resolved.dictionaryBackend, 'yomitan');
assert.equal(warnings.length, 1);
assert.equal(warnings[0]?.path, 'dictionaryBackend');
}
});
test('Hachidori external import URL accepts HTTP origins and rejects invalid targets', () => {
assert.equal(resolveConfig({}).resolved.hachidori.externalHostManagementUrl, '');
const result = resolveConfig({
hachidori: { externalHostManagementUrl: 'http://127.0.0.1:8780/' },
});
assert.equal(result.resolved.hachidori.externalHostManagementUrl, 'http://127.0.0.1:8780');
assert.deepEqual(result.warnings, []);
for (const value of [
'file:///tmp/dict',
'http://host/import',
'http://user:password@host',
true,
]) {
const { context, warnings } = createResolveContext({});
context.src.hachidori = { externalHostManagementUrl: value };
applyCoreDomainConfig(context);
assert.equal(context.resolved.hachidori.externalHostManagementUrl, '');
assert.equal(warnings[0]?.path, 'hachidori.externalHostManagementUrl');
}
});
+3 -7
View File
@@ -146,7 +146,6 @@ const SECTION_ORDER = new Map<string, number>(
'Discord Rich Presence',
'Jellyfin',
'Texthooker',
'Dictionary Lookup',
'Yomitan',
'Stats dashboard',
'Startup warmups',
@@ -229,8 +228,8 @@ const LABEL_OVERRIDES: Record<string, string> = {
'shortcuts.openCharacterDictionaryManager': 'Open Character Dictionary Manager',
'subtitleSidebar.pauseVideoOnHover': 'Pause Video On Hover - Sidebar',
'subtitleStyle.autoPauseVideoOnHover': 'Pause Video On Hover - Subtitles',
'subtitleStyle.autoPauseVideoOnYomitanPopup': 'Pause Video On Dictionary Popup',
'subtitleStyle.primaryVisibleOnYomitanPopup': 'Keep Primary Visible On Dictionary Popup',
'subtitleStyle.autoPauseVideoOnYomitanPopup': 'Pause Video On Yomitan Popup',
'subtitleStyle.primaryVisibleOnYomitanPopup': 'Keep Primary Visible On Yomitan Popup',
'subtitleStyle.primaryDefaultMode': 'Primary Subtitle Visibility Mode',
'subtitleStyle.frequencyDictionary.mode': 'Frequency Mode',
'subtitleStyle.css': 'CSS Declarations',
@@ -271,7 +270,7 @@ const DESCRIPTION_OVERRIDES: Record<string, string> = {
'subtitleSidebar.css':
'CSS declarations applied to the subtitle sidebar. Includes color, background-color, all font properties, and sidebar CSS variables.',
'subtitleStyle.primaryVisibleOnYomitanPopup':
'When primary subtitles are in hover mode, keep the primary subtitle bar visible while a dictionary popup is open.',
'When primary subtitles are in hover mode, keep the primary subtitle bar visible while a Yomitan popup is open.',
'websocket.enabled':
'Built-in subtitle WebSocket server mode. Auto starts the built-in server only when mpv_websocket is not detected; otherwise it defers to the plugin.',
'discordPresence.updateIntervalMs':
@@ -335,9 +334,6 @@ function humanizePath(path: string): string {
}
function categoryAndSection(path: string): { category: ConfigSettingsCategory; section: string } {
if (path === 'dictionaryBackend' || path.startsWith('hachidori.')) {
return { category: 'integrations', section: 'Dictionary Lookup' };
}
if (
path === 'subtitleStyle.autoPauseVideoOnHover' ||
path === 'subtitleStyle.autoPauseVideoOnYomitanPopup' ||
@@ -2525,8 +2525,6 @@ Aligned English subtitle
await new Promise((resolve) => setTimeout(resolve, 1));
}
const addedBeforeMediaFinished = requests.some((request) => request.action === 'addNote');
const addRequest = requests.find((request) => request.action === 'addNote');
assert.deepEqual(addRequest?.params?.note?.tags, ['SubMiner', 'SubMiner::Stats']);
mediaRelease.audio?.();
mediaRelease.image?.();
@@ -6,7 +6,6 @@ import test from 'node:test';
import { createCoverArtFetcher, stripFilenameTags } from './cover-art-fetcher.js';
import { Database } from '../immersion-tracker/sqlite.js';
import {
applyPragmas,
ensureSchema,
getOrCreateAnimeRecord,
getOrCreateVideoRecord,
@@ -48,10 +47,9 @@ test('stripFilenameTags normalizes common media-title formats', () => {
);
});
test('fetchIfMissing backfills a missing blob from an existing cover URL', async () => {
async function backfillMissingCoverBlob(): Promise<void> {
const dbPath = makeDbPath();
const db = new Database(dbPath);
applyPragmas(db);
ensureSchema(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/cover-fetcher-test.mkv', {
canonicalTitle: 'Cover Fetcher Test',
@@ -105,12 +103,17 @@ test('fetchIfMissing backfills a missing blob from an existing cover URL', async
db.close();
cleanupDbPath(dbPath);
}
});
}
test('fetchIfMissing reuses cached cover art from another video in the same anime', async () => {
test(
'fetchIfMissing backfills a missing blob from an existing cover URL',
{ timeout: 15_000 },
backfillMissingCoverBlob,
);
async function reuseCachedAnimeCoverArt(): Promise<void> {
const dbPath = makeDbPath();
const db = new Database(dbPath);
applyPragmas(db);
ensureSchema(db);
const firstVideoId = getOrCreateVideoRecord(db, 'local:/tmp/cover-fetcher-cache-1.mkv', {
canonicalTitle: 'Shared Cover Show',
@@ -182,7 +185,13 @@ test('fetchIfMissing reuses cached cover art from another video in the same anim
db.close();
cleanupDbPath(dbPath);
}
});
}
test(
'fetchIfMissing reuses cached cover art from another video in the same anime',
{ timeout: 15_000 },
reuseCachedAnimeCoverArt,
);
function createJsonResponse(payload: unknown): Response {
return new Response(JSON.stringify(payload), {
@@ -194,7 +203,6 @@ function createJsonResponse(payload: unknown): Response {
test('fetchIfMissing uses guessit primary title and season when available', async () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
applyPragmas(db);
ensureSchema(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/cover-fetcher-season-test.mkv', {
canonicalTitle:
@@ -300,7 +308,6 @@ test('fetchIfMissing uses guessit primary title and season when available', asyn
test('fetchIfMissing falls back to internal parser when guessit throws', async () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
applyPragmas(db);
ensureSchema(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/cover-fetcher-fallback-test.mkv', {
canonicalTitle: 'School Vlog S01E01',
@@ -365,7 +372,6 @@ test('fetchIfMissing falls back to internal parser when guessit throws', async (
test('fetchIfMissing caches a no-match when the season cannot be resolved', async () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
applyPragmas(db);
ensureSchema(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/cover-fetcher-unresolved.mkv', {
canonicalTitle: 'Unresolved Show (2013) - S03E01 - Something [1080p].mkv',
@@ -444,7 +450,6 @@ test('fetchIfMissing caches a no-match when the season cannot be resolved', asyn
test('fetchIfMissing re-resolves an unresolved season once AniList publishes the relation', async () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
applyPragmas(db);
ensureSchema(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/cover-fetcher-recovers.mkv', {
canonicalTitle: 'Recovering Show (2013) - S02E01 - Something [1080p].mkv',
@@ -552,7 +557,6 @@ for (const linkedToAnilist of [false, true]) {
test(`TMDB fallback preserves AniList identity when linked=${linkedToAnilist}`, async () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
applyPragmas(db);
ensureSchema(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/hanzawa-01.mkv', {
canonicalTitle: 'Hanzawa Naoki - 01.mkv',
@@ -670,7 +674,6 @@ for (const linkedToAnilist of [false, true]) {
test('fetchIfMissing skips AniList for an entry already linked to TMDB', async () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
applyPragmas(db);
ensureSchema(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/hanzawa-02.mkv', {
canonicalTitle: 'Hanzawa Naoki - 02.mkv',
-1
View File
@@ -15,7 +15,6 @@ function makeArgs(overrides: Partial<CliArgs> = {}): CliArgs {
toggleVisibleOverlay: false,
togglePrimarySubtitleBar: false,
yomitan: false,
hachidori: false,
settings: false,
syncWindow: false,
youtubeBrowser: false,
-16
View File
@@ -19,7 +19,6 @@ function makeArgs(overrides: Partial<CliArgs> = {}): CliArgs {
toggle: false,
toggleVisibleOverlay: false,
yomitan: false,
hachidori: false,
settings: false,
syncWindow: false,
youtubeBrowser: false,
@@ -134,9 +133,6 @@ function createDeps(overrides: Partial<CliCommandServiceDeps> = {}) {
togglePrimarySubtitleBar: () => {
calls.push('togglePrimarySubtitleBar');
},
openHachidoriSettingsDelayed: (delayMs) => {
calls.push(`openHachidoriSettingsDelayed:${delayMs}`);
},
openYomitanSettingsDelayed: (delayMs) => {
calls.push(`openYomitanSettingsDelayed:${delayMs}`);
},
@@ -660,7 +656,6 @@ test('createCliCommandDepsRuntime reconnects MPV client when reconnect hook exis
},
ui: {
openFirstRunSetup: () => {},
openHachidoriSettings: () => {},
openYomitanSettings: () => {},
openConfigSettingsWindow: () => {},
openSyncUiWindow: () => {},
@@ -1124,14 +1119,3 @@ test('handleCliCommand reports async refresh-known-words errors to OSD', async (
assert.ok(osd.some((value) => value.includes('Refresh known words failed: refresh boom')));
assert.ok(calls.includes('stopApp'));
});
for (const source of ['initial', 'second-instance'] as const) {
test(`Hachidori settings command opens its own settings on ${source} invocation`, () => {
const { deps, calls } = createDeps();
handleCliCommand(makeArgs({ hachidori: true }), source, deps);
assert.ok(calls.includes('openHachidoriSettingsDelayed:1000'));
assert.equal(calls.includes('openYomitanSettingsDelayed:1000'), false);
assert.equal(calls.includes('initializeOverlayRuntime'), false);
assert.equal(calls.includes('connectMpvClient'), false);
});
}
-9
View File
@@ -43,7 +43,6 @@ export interface CliCommandServiceDeps {
togglePrimarySubtitleBar: () => void;
openFirstRunSetup: (force?: boolean) => void;
openYomitanSettingsDelayed: (delayMs: number) => void;
openHachidoriSettingsDelayed: (delayMs: number) => void;
openConfigSettingsWindow: () => void;
openSyncUiWindow: () => void;
openYoutubeBrowserWindow: () => void;
@@ -168,7 +167,6 @@ interface MiningCliRuntime {
interface UiCliRuntime {
openFirstRunSetup: (force?: boolean) => void;
openYomitanSettings: () => void;
openHachidoriSettings: () => void;
openConfigSettingsWindow: () => void;
openSyncUiWindow: () => void;
openYoutubeBrowserWindow: () => void;
@@ -275,11 +273,6 @@ export function createCliCommandDepsRuntime(
options.ui.openYomitanSettings();
}, delayMs);
},
openHachidoriSettingsDelayed: (delayMs) => {
options.schedule(() => {
options.ui.openHachidoriSettings();
}, delayMs);
},
openConfigSettingsWindow: options.ui.openConfigSettingsWindow,
openSyncUiWindow: options.ui.openSyncUiWindow,
openYoutubeBrowserWindow: options.ui.openYoutubeBrowserWindow,
@@ -431,8 +424,6 @@ export function handleCliCommand(
deps.logDebug('Opened first-run setup flow.');
} else if (args.yomitan) {
deps.openYomitanSettingsDelayed(1000);
} else if (args.hachidori) {
deps.openHachidoriSettingsDelayed(1000);
} else if (args.settings) {
deps.openConfigSettingsWindow();
} else if (args.syncWindow) {
@@ -1,55 +0,0 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { resolveHachidoriExtensionPath } from './hachidori-extension';
import { ensureExtensionCopyAsync } from './yomitan-extension-copy';
test('Hachidori resolves development and packaged artifacts without falling back to Yomitan', () => {
const options = { moduleDir: '/app/dist/core/services', resourcesPath: '/resources' };
assert.equal(
resolveHachidoriExtensionPath({
...options,
exists: (p) => p === '/app/build/hachidori/manifest.json',
}),
'/app/build/hachidori',
);
assert.equal(
resolveHachidoriExtensionPath({
...options,
exists: (p) => p === '/resources/hachidori/manifest.json',
}),
'/resources/hachidori',
);
assert.throws(
() => resolveHachidoriExtensionPath({ ...options, exists: () => false }),
/build:hachidori/,
);
});
test('Hachidori updates its own extension copy and preserves the Yomitan copy', async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-backend-copy-'));
try {
const source = path.join(root, 'source');
const profile = path.join(root, 'profile');
fs.mkdirSync(source);
fs.writeFileSync(path.join(source, 'manifest.json'), '{"name":"Hachidori","version":"1"}');
fs.mkdirSync(path.join(profile, 'extensions/yomitan'), { recursive: true });
fs.writeFileSync(path.join(profile, 'extensions/yomitan/marker'), 'preserved');
const options = { extensionName: 'hachidori', platform: 'linux' } satisfies Parameters<
typeof ensureExtensionCopyAsync
>[2];
const copy = await ensureExtensionCopyAsync(source, profile, options);
assert.equal(copy.targetDir, path.join(profile, 'extensions/hachidori'));
assert.equal((await ensureExtensionCopyAsync(source, profile, options)).copied, false);
fs.writeFileSync(path.join(source, 'bridge.js'), 'updated');
assert.equal((await ensureExtensionCopyAsync(source, profile, options)).copied, true);
assert.equal(
fs.readFileSync(path.join(profile, 'extensions/yomitan/marker'), 'utf8'),
'preserved',
);
} finally {
fs.rmSync(root, { recursive: true, force: true });
}
});
-59
View File
@@ -1,59 +0,0 @@
import electron from 'electron';
import type { Extension, Session } from 'electron';
import { existsSync } from 'node:fs';
import * as path from 'node:path';
import { ensureExtensionCopyAsync } from './yomitan-extension-copy';
import { HACHIDORI_SESSION_PARTITION } from './tokenizer/hachidori-parser-bridge';
export function getHachidoriSession(): Session {
return electron.session.fromPartition(HACHIDORI_SESSION_PARTITION);
}
export function resolveHachidoriExtensionPath(options: {
moduleDir: string;
resourcesPath: string;
exists?: (candidate: string) => boolean;
}): string {
const candidates = [
path.resolve(options.moduleDir, '../../../build/hachidori'),
path.join(options.resourcesPath, 'hachidori'),
];
const found = candidates.find((candidate) =>
(options.exists ?? existsSync)(path.join(candidate, 'manifest.json')),
);
if (!found) throw new Error('Hachidori is not bundled. Run bun run build:hachidori.');
return found;
}
/** Separate session keeps settings-only launches from injecting into the other backend's overlay. */
export function createHachidoriExtensionRuntime(userDataPath: string) {
let extension: Extension | null = null;
let loading: Promise<Extension> | null = null;
return {
getSession: getHachidoriSession,
ensureLoaded(): Promise<Extension> {
if (extension) return Promise.resolve(extension);
if (loading) return loading;
loading = (async () => {
const source = resolveHachidoriExtensionPath({
moduleDir: __dirname,
resourcesPath: process.resourcesPath,
});
const copy = await ensureExtensionCopyAsync(source, userDataPath, {
extensionName: 'hachidori',
});
const session = getHachidoriSession();
// Electron can reuse an old extension worker after its files change.
// Drop worker registrations before loading, preserving dictionaries and settings.
await session.clearStorageData({ storages: ['serviceworkers'] });
extension = await session.extensions.loadExtension(copy.targetDir, {
allowFileAccess: true,
});
return extension;
})().finally(() => {
loading = null;
});
return loading;
},
};
}
@@ -5,8 +5,9 @@ import path from 'node:path';
import test from 'node:test';
import { Database } from '../sqlite.js';
import type { DatabaseSync } from '../sqlite.js';
import { ensureSchema } from '../storage.js';
import { createTrackerPreparedStatements, ensureSchema, executeQueuedWrite } from '../storage.js';
import { deleteSession, deleteSessions, deleteVideo } from '../query-maintenance.js';
import { pruneRawRetention } from '../maintenance.js';
const DAY_MS = 86_400_000;
const BASE_MS = 1_700_000_000_000;
@@ -30,7 +31,13 @@ function cleanupDbPath(dbPath: string): void {
*/
function seed(
db: DatabaseSync,
lines: Array<{ session: 1 | 2; wordId: number; dayOffset: number; count?: number }>,
lines: Array<{
session: 1 | 2;
wordId: number;
dayOffset: number;
msOffset?: number;
count?: number;
}>,
options: { legacyRows?: boolean } = {},
): void {
db.exec(`
@@ -65,7 +72,7 @@ function seed(
let lineId = 0;
for (const line of lines) {
lineId += 1;
const seenMs = BASE_MS + line.dayOffset * DAY_MS;
const seenMs = BASE_MS + line.dayOffset * DAY_MS + (line.msOffset ?? 0);
insertLine.run(lineId, line.session, line.session, lineId, `line ${lineId}`, seenMs, seenMs);
insertWord.run(line.wordId, `語${line.wordId}`, `語${line.wordId}`);
if (options.legacyRows) {
@@ -83,13 +90,13 @@ function seed(
FROM imm_word_line_occurrences o WHERE o.word_id = imm_words.id
),
first_seen = (
SELECT MIN(sl.CREATED_DATE) / 1000
SELECT MIN(sl.CREATED_DATE) / 1000.0
FROM imm_word_line_occurrences o
JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
WHERE o.word_id = imm_words.id
),
last_seen = (
SELECT MAX(sl.LAST_UPDATE_DATE) / 1000
SELECT MAX(sl.LAST_UPDATE_DATE) / 1000.0
FROM imm_word_line_occurrences o
JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
WHERE o.word_id = imm_words.id
@@ -200,6 +207,167 @@ test('deleting the latest session moves last_seen back to the surviving line', (
}
});
for (const legacyRows of [false, true]) {
test(`deleting the latest session keeps fractional seen timestamps exact (legacyRows=${legacyRows})`, () => {
// Stored seen values are fractional seconds; whole-second rounding used to miss
// the removed extreme (or truncate the refreshed one).
const { db, dbPath } = createDb(
[
{ session: 1, wordId: 22, dayOffset: 0, msOffset: 251 },
{ session: 2, wordId: 22, dayOffset: 5, msOffset: 882 },
],
{ legacyRows },
);
try {
deleteSession(db, 2);
const word = readWord(db, 22);
assert.equal(word?.frequency, 1);
assert.equal(word?.firstSeen, (BASE_MS + 251) / 1000);
assert.equal(word?.lastSeen, (BASE_MS + 251) / 1000);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
}
test('deleting legacy history preserves first_seen when it predates the flush timestamp', () => {
const { db, dbPath } = createDb([
{ session: 1, wordId: 23, dayOffset: 0, msOffset: 400 },
{ session: 2, wordId: 23, dayOffset: 5, msOffset: 400 },
]);
try {
// A queue delay and retention-pruned history cannot be distinguished from
// the old flush-time timestamp. Preserve the earlier date in either case.
db.prepare('UPDATE imm_words SET first_seen = ? WHERE id = 23').run(BASE_MS / 1000);
deleteSession(db, 1);
assert.equal(readWord(db, 23)?.firstSeen, BASE_MS / 1000);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
for (const legacyRows of [false, true]) {
for (const deletedSession of [2, 3]) {
test(`deleting retained session ${deletedSession} preserves pruned first_seen (legacyRows=${legacyRows})`, () => {
const { db, dbPath } = createDb(
[
{ session: 1, wordId: 24, dayOffset: 0 },
{ session: 2, wordId: 24, dayOffset: 1 },
],
{ legacyRows },
);
try {
const latestMs = BASE_MS + 2 * DAY_MS;
db.exec(`
INSERT INTO imm_sessions(session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (3, 's3', 2, '${latestMs}', '${latestMs + 1000}', 2, ${latestMs}, ${latestMs});
INSERT INTO imm_subtitle_lines(line_id, session_id, video_id, line_index, text, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (3, 3, 2, 3, '猫', ${latestMs}, ${latestMs});
INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms)
VALUES (3, 24, 1, ${legacyRows ? 'NULL' : latestMs});
UPDATE imm_words SET frequency = 3, last_seen = ${latestMs / 1000} WHERE id = 24;
INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency)
SELECT 24, '猫', first_seen, last_seen, frequency FROM imm_words WHERE id = 24;
INSERT INTO imm_kanji_line_occurrences(line_id, kanji_id, occurrence_count, seen_ms)
SELECT line_id, 24, occurrence_count, seen_ms FROM imm_word_line_occurrences WHERE word_id = 24;
`);
pruneRawRetention(db, latestMs, {
eventsRetentionMs: Infinity,
telemetryRetentionMs: Infinity,
sessionsRetentionMs: 1.5 * DAY_MS,
});
deleteSession(db, deletedSession);
for (const table of ['imm_words', 'imm_kanji']) {
const row = db
.prepare(
`SELECT frequency, first_seen AS firstSeen, last_seen AS lastSeen FROM ${table} WHERE id = 24`,
)
.get() as { frequency: number; firstSeen: number; lastSeen: number };
assert.equal(row.frequency, 2);
assert.equal(row.firstSeen, BASE_MS / 1000, `${table} keeps the pruned discovery date`);
assert.equal(row.lastSeen, (BASE_MS + (deletedSession === 2 ? 2 : 1) * DAY_MS) / 1000);
}
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
}
}
test('deleting a queued subtitle uses observation time despite a delayed flush', () => {
const { db, dbPath } = createDb([]);
try {
const stmts = createTrackerPreparedStatements(db);
for (const sessionId of [1, 2]) {
const seenMs = BASE_MS + (sessionId - 1) * DAY_MS + 400;
executeQueuedWrite(
{
kind: 'subtitleLine',
sessionId,
videoId: sessionId,
lineIndex: 1,
segmentStartMs: null,
segmentEndMs: null,
text: '猫',
wordOccurrences: [
{
headword: '猫',
word: '猫',
reading: 'ねこ',
partOfSpeech: 'noun',
pos1: '名詞',
pos2: '一般',
pos3: '',
occurrenceCount: 1,
frequencyRank: null,
},
],
kanjiOccurrences: [{ kanji: '猫', occurrenceCount: 1 }],
firstSeen: seenMs / 1000,
lastSeen: seenMs / 1000,
},
stmts,
);
for (const table of ['imm_word_line_occurrences', 'imm_kanji_line_occurrences']) {
const row = db
.prepare(`SELECT seen_ms AS seenMs FROM ${table} WHERE line_id = ?`)
.get(sessionId) as { seenMs: number };
assert.equal(row.seenMs, seenMs);
}
}
deleteSession(db, 1);
for (const table of ['imm_words', 'imm_kanji']) {
const row = db
.prepare(`SELECT frequency, first_seen AS firstSeen, last_seen AS lastSeen FROM ${table}`)
.get() as {
frequency: number;
firstSeen: number;
lastSeen: number;
};
assert.equal(row.frequency, 1);
assert.equal(row.firstSeen, (BASE_MS + DAY_MS + 400) / 1000);
assert.equal(row.lastSeen, (BASE_MS + DAY_MS + 400) / 1000);
}
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('deleting an interior occurrence leaves the surrounding extremes untouched', () => {
// Session 2 carries the middle occurrence; sessions bracket it in time.
const { db, dbPath } = createDb([
@@ -4567,13 +4567,13 @@ test('deleteSession rebuilds word and kanji aggregates from retained subtitle li
assert.ok(sharedWordRow);
assert.equal(sharedWordRow.frequency, 1);
assert.equal(sharedWordRow.first_seen, Math.floor(keptTs / 1000));
assert.equal(sharedWordRow.last_seen, Math.floor(keptTs / 1000));
assert.equal(sharedWordRow.first_seen, keptTs / 1000);
assert.equal(sharedWordRow.last_seen, keptTs / 1000);
assert.equal(deletedOnlyWordRow ?? null, null);
assert.ok(sharedKanjiRow);
assert.equal(sharedKanjiRow.frequency, 1);
assert.equal(sharedKanjiRow.first_seen, Math.floor(keptTs / 1000));
assert.equal(sharedKanjiRow.last_seen, Math.floor(keptTs / 1000));
assert.equal(sharedKanjiRow.first_seen, keptTs / 1000);
assert.equal(sharedKanjiRow.last_seen, keptTs / 1000);
assert.equal(deletedOnlyKanjiRow ?? null, null);
} finally {
db.close();
@@ -16,7 +16,8 @@ import {
} from './query-lexical';
import { Database } from './sqlite';
import type { DatabaseSync } from './sqlite';
import { ensureSchema } from './storage';
import { createTrackerPreparedStatements, ensureSchema, executeQueuedWrite } from './storage';
import { BASE_MS, createDb, seedEndedSession, seedVideo } from './__tests__/lifetime-test-fixtures';
function makeDbPath(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollups-'));
@@ -125,6 +126,121 @@ test('lexical rollup rebuild excludes rows hidden by vocabulary persistence rule
}
});
test('re-seeing a filtered legacy word keeps charts consistent with the summary', () => {
const db = createDb();
try {
const videoId = seedVideo(db, null, 'episode');
const sessionId = seedEndedSession(db, videoId, BASE_MS, { activeMs: 1_000 });
const firstSeen = BASE_MS / 1000;
// Legacy row whose stored merged-token POS fails the persistence filter.
db.prepare(
`INSERT INTO imm_words(
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency
) VALUES ('二死', 'にし', 'にし', 'other', '助詞|動詞', '格助詞|自立', '一般|*', ?, ?, 1)`,
).run(firstSeen, firstSeen);
rebuildLexicalDailyRollups(db);
// The incoming token passes the filter, but the upsert keeps the stored POS.
executeQueuedWrite(
{
kind: 'subtitleLine',
sessionId,
videoId,
lineIndex: 1,
segmentStartMs: null,
segmentEndMs: null,
text: 'にし',
wordOccurrences: [
{
headword: '二死',
word: 'にし',
reading: 'にし',
partOfSpeech: 'verb',
pos1: '動詞',
pos2: '自立',
pos3: '*',
frequencyRank: null,
occurrenceCount: 1,
},
],
kanjiOccurrences: [],
firstSeen: firstSeen + 60,
lastSeen: firstSeen + 60,
},
createTrackerPreparedStatements(db),
);
const plotted = getVocabularyChartData(db).newWordsTimeline.reduce(
(sum, day) => sum + day.wordCount,
0,
);
assert.equal(getVocabularySummary(db, null).uniqueWords, 0);
assert.equal(plotted, 0);
} finally {
db.close();
}
});
test('lexical rollup rebuild repairs last_seen left pointing at deleted history', () => {
const db = createDb();
try {
const videoId = seedVideo(db, null, 'episode');
const sessionId = seedEndedSession(db, videoId, BASE_MS, { activeMs: 1_000 });
const lineId = Number(
db
.prepare(
`INSERT INTO imm_subtitle_lines(session_id, video_id, line_index, text, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (?, ?, 1, '猫', ?, ?)`,
)
.run(sessionId, videoId, BASE_MS, BASE_MS).lastInsertRowid,
);
const seen = (BASE_MS + 251) / 1000;
// first_seen this early may be real history pruned by retention; never touched.
const earlyFirst = seen - 86_400.5;
const staleLast = seen + 86_400.631;
db.prepare(
`INSERT INTO imm_words(id, headword, word, reading, first_seen, last_seen, frequency)
VALUES (1, '猫', '猫', 'ねこ', ?, ?, 1), (2, '犬', '犬', 'いぬ', ?, ?, 1)`,
).run(earlyFirst, staleLast, seen - 0.1, seen - 0.1);
db.prepare(
`INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency) VALUES (1, '猫', ?, ?, 1)`,
).run(earlyFirst, staleLast);
db.prepare(
`INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms)
VALUES (?, 1, 1, ?), (?, 2, 1, ?)`,
).run(lineId, BASE_MS + 251, lineId, BASE_MS + 251);
db.prepare(
`INSERT INTO imm_kanji_line_occurrences(line_id, kanji_id, occurrence_count, seen_ms)
VALUES (?, 1, 1, ?)`,
).run(lineId, BASE_MS + 251);
rebuildLexicalDailyRollups(db);
const bounds = (table: string): Array<[number, number]> =>
(
db
.prepare(`SELECT first_seen AS first, last_seen AS last FROM ${table} ORDER BY id`)
.all() as Array<{
first: number;
last: number;
}>
).map((row) => [row.first, row.last]);
assert.deepEqual(
bounds('imm_words'),
[
[earlyFirst, seen],
[seen - 0.1, seen - 0.1],
],
'healthy rows are untouched',
);
assert.deepEqual(bounds('imm_kanji'), [[earlyFirst, seen]]);
} finally {
db.close();
}
});
test('lexical rollup rebuild tolerates nullable legacy vocabulary text', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -268,7 +384,7 @@ test('current lexical rollup readiness accepts legacy integer state storage', ()
state_value INTEGER NOT NULL
);
INSERT INTO imm_rollup_state(state_key, state_value)
VALUES ('lexical_daily_rollups_version', 2);
VALUES ('lexical_daily_rollups_version', 3);
`);
assert.equal(areLexicalDailyRollupsReady(db), true);
@@ -18,7 +18,9 @@ const LOCAL_EPOCH_DAY_SQL = `
) - 2440587.5 AS INTEGER)
`;
const LEXICAL_DAILY_ROLLUP_VERSION = '2';
// v3 re-derives visibility for rows a re-seen word forced visible and repairs
// last_seen values left behind by session deletes.
const LEXICAL_DAILY_ROLLUP_VERSION = '3';
const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
@@ -26,6 +28,32 @@ export function localEpochDaySql(value: string): string {
return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
}
// Session deletes once rounded removed timestamps to whole seconds, missing
// fractional last_seen extremes that pointed at deleted history. New seen_ms
// values match observation time; legacy values were stamped later at flush.
// Retention prunes the oldest sessions first, so healthy rows never exceed
// their newest surviving occurrence. Rows with undated occurrences are skipped.
//
// first_seen is deliberately not repaired: a stale bound is indistinguishable
// from real history whose occurrences retention pruned, and frequency does not
// reliably tell them apart.
function repairStaleLastSeen(db: DatabaseSync): void {
for (const [table, occurrenceTable, col] of [
['imm_words', 'imm_word_line_occurrences', 'word_id'],
['imm_kanji', 'imm_kanji_line_occurrences', 'kanji_id'],
] as const) {
const newestSeen = `(SELECT MAX(o.seen_ms) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id) / 1000.0`;
db.exec(`
UPDATE ${table}
SET last_seen = ${newestSeen}
WHERE last_seen > ${newestSeen}
AND NOT EXISTS (
SELECT 1 FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id AND o.seen_ms IS NULL
)
`);
}
}
function createWordRollupTriggers(db: DatabaseSync): void {
const dayForNew = localEpochDaySql('NEW.first_seen');
const dayForOld = localEpochDaySql('OLD.first_seen');
@@ -186,6 +214,7 @@ export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
lastId = vocabularyRows[vocabularyRows.length - 1]!.id;
if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break;
}
repairStaleLastSeen(db);
db.exec('DELETE FROM imm_lexical_daily_rollups');
db.exec(`
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
@@ -298,9 +298,11 @@ export function planLexicalRemovalsForVideos(
return planLexicalRemovals(db, `sl.video_id IN (${makePlaceholders(videoIds)})`, videoIds);
}
// first_seen/last_seen are REAL seconds that keep millisecond fractions; never round
// them, or comparisons against stored extremes miss and refreshes drift.
function toStoredSeenSeconds(ms: number | null): number | null {
if (ms === null || !Number.isFinite(ms)) return null;
return Math.floor(ms / 1000);
return ms / 1000;
}
/**
@@ -320,6 +322,13 @@ export function applyLexicalRemovals(db: DatabaseSync, plan: LexicalRemovalPlan)
applyRemovalsForEntity(db, 'kanji', plan.kanji);
}
// A timestamp repair must keep subtraction-exact lifetime counts and any
// first_seen date that the deleted occurrences cannot account for.
type LexicalAggregateOverride = {
frequency: number | null;
firstSeen: number | null;
};
function applyRemovalsForEntity(
db: DatabaseSync,
entity: LexicalEntity,
@@ -358,7 +367,7 @@ function applyRemovalsForEntity(
`UPDATE ${entityTable} SET frequency = ?, first_seen = ?, last_seen = ? WHERE id = ?`,
);
const needsExactRefresh: number[] = [];
const needsExactRefresh = new Map<number, LexicalAggregateOverride>();
for (const removal of removals) {
const current = selectStmt.get(removal.id) as {
@@ -369,17 +378,9 @@ function applyRemovalsForEntity(
if (!current) continue;
const nextFrequency = (current.frequency ?? 0) - removal.removedFrequency;
if (nextFrequency <= 0) {
// The rows in scope are already gone by now, so anything still pointing at
// this entity means the stored frequency was stale rather than exhausted.
if (hasOccurrencesStmt.get(removal.id)) {
needsExactRefresh.push(removal.id);
} else {
deleteStmt.run(removal.id);
}
continue;
}
// Legacy occurrences use flush time, which can follow the actual observation.
// An earlier stored date may also belong to retention-pruned history. Only
// move it when the removed timestamps prove that its observation was deleted.
const removedFirstSeen = toStoredSeenSeconds(removal.removedFirstSeenMs);
const removedLastSeen = toStoredSeenSeconds(removal.removedLastSeenMs);
const firstSeenMayHaveMoved =
@@ -388,11 +389,26 @@ function applyRemovalsForEntity(
const lastSeenMayHaveMoved =
current.lastSeen === null ||
(removedLastSeen !== null && removedLastSeen >= current.lastSeen);
const exactRefreshOverride = {
frequency: nextFrequency > 0 ? nextFrequency : null,
firstSeen: firstSeenMayHaveMoved ? null : current.firstSeen,
};
if (nextFrequency <= 0) {
// The rows in scope are already gone by now, so anything still pointing at
// this entity means the stored frequency was stale rather than exhausted.
if (hasOccurrencesStmt.get(removal.id)) {
needsExactRefresh.set(removal.id, exactRefreshOverride);
} else {
deleteStmt.run(removal.id);
}
continue;
}
if (firstSeenMayHaveMoved || lastSeenMayHaveMoved) {
// Undated pre-migration occurrences are invisible to the seeks below;
// fall back to the full re-aggregate that resolves their dates.
// resolve their dates without discarding retention-pruned aggregates.
if (hasUndatedOccurrenceStmt.get(removal.id)) {
needsExactRefresh.push(removal.id);
needsExactRefresh.set(removal.id, exactRefreshOverride);
continue;
}
const minSeenMs = (minSeenStmt.get(removal.id) as { value: number | null }).value;
@@ -400,13 +416,13 @@ function applyRemovalsForEntity(
if (minSeenMs === null || maxSeenMs === null) {
// Frequency says occurrences remain but none exist: stale row, let the
// exact refresh reconcile (it deletes rows with nothing left).
needsExactRefresh.push(removal.id);
needsExactRefresh.set(removal.id, exactRefreshOverride);
continue;
}
updateAggregatesStmt.run(
nextFrequency,
Math.floor(Number(minSeenMs) / 1000),
Math.floor(Number(maxSeenMs) / 1000),
firstSeenMayHaveMoved ? toStoredSeenSeconds(Number(minSeenMs)) : current.firstSeen,
toStoredSeenSeconds(Number(maxSeenMs)),
removal.id,
);
continue;
@@ -416,13 +432,17 @@ function applyRemovalsForEntity(
}
if (entity === 'word') {
refreshWordAggregates(db, needsExactRefresh);
refreshWordAggregates(db, [...needsExactRefresh.keys()], needsExactRefresh);
} else {
refreshKanjiAggregates(db, needsExactRefresh);
refreshKanjiAggregates(db, [...needsExactRefresh.keys()], needsExactRefresh);
}
}
function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void {
function refreshWordAggregates(
db: DatabaseSync,
wordIds: number[],
overrides?: ReadonlyMap<number, LexicalAggregateOverride>,
): void {
if (wordIds.length === 0) {
return;
}
@@ -470,15 +490,19 @@ function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void {
continue;
}
updateStmt.run(
row.frequency,
Math.floor(row.firstSeen / 1000),
Math.floor(row.lastSeen / 1000),
overrides?.get(row.wordId)?.frequency ?? row.frequency,
overrides?.get(row.wordId)?.firstSeen ?? toStoredSeenSeconds(row.firstSeen),
toStoredSeenSeconds(row.lastSeen),
row.wordId,
);
}
}
function refreshKanjiAggregates(db: DatabaseSync, kanjiIds: number[]): void {
function refreshKanjiAggregates(
db: DatabaseSync,
kanjiIds: number[],
overrides?: ReadonlyMap<number, LexicalAggregateOverride>,
): void {
if (kanjiIds.length === 0) {
return;
}
@@ -526,9 +550,9 @@ function refreshKanjiAggregates(db: DatabaseSync, kanjiIds: number[]): void {
continue;
}
updateStmt.run(
row.frequency,
Math.floor(row.firstSeen / 1000),
Math.floor(row.lastSeen / 1000),
overrides?.get(row.kanjiId)?.frequency ?? row.frequency,
overrides?.get(row.kanjiId)?.firstSeen ?? toStoredSeenSeconds(row.firstSeen),
toStoredSeenSeconds(row.lastSeen),
row.kanjiId,
);
}
+25 -16
View File
@@ -6,6 +6,7 @@ import { normalizeTitleIdentity } from '../../utils/title-normalization';
import type { DatabaseSync } from './sqlite';
import { nowMs } from './time';
import { ensureLexicalDailyRollupTables, markLexicalDailyRollupsReady } from './lexical-rollups';
import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility';
import { SCHEMA_VERSION } from './types';
import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types';
import { toDbMs, toDbTimestamp } from './query-shared';
@@ -17,7 +18,8 @@ export interface TrackerPreparedStatements {
wordUpsertStmt: ReturnType<DatabaseSync['prepare']>;
kanjiUpsertStmt: ReturnType<DatabaseSync['prepare']>;
subtitleLineInsertStmt: ReturnType<DatabaseSync['prepare']>;
wordIdSelectStmt: ReturnType<DatabaseSync['prepare']>;
wordSelectStmt: ReturnType<DatabaseSync['prepare']>;
wordVisibilityUpdateStmt: ReturnType<DatabaseSync['prepare']>;
kanjiIdSelectStmt: ReturnType<DatabaseSync['prepare']>;
wordLineOccurrenceUpsertStmt: ReturnType<DatabaseSync['prepare']>;
kanjiLineOccurrenceUpsertStmt: ReturnType<DatabaseSync['prepare']>;
@@ -1759,7 +1761,6 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
pos1 = COALESCE(NULLIF(imm_words.pos1, ''), excluded.pos1),
pos2 = COALESCE(NULLIF(imm_words.pos2, ''), excluded.pos2),
pos3 = COALESCE(NULLIF(imm_words.pos3, ''), excluded.pos3),
vocabulary_visible = 1,
first_seen = MIN(COALESCE(first_seen, excluded.first_seen), excluded.first_seen),
last_seen = MAX(COALESCE(last_seen, excluded.last_seen), excluded.last_seen),
frequency_rank = CASE
@@ -1787,10 +1788,16 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
`),
wordIdSelectStmt: db.prepare(`
SELECT id FROM imm_words
wordSelectStmt: db.prepare(`
SELECT id, word, headword, reading, part_of_speech AS partOfSpeech,
pos1, pos2, pos3, frequency_rank AS frequencyRank,
vocabulary_visible AS vocabularyVisible
FROM imm_words
WHERE headword = ? AND word = ? AND reading = ?
`),
wordVisibilityUpdateStmt: db.prepare(`
UPDATE imm_words SET vocabulary_visible = ? WHERE id = ?
`),
kanjiIdSelectStmt: db.prepare(`
SELECT id FROM imm_kanji
WHERE kanji = ?
@@ -1842,14 +1849,18 @@ function incrementWordAggregate(
occurrence.frequencyRank ?? null,
);
}
const row = stmts.wordIdSelectStmt.get(
occurrence.headword,
occurrence.word,
occurrence.reading,
) as { id: number } | null;
const row = stmts.wordSelectStmt.get(occurrence.headword, occurrence.word, occurrence.reading) as
| (VocabularyVisibilityRow & { id: number; vocabularyVisible: number })
| null;
if (!row?.id) {
throw new Error(`Failed to resolve imm_words id for ${occurrence.headword}`);
}
// The upsert keeps the first-seen POS metadata, so derive visibility from the
// stored row (as the vocabulary summary does) rather than the incoming token.
const visible = isVocabularyStatsRowVisible(row) ? 1 : 0;
if (row.vocabularyVisible !== visible) {
stmts.wordVisibilityUpdateStmt.run(visible, row.id);
}
return row.id;
}
@@ -1952,18 +1963,16 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta
currentMs,
);
const lineId = Number(lineResult.lastInsertRowid);
// Match aggregate bounds to the observation, even if this write was queued
// for a while. Flush time cannot identify which occurrence set first_seen.
const seenMs = toDbTimestamp(Math.round(write.firstSeen * 1000));
for (const occurrence of write.wordOccurrences) {
const wordId = incrementWordAggregate(stmts, occurrence, write.firstSeen, write.lastSeen);
stmts.wordLineOccurrenceUpsertStmt.run(lineId, wordId, occurrence.occurrenceCount, currentMs);
stmts.wordLineOccurrenceUpsertStmt.run(lineId, wordId, occurrence.occurrenceCount, seenMs);
}
for (const occurrence of write.kanjiOccurrences) {
const kanjiId = incrementKanjiAggregate(stmts, occurrence, write.firstSeen, write.lastSeen);
stmts.kanjiLineOccurrenceUpsertStmt.run(
lineId,
kanjiId,
occurrence.occurrenceCount,
currentMs,
);
stmts.kanjiLineOccurrenceUpsertStmt.run(lineId, kanjiId, occurrence.occurrenceCount, seenMs);
}
return;
}
-35
View File
@@ -276,7 +276,6 @@ test('handleMultiCopyDigit copies backward from the current subtitle after a bac
test('handleMineSentenceDigit reports async create failures', async () => {
const osd: string[] = [];
const logs: Array<{ message: string; err: unknown }> = [];
let cardsMined = 0;
handleMineSentenceDigit(2, {
subtitleTimingTracker: {
@@ -296,9 +295,6 @@ test('handleMineSentenceDigit reports async create failures', async () => {
getCurrentSecondarySubText: () => 'sub2',
showMpvOsd: (text) => osd.push(text),
logError: (message, err) => logs.push({ message, err }),
onCardsMined: (count) => {
cardsMined += count;
},
});
await new Promise((resolve) => setImmediate(resolve));
@@ -307,37 +303,6 @@ test('handleMineSentenceDigit reports async create failures', async () => {
assert.equal(logs[0]?.message, 'mineSentenceMultiple failed:');
assert.equal((logs[0]?.err as Error).message, 'mine boom');
assert.ok(osd.some((entry) => entry.includes('Mine sentence failed: mine boom')));
assert.equal(cardsMined, 0);
});
test('handleMineSentenceDigit increments successful card count', async () => {
const osd: string[] = [];
let cardsMined = 0;
handleMineSentenceDigit(2, {
subtitleTimingTracker: {
getRecentBlocks: () => ['one', 'two'],
getCurrentSubtitle: () => null,
findTiming: (text) =>
text === 'one' ? { startTime: 1, endTime: 3 } : { startTime: 4, endTime: 7 },
},
ankiIntegration: {
updateLastAddedFromClipboard: async () => {},
triggerFieldGroupingForLastAddedCard: async () => {},
markLastCardAsAudioCard: async () => {},
createSentenceCard: async () => true,
},
getCurrentSecondarySubText: () => 'sub2',
showMpvOsd: (text) => osd.push(text),
logError: () => {},
onCardsMined: (count) => {
cardsMined += count;
},
});
await new Promise((resolve) => setImmediate(resolve));
assert.equal(cardsMined, 1);
});
test('handleMineSentenceDigit keeps per-entry timings when subtitle text repeats', async () => {
+1 -7
View File
@@ -201,7 +201,6 @@ export function handleMineSentenceDigit(
getCurrentSecondarySubText: () => string | undefined;
showMpvOsd: (text: string) => void;
logError: (message: string, err: unknown) => void;
onCardsMined?: (count: number) => void;
},
): void {
if (!deps.subtitleTimingTracker || !deps.ankiIntegration) return;
@@ -233,14 +232,9 @@ export function handleMineSentenceDigit(
entries,
deps.getCurrentSecondarySubText,
);
const cardsToMine = 1;
// Card creation records the mined card (with its note id) itself.
deps.ankiIntegration
.createSentenceCard(sentence, rangeStart, rangeEnd, secondarySubText)
.then((created) => {
if (created) {
deps.onCardsMined?.(cardsToMine);
}
})
.catch((err) => {
deps.logError('mineSentenceMultiple failed:', err);
deps.showMpvOsd(`Mine sentence failed: ${(err as Error).message}`);
@@ -39,6 +39,7 @@ test('non-macOS modal overlay remains a regular window', () => {
});
assert.equal(options.type, undefined);
assert.equal(options.roundedCorners, false);
});
test('Linux visible overlay window allows compositor resize for mpv-sized placement', () => {
@@ -37,6 +37,7 @@ export function buildOverlayWindowOptions(
paintWhenInitiallyHidden: true,
backgroundColor: '#00000000',
frame: false,
...(platform === 'linux' ? { roundedCorners: false } : {}),
alwaysOnTop: shouldStartAlwaysOnTop,
skipTaskbar: true,
resizable: shouldAllowCompositorResize,
@@ -15,7 +15,6 @@ function makeArgs(overrides: Partial<CliArgs> = {}): CliArgs {
toggleVisibleOverlay: false,
togglePrimarySubtitleBar: false,
yomitan: false,
hachidori: false,
settings: false,
syncWindow: false,
youtubeBrowser: false,
@@ -7,7 +7,6 @@ import { resolveAnimatedImageLeadInSeconds } from '../../../anki-integration/ani
import { clampMediaEndTime } from '../../../anki-integration/media-duration.js';
import { MediaGenerator } from '../../../media-generator.js';
import { statsJson } from '../../../types/stats-http-contract.js';
import { STATS_MINING_TAG } from '../../../shared/anki-source.js';
import {
resolveRetimedSecondarySubtitleTextFromSidecar,
resolveSecondarySubtitleTextFromSidecar,
@@ -369,7 +368,7 @@ export function registerStatsMiningRoutes(app: Hono, options?: StatsMiningRouteO
}
const model = ankiConfig.isLapis?.sentenceCardModel || 'Basic';
const tags = [...new Set([...(ankiConfig.tags ?? ['SubMiner']), STATS_MINING_TAG])];
const tags = ankiConfig.tags ?? ['SubMiner'];
const addNotePromise = timeMiningPhase(
mode,
@@ -67,6 +67,7 @@ export function buildStatsWindowOptions(options: {
width: options.bounds?.width ?? DEFAULT_STATS_WINDOW_WIDTH,
height: options.bounds?.height ?? DEFAULT_STATS_WINDOW_HEIGHT,
frame: false,
...(platform === 'linux' ? { roundedCorners: false } : {}),
transparent: false,
alwaysOnTop: true,
resizable: false,
+1
View File
@@ -57,6 +57,7 @@ test('buildStatsWindowOptions remains a regular window off macOS', () => {
});
assert.equal(options.type, undefined);
assert.equal(options.roundedCorners, false);
});
test('stats panels present after document load on macOS', () => {
@@ -300,65 +300,3 @@ test('prefetch service deduplicates repeated cue text within a run', async () =>
);
assert.ok(tokenizedTexts.includes('other'));
});
test('prefetch service keeps an in-flight result from a superseded run', async () => {
const cues = makeCues(5);
const cachedTexts: string[] = [];
let releaseFirst!: () => void;
const firstGate = new Promise<void>((resolve) => {
releaseFirst = resolve;
});
const service = createSubtitlePrefetchService({
cues,
tokenizeSubtitle: async (text) => {
if (text === 'line-0') await firstGate;
return { text, tokens: [] };
},
preCacheTokenization: (text) => {
cachedTexts.push(text);
},
getCacheGeneration: () => 0,
priorityWindowSize: 3,
});
service.start(0);
await flushMicrotasks();
service.stop();
releaseFirst();
await flushMicrotasks();
assert.deepEqual(cachedTexts, ['line-0']);
});
test('prefetch service drops an in-flight result after cache invalidation', async () => {
const cues = makeCues(5);
const cachedTexts: string[] = [];
let generation = 0;
let releaseFirst!: () => void;
const firstGate = new Promise<void>((resolve) => {
releaseFirst = resolve;
});
const service = createSubtitlePrefetchService({
cues,
tokenizeSubtitle: async (text) => {
if (text === 'line-0') await firstGate;
return { text, tokens: [] };
},
preCacheTokenization: (text) => {
cachedTexts.push(text);
},
getCacheGeneration: () => generation,
priorityWindowSize: 3,
});
service.start(0);
await flushMicrotasks();
service.stop();
generation += 1;
releaseFirst();
await flushMicrotasks();
assert.deepEqual(cachedTexts, []);
});
+1 -10
View File
@@ -7,12 +7,6 @@ export interface SubtitlePrefetchServiceDeps {
tokenizeSubtitle: (text: string) => Promise<SubtitleData | null>;
preCacheTokenization: (text: string, data: SubtitleData) => void;
hasCachedTokenization?: (text: string) => boolean;
/**
* Bumped whenever the tokenization cache is invalidated. A result is cached only if
* the generation is unchanged since its tokenization began; without this dep every
* result counts as fresh.
*/
getCacheGeneration?: () => number;
priorityWindowSize?: number;
}
@@ -94,12 +88,9 @@ export function createSubtitlePrefetchService(
}
warmedKeys.add(cacheKey);
// Freshness follows the cache generation, not the run: a seek or restart that
// supersedes this run leaves the result valid, and the parser time is already spent.
const generation = deps.getCacheGeneration?.();
try {
const result = await deps.tokenizeSubtitle(cue.text);
if (result && generation === deps.getCacheGeneration?.()) {
if (result && !stopped && runId === currentRunId) {
deps.preCacheTokenization(cue.text, result);
}
} catch {
@@ -43,8 +43,6 @@ export interface SubtitleProcessingController {
*/
notePlainSubtitleEmitted: (text: string) => void;
invalidateTokenizationCache: () => void;
/** Incremented by invalidateTokenizationCache; lets async writers detect stale results. */
getCacheGeneration: () => number;
preCacheTokenization: (text: string, data: SubtitleData) => void;
consumeCachedSubtitle: (text: string) => SubtitleData | null;
hasCachedSubtitle: (text: string) => boolean;
@@ -255,7 +253,6 @@ export function createSubtitleProcessingController(
tokenizationCache.clear();
cacheGeneration += 1;
},
getCacheGeneration: () => cacheGeneration,
preCacheTokenization: (text: string, data: SubtitleData) => {
setCachedTokenization(text, data);
},
-20
View File
@@ -4362,26 +4362,6 @@ test('tokenizeSubtitle keeps Yomitan frequency for noun-particle-noun compounds'
assert.equal(result.tokens?.[0]?.frequencyRank, 581);
});
test('tokenizeSubtitle skips frequency requests for ranks supplied by the scanner', async () => {
const deps = makeDepsFromYomitanTokens(
[{ surface: '猫', reading: 'ねこ', headword: '猫', frequencyRank: 42 }],
{ getFrequencyDictionaryEnabled: () => true },
);
const parserWindow = deps.getYomitanParserWindow();
assert.ok(parserWindow);
deps.getYomitanParserWindow = () => parserWindow;
const scripts: string[] = [];
const execute = parserWindow.webContents.executeJavaScript.bind(parserWindow.webContents);
parserWindow.webContents.executeJavaScript = async (script) => {
scripts.push(script);
return execute(script);
};
const result = await tokenizeSubtitle('猫', deps);
assert.equal(result.tokens?.[0]?.frequencyRank, 42);
assert.ok(scripts.length > 0);
assert.equal(scripts.filter((script) => script.includes('getTermFrequencies')).length, 0);
});
test('tokenizeSubtitle keeps frequency for ordinal prefix-noun tokens', async () => {
const result = await tokenizeSubtitle(
'第二走者',
-1
View File
@@ -494,7 +494,6 @@ function buildYomitanFrequencyTermReadingList(
): Array<{ term: string; reading: string | null }> {
const termReadingList: Array<{ term: string; reading: string | null }> = [];
for (const token of tokens) {
if (normalizePositiveFrequencyRank(token.frequencyRank) !== null) continue;
const readingRaw =
token.reading && token.reading.trim().length > 0 ? token.reading.trim() : null;
for (const term of resolveYomitanFrequencyLookupTexts(token, matchMode)) {
@@ -1,181 +0,0 @@
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import path from 'node:path';
import test from 'node:test';
import { pathToFileURL } from 'node:url';
import vm from 'node:vm';
import { buildHachidoriAnkiHints, HACHIDORI_ANKI_SETTINGS_SCRIPT } from './hachidori-anki-settings';
const extensionPath = path.resolve(__dirname, '../../../../vendor/hachidori/extension');
const config = {
tags: ['SubMiner', 'Japanese'],
fields: { word: 'Term', sentence: 'Context', wordAudio: 'Pronunciation', image: 'Image' },
};
const modelFields = ['Term', 'Reading', 'Definition', 'Context', 'Pronunciation', 'Image'];
async function harness(
anki: unknown = {},
models: Record<string, string[]> = { Japanese: modelFields },
) {
const templates: unknown = await import(
pathToFileURL(path.join(extensionPath, 'anki-templates.js')).href
);
const setup: unknown = await import(
pathToFileURL(path.join(extensionPath, 'anki-setup.js')).href
);
const context = vm.createContext({
__templates: templates,
__setup: setup,
__initialAnki: anki,
__models: models,
structuredClone,
URL,
});
vm.runInContext('globalThis.window = globalThis', context);
vm.runInContext(await readFile(path.join(extensionPath, 'reader-options.js'), 'utf8'), context);
vm.runInContext(
`
let options = { ...HDReaderOptions.normaliseOptions({ anki: __initialAnki }), revision: 1 };
let writes = 0, online = true, race = false, proxy = null, failWrites = false;
const readOptions = async () => structuredClone(options);
globalThis.__subminerSetAnkiProxyUrl = async value => { const old = proxy; proxy = value; return old; };
const send = async (type, request, target) => {
if (type !== 'hd_options_write' || target !== 'hoshidicts-worker') throw Error('Unexpected request');
if (failWrites) throw Error('offline');
if (race) {
race = false;
options.anki.templates[0].tags = options.anki.tags = ['User edit'];
options.revision++;
}
if (request.baseRevision !== options.revision) throw Error('conflict');
options = { ...HDReaderOptions.normaliseOptions({ ...options, ...request.options }), revision: options.revision + 1 };
writes++;
};
const __gateway = { createAnkiGateway: () => ({ discover: async ({ model }) => ({
connected: online, model, models: Object.keys(__models), decks: ['Mining'],
fields: __models[model] || [], errors: online ? [] : ['offline'],
}) }) };
`,
context,
);
await vm.runInContext(
HACHIDORI_ANKI_SETTINGS_SCRIPT.replace("await import('./anki-templates.js')", '__templates')
.replace("await import('./anki-setup.js')", '__setup')
.replace("await import('./anki.js')", '__gateway'),
context,
);
const run = async (script: string): Promise<unknown> =>
structuredClone(await vm.runInContext(script, context));
const sync = () =>
run(
`__subminerSyncAnkiSettings(${JSON.stringify({
server: 'http://127.0.0.1:8766',
deck: 'Mining',
forceOverride: true,
hints: buildHachidoriAnkiHints(config),
})})`,
);
return { run, sync };
}
test('fresh Hachidori settings inherit deck, tags, a unique model and configured fields', async () => {
const h = await harness();
assert.deepEqual(await h.sync(), { updated: true, matched: true, pending: false });
assert.deepEqual(
await h.run('[options.anki.url, options.anki.deck, options.anki.model, options.anki.tags]'),
['http://127.0.0.1:8766', 'Mining', 'Japanese', config.tags],
);
assert.deepEqual(
await h.run(
'Object.fromEntries(Object.entries(options.anki.fieldTemplates).map(([key, row]) => [key, row.value]))',
),
{
Term: '{expression}',
Reading: '{reading}',
Definition: '{definition}',
Context: '{sentence}',
Pronunciation: '{audio}',
Image: '{screenshot}',
},
);
await h.sync();
assert.equal(await h.run('writes'), 1);
});
// SubMiner's new-card polling only watches ankiConnect.deck, so the first
// template's deck follows it the way Yomitan's term card deck does.
test('moves the first template to the SubMiner deck and preserves the rest of custom templates', async () => {
const h = await harness({
templates: [
{
id: 'default',
name: 'Custom',
deck: 'Own deck',
model: 'Japanese',
tags: ['own'],
fieldTemplates: {
Term: { value: '{reading}', overwriteMode: 'overwrite' },
Context: { value: '', overwriteMode: 'coalesce' },
},
},
{ id: 'second', name: 'Second', deck: 'Other', model: 'Other', tags: [] },
],
});
const before = (await h.run('options.anki.templates')) as Array<Record<string, unknown>>;
await h.sync();
assert.deepEqual(await h.run('options.anki.templates'), [
{ ...before[0], deck: 'Mining' },
...before.slice(1),
]);
assert.equal(await h.run('options.anki.deck'), 'Mining');
});
test('leaves an ambiguous model unset and retries discovery after Anki reconnects', async () => {
const h = await harness({}, { Japanese: modelFields, Second: modelFields });
await h.run('online = false');
assert.deepEqual(await h.sync(), { updated: true, matched: false, pending: true });
assert.equal(await h.run('options.anki.deck'), 'Mining');
await h.run('online = true');
await h.sync();
assert.equal(await h.run('options.anki.model'), '');
await h.run('delete __models.Second');
await h.sync();
assert.equal(await h.run('options.anki.model'), 'Japanese');
});
test('fills missing basic mappings only with fields belonging to the selected model', async () => {
const h = await harness(
{ model: 'Japanese', fields: { expression: 'Reading' } },
{ Japanese: ['Term', 'Reading', 'Context'] },
);
await h.sync();
assert.deepEqual(
await h.run(
'[options.anki.fields.expression, options.anki.fields.sentence, options.anki.fields.audio, options.anki.fieldTemplates]',
),
['Reading', 'Context', '', null],
);
});
test('re-reads concurrent settings edits before retrying its revisioned write', async () => {
const h = await harness();
await h.run('race = true');
await h.sync();
assert.deepEqual(await h.run('options.anki.tags'), ['User edit']);
assert.equal(await h.run('options.anki.model'), 'Japanese');
});
test('restores the previous proxy marker when every settings write fails', async () => {
const h = await harness();
await h.run(`proxy = 'http://127.0.0.1:9000'; failWrites = true`);
await assert.rejects(h.sync(), /offline/);
assert.equal(await h.run('proxy'), 'http://127.0.0.1:9000');
});
test('keeps sentence audio out of captured-audio settings and uses wordAudio first', () => {
const hints = buildHachidoriAnkiHints({
fields: { audio: 'SentenceAudio', wordAudio: 'WordAudio' },
});
assert.equal(hints.fields.audio, 'WordAudio');
assert.equal('captureAudio' in hints.fields, false);
});
@@ -1,124 +0,0 @@
import type { AnkiConnectConfig } from '../../../types';
// Only settings with the same meaning in both apps cross this boundary.
export function buildHachidoriAnkiHints(config: AnkiConnectConfig) {
return {
tags: config.tags,
fields: {
expression: config.fields?.word?.trim(),
audio: (config.fields?.wordAudio || config.fields?.audio)?.trim(),
sentence: config.fields?.sentence?.trim(),
screenshot: config.fields?.image?.trim(),
},
model: config.isLapis?.enabled ? config.isLapis.sentenceCardModel?.trim() : undefined,
family: config.isLapis?.enabled
? 'lapis'
: config.isKiku?.enabled
? 'kiku'
: config.isSenren?.enabled
? 'senren'
: undefined,
};
}
// Installed inside the hidden Hachidori settings page. Uses local option writes
// even when dictionaries are remote or their host is offline.
export const HACHIDORI_ANKI_SETTINGS_SCRIPT = String.raw`
globalThis.__subminerSyncAnkiSettings = async ({ server, deck, forceOverride, hints }) => {
const previousProxy = await globalThis.__subminerSetAnkiProxyUrl(forceOverride ? server : null);
const { applyAnkiPreset, resolveAnkiTemplates } = await import('./anki-templates.js');
const { ankiSetupFamily } = await import('./anki-setup.js');
const { createAnkiGateway } = await import('./anki.js');
const gateway = createAnkiGateway({ timeoutMs: 2000 });
const discoveries = new Map();
const discover = async (anki, model) => {
const key = JSON.stringify([anki.url, anki.apiKey, model]);
if (!discoveries.has(key)) discoveries.set(key, gateway.discover({ ...anki, model }));
return discoveries.get(key);
};
for (let attempt = 0; attempt < 2; attempt += 1) {
const options = await readOptions();
let anki = options.anki;
const canReplaceServer = forceOverride || !anki.url || anki.url === server
|| anki.url === 'http://127.0.0.1:8765' || anki.url === previousProxy;
if (!canReplaceServer) return { updated: false, matched: false, reason: 'blocked-existing-server' };
anki = { ...anki, url: server };
const first = anki.templates[0];
if (!first) return { updated: false, matched: false, reason: 'no-templates' };
let template = { ...first, fields: { ...first.fields } };
// SubMiner's new-card polling only watches its configured deck, so the
// first template follows it like Yomitan's term card deck.
if (deck) template.deck = deck;
if (hints?.tags && JSON.stringify(first.tags) === JSON.stringify(['hachidori'])) {
template.tags = [...hints.tags];
}
let pending = false;
// Advanced templates include intentionally blank fields. Preserve them
// as a unit rather than replacing them with inferred mappings.
if (hints && first.fieldTemplates === null) {
let discovery = await discover(anki, template.model || hints.model || '');
if (!discovery.connected) {
pending = true;
} else {
if (!template.model) {
const candidates = hints.model
? discovery.models.filter(model => model === hints.model)
: discovery.models.filter(model => !hints.family || ankiSetupFamily(model) === hints.family);
const anchors = [hints.fields.expression, hints.fields.sentence].filter(Boolean);
const matches = [];
// Require a configured word and sentence field for an inferred
// model. Never choose the first of several compatible note types.
if (anchors.length === 2) for (const model of candidates) {
const result = await discover(anki, model);
if (!result.connected || result.errors.length) { pending = true; break; }
if (anchors.every(field => result.fields.includes(field))) matches.push(result);
}
if (!pending && matches.length === 1) {
discovery = matches[0];
template.model = discovery.model;
}
}
if (template.model && discovery.model === template.model && discovery.fields.length) {
const fields = discovery.fields;
for (const [semantic, field] of Object.entries(hints.fields)) {
if (!template.fields[semantic] && field && fields.includes(field)) template.fields[semantic] = field;
}
if (Object.values(first.fields).every(value => !value)) {
const preset = applyAnkiPreset(template, fields, ankiSetupFamily(template.model) || 'automatic');
const configured = resolveAnkiTemplates(template, fields).templates;
const markers = new Set(Object.entries(template.fields)
.filter(([, value]) => value).map(([key]) => '{' + key + '}'));
for (const row of Object.values(preset.fieldTemplates)) {
for (const marker of markers) row.value = row.value.replaceAll(marker, '');
}
for (const [field, row] of Object.entries(configured)) {
if (row.value) preset.fieldTemplates[field] = row;
}
template = preset;
}
}
}
}
// Hachidori retains a compatibility projection of its first template.
// Updating both prevents its normalizer from restoring stale values.
const templateConfig = Object.fromEntries(globalThis.HDReaderOptions.ANKI_TEMPLATE_CONFIG_KEYS
.map(key => [key, template[key]]));
anki = { ...anki, ...templateConfig,
templates: [template, ...anki.templates.slice(1)] };
const changed = JSON.stringify(anki) !== JSON.stringify(options.anki);
try {
if (changed) await send('hd_options_write', {
baseRevision: options.revision, options: { anki },
}, 'hoshidicts-worker');
return { updated: changed, matched: !pending, pending };
} catch (error) {
if (attempt > 0) {
// The server was never written, so the marker must not claim it.
await globalThis.__subminerSetAnkiProxyUrl(previousProxy);
throw error;
}
// Re-read after a concurrent settings save before filling anything.
}
}
};
`;
@@ -1,158 +0,0 @@
import assert from 'node:assert/strict';
import { mkdtemp, writeFile } from 'node:fs/promises';
import { createServer } from 'node:http';
import { once } from 'node:events';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import {
resolveHachidoriManagementUrl,
uploadHachidoriDictionary,
} from './hachidori-dictionary-import';
import { importYomitanDictionaryFromZip } from './yomitan-parser-runtime';
import { createDeps } from './yomitan-scan-test-harness';
test('linked Hachidori uploads replacement bytes, retries a busy host, and never invokes local import', async () => {
const zipPath = path.join(await mkdtemp(path.join(os.tmpdir(), 'hachi-import-')), 'merged.zip');
const archive = Buffer.from('PK-test-archive');
await writeFile(zipPath, archive);
let attempts = 0;
const server = createServer(async (request, response) => {
attempts += 1;
assert.equal(request.url, '/import?name=merged.zip&replace=true');
assert.equal(request.method, 'POST');
const chunks: Buffer[] = [];
for await (const chunk of request) chunks.push(Buffer.from(chunk));
assert.deepEqual(Buffer.concat(chunks), archive);
response.writeHead(attempts === 1 ? 409 : 200, { 'Content-Type': 'application/json' });
response.end(
JSON.stringify(attempts === 1 ? { error: 'busy' } : { ok: true, report: { success: true } }),
);
});
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const address = server.address();
assert.ok(address && typeof address === 'object');
const managementUrl = `http://127.0.0.1:${address.port}`;
let connected = true;
const deps = {
...createDeps(async (script) => {
assert.ok(script.includes('hd_sharing_status'), 'must not invoke local ZIP automation');
return {
ok: true,
sharing: {
client: {
linked: true,
connected,
address: 'ws://127.0.0.1:8771/link',
host: { name: 'Hachidori Docker host', dictionaryCount: 8 },
},
},
};
}),
getYomitanExt: () => ({
id: 'hachi',
name: 'Hachidori',
version: '1',
path: '',
url: '',
manifest: {},
}),
};
try {
assert.equal(
await importYomitanDictionaryFromZip(zipPath, deps, { error: assert.fail }, managementUrl),
true,
);
assert.equal(attempts, 2);
const logger = { error: () => {} };
connected = false;
assert.equal(await importYomitanDictionaryFromZip(zipPath, deps, logger, managementUrl), false);
assert.equal(attempts, 2);
} finally {
server.closeAllConnections();
await new Promise<void>((resolve) => server.close(() => resolve()));
}
});
test('Hachidori upload requires a successful import report, not just HTTP success', async () => {
const zipPath = path.join(await mkdtemp(path.join(os.tmpdir(), 'hachi-import-')), 'merged.zip');
await writeFile(zipPath, 'bad archive');
const server = createServer((_request, response) => {
response.writeHead(200, { 'Content-Type': 'application/json' });
response.end(
JSON.stringify({ ok: true, report: { success: false, error: 'Invalid archive' } }),
);
});
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const address = server.address();
assert.ok(address && typeof address === 'object');
try {
await assert.rejects(
uploadHachidoriDictionary(zipPath, `http://127.0.0.1:${address.port}`),
/Invalid archive/,
);
} finally {
server.closeAllConnections();
await new Promise<void>((resolve) => server.close(() => resolve()));
}
});
test('management URL comes from the linked Docker host unless overridden', () => {
const docker = { address: 'ws://pve-main:8771/link', name: 'Hachidori Docker host' };
const warnings: string[] = [];
const warn = (message: string) => warnings.push(message);
assert.equal(
resolveHachidoriManagementUrl(docker, '', 'merged.zip', warn),
'http://pve-main:8780',
);
assert.equal(
resolveHachidoriManagementUrl(docker, 'http://pve-main:9000', 'merged.zip', warn),
'http://pve-main:9000',
);
assert.deepEqual(warnings, []);
assert.equal(
resolveHachidoriManagementUrl(
{ address: 'ws://127.0.0.1:8771/link', name: 'Hachidori Docker host' },
'http://localhost:8780',
'merged.zip',
warn,
),
'http://localhost:8780',
);
assert.deepEqual(warnings, []);
assert.equal(
resolveHachidoriManagementUrl(docker, 'http://127.0.0.1:8780', 'merged.zip', warn),
'http://127.0.0.1:8780',
);
assert.match(warnings[0] ?? '', /linked to pve-main/);
});
test('browser and app hosts without an override explain the manual import', () => {
assert.throws(
() =>
resolveHachidoriManagementUrl(
{ address: 'ws://desktop:8771/link', name: 'Chrome' },
'',
'/dicts/merged.zip',
),
/Chrome at desktop cannot receive dictionary uploads\. Import \/dicts\/merged\.zip/,
);
});
test('unreachable management API names the origin instead of a bare fetch failure', async () => {
const zipPath = path.join(await mkdtemp(path.join(os.tmpdir(), 'hachi-import-')), 'merged.zip');
await writeFile(zipPath, 'archive');
const server = createServer();
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const address = server.address();
assert.ok(address && typeof address === 'object');
await new Promise<void>((resolve) => server.close(() => resolve()));
const origin = `http://127.0.0.1:${address.port}`;
await assert.rejects(
uploadHachidoriDictionary(zipPath, origin),
new RegExp(`Could not reach the Hachidori management API at ${origin}: `),
);
});
@@ -1,96 +0,0 @@
import { readFile } from 'node:fs/promises';
import path from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import type { HachidoriHostStatus } from '../../../shared/hachidori-sharing';
// hachidori-docker names itself this in the sharing hello; browser and app hosts
// report their own name and have no management API to upload to.
export const HACHIDORI_DOCKER_HOST_NAME = 'Hachidori Docker host';
export const HACHIDORI_DOCKER_MANAGEMENT_PORT = 8780;
const LOOPBACK_HOSTNAMES = new Set(['127.0.0.1', 'localhost', '[::1]']);
function sameMachine(left: string, right: string): boolean {
return left === right || (LOOPBACK_HOSTNAMES.has(left) && LOOPBACK_HOSTNAMES.has(right));
}
// Picks the management origin that holds the linked host's dictionaries. The link
// address names the machine; `override` (hachidori.externalHostManagementUrl, already
// normalized to an origin) covers a non-default port or a reverse proxy.
export function resolveHachidoriManagementUrl(
host: Pick<Extract<HachidoriHostStatus, { kind: 'connected' }>, 'address' | 'name'>,
override: string,
zipPath: string,
warn?: (message: string) => void,
): string {
const linkHostname = new URL(host.address).hostname;
if (override) {
const overrideHostname = new URL(override).hostname;
if (!sameMachine(overrideHostname, linkHostname)) {
warn?.(
`hachidori.externalHostManagementUrl (${override}) points at ${overrideHostname}, but Hachidori is linked to ${linkHostname}; uploads may miss the linked host.`,
);
}
return override;
}
if (host.name !== HACHIDORI_DOCKER_HOST_NAME) {
throw new Error(
`The linked ${host.name} at ${linkHostname} cannot receive dictionary uploads. Import ${zipPath} from its Hachidori settings, or link a Hachidori Docker host.`,
);
}
return new URL(`http://${linkHostname}:${HACHIDORI_DOCKER_MANAGEMENT_PORT}`).origin;
}
// Upload from the main process: extension blob URLs cannot cross the sharing link,
// and the Docker management API deliberately rejects browser cross-origin writes.
export async function uploadHachidoriDictionary(zipPath: string, origin: string): Promise<void> {
const url = new URL('/import', origin);
url.searchParams.set('name', path.basename(zipPath));
url.searchParams.set('replace', 'true');
const bytes = await readFile(zipPath);
const signal = AbortSignal.timeout(300_000);
for (;;) {
let response: Response;
try {
response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/zip' },
body: bytes,
signal,
redirect: 'error',
});
} catch (error) {
// undici reports every network failure as "fetch failed"; the cause says why.
const cause = error instanceof Error && error.cause instanceof Error ? error.cause : error;
const reason = cause instanceof Error ? cause.message : String(cause);
throw new Error(`Could not reach the Hachidori management API at ${origin}: ${reason}`);
}
if (response.status === 409) {
await response.body?.cancel();
await delay(500, undefined, { signal });
continue;
}
const result: unknown = await response.json();
if (
response.ok &&
typeof result === 'object' &&
result !== null &&
'ok' in result &&
result.ok === true &&
'report' in result &&
typeof result.report === 'object' &&
result.report !== null &&
'success' in result.report &&
result.report.success === true
)
return;
const detail =
typeof result === 'object' &&
result !== null &&
'error' in result &&
typeof result.error === 'string'
? result.error
: JSON.stringify(result);
throw new Error(`Hachidori dictionary upload failed (${response.status}): ${detail}`);
}
}
@@ -1,493 +0,0 @@
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import path from 'node:path';
import test from 'node:test';
import { pathToFileURL } from 'node:url';
import * as vm from 'node:vm';
import { HACHIDORI_PARSER_BRIDGE_SCRIPT } from './hachidori-parser-bridge';
import {
requestYomitanScanTokens,
requestYomitanParseResults,
requestYomitanTermFrequencies,
syncYomitanDefaultAnkiServer,
addYomitanNoteViaSearch,
} from './yomitan-parser-runtime';
import { createDeps } from './yomitan-scan-test-harness';
import { selectYomitanParseTokens } from './parser-selection-stage';
const extensionPath = path.resolve(__dirname, '../../../../vendor/hachidori/extension');
const characterDictionary = 'SubMiner Character Dictionary (AniList 1)';
function lookupResult(matched: string, expression: string, reading: string, dictionary = 'JMdict') {
return {
matched,
deinflected: expression,
trace: [],
term: {
expression,
reading,
score: 0,
rules: 'v1',
glossaries: [{ dictionary, glossary: '["definition"]', termTags: '', definitionTags: '' }],
pitches: [],
frequencies: [{ dictionary: 'Frequency', frequencies: [{ value: 42, displayValue: '' }] }],
},
};
}
async function createHarness(emptyLibrary = false) {
const messages: Array<Record<string, unknown>> = [];
let dictionaryRevision = 2;
let optionRevision = 3;
let anki = {
url: 'http://127.0.0.1:8765',
templates: [
{ id: 'default', name: 'Default', deck: 'Old', model: 'Japanese', fields: {} },
{ id: 'second', name: 'Second', deck: 'Other', model: 'Japanese', fields: {} },
],
};
let dictionaries = [
{ id: 'terms', title: 'JMdict', displayName: 'Main dictionary', enabled: true, revision: '1' },
{ id: 'names', title: characterDictionary, enabled: true, revision: '1' },
{
id: 'frequency',
title: 'Frequency',
enabled: true,
revision: '1',
frequencyMode: 'rank-based',
frequencyCount: 1,
},
];
let duplicate = false;
let proxyUrl: unknown;
let loadingStatusReplies = 0;
let busyEngineReplies = 0;
const apiModule: unknown = await import(
pathToFileURL(path.join(extensionPath, 'api-host.js')).href
);
const context = vm.createContext({
__testApiModule: apiModule,
setTimeout,
crypto,
URL,
Blob,
Uint8Array,
atob,
chrome: {
storage: {
local: {
set: async (value: Record<string, unknown>) => {
proxyUrl = value.subminerAnkiProxyUrl;
},
get: async () => ({
options: { anki, scanLength: 40, revision: optionRevision },
subminerAnkiProxyUrl: proxyUrl,
}),
},
},
runtime: {
getManifest: () => ({ version: 'test' }),
sendMessage: async (message: Record<string, unknown>) => {
messages.push(structuredClone(message));
if (message.type === 'hd_status') {
const loading = loadingStatusReplies > 0;
if (loading) loadingStatusReplies -= 1;
return { ok: true, ready: !loading, loading };
}
if (busyEngineReplies > 0 && message.target !== 'hachidori-anki') {
busyEngineReplies -= 1;
return { ok: false, error: 'the dictionary engine is busy mutating' };
}
switch (message.type) {
case 'hd_state_read':
return {
ok: true,
state: emptyLibrary ? null : { revision: dictionaryRevision, dictionaries },
};
case 'hd_lookup': {
const text = String(message.text);
const candidates = [
lookupResult('食べた', '食べる', 'たべる'),
lookupResult('食べる', '食べる', 'たべる'),
lookupResult('ミナト', 'ミナト', 'みなと', characterDictionary),
];
return {
ok: true,
generation: 1,
results: candidates.filter((result) => text.startsWith(result.matched)),
};
}
case 'hd_options_write': {
if (message.baseRevision !== optionRevision) return { ok: false, error: 'conflict' };
const update = message.options;
assert.ok(update && typeof update === 'object' && 'anki' in update);
const value = update.anki;
assert.ok(
value &&
typeof value === 'object' &&
'url' in value &&
typeof value.url === 'string',
);
assert.ok('templates' in value && Array.isArray(value.templates));
anki = { url: value.url, templates: value.templates };
optionRevision += 1;
return { ok: true };
}
case 'hd_apply_state': {
assert.equal(message.baseRevision, dictionaryRevision);
assert.ok(Array.isArray(message.dictionaries));
dictionaries = message.dictionaries;
dictionaryRevision += 1;
return { ok: true };
}
case 'hd_anki_status':
return { ok: true, configKey: 'configuration' };
case 'hd_anki_preflight':
return {
ok: true,
canAdd: !duplicate,
state: duplicate ? 'duplicate' : 'addable',
noteIds: duplicate ? [15] : [],
};
case 'hd_anki_submit':
return { ok: true, state: 'added', noteId: 19 };
case 'hd_import':
return { ok: true, report: { success: true } };
case 'hd_remove':
return { ok: true };
default:
throw new Error('Unexpected native request: ' + String(message.type));
}
},
},
},
});
vm.runInContext('globalThis.window = globalThis', context);
vm.runInContext(await readFile(path.join(extensionPath, 'reader-options.js'), 'utf8'), context);
const script = HACHIDORI_PARSER_BRIDGE_SCRIPT.replace(
"await import('./api-host.js')",
'__testApiModule',
).replace("await import('./reader-options.js')", 'Promise.resolve()');
await vm.runInContext(script, context);
const run = async (code: string): Promise<unknown> =>
structuredClone(await vm.runInContext(code, context));
const invoke = (action: string, params?: unknown) =>
run(`new Promise((resolve, reject) => {
__subminerDictionarySendMessage(${JSON.stringify({ action, params })}, response => {
if (response.error) reject(new Error(response.error.message)); else resolve(response.result);
});
})`);
return {
deps: createDeps(run),
run,
invoke,
messages,
anki: () => anki,
proxyUrl: () => proxyUrl,
setAnkiServer: (url: string) => {
anki = { ...anki, url };
},
setAnkiTemplates: (templates: typeof anki.templates) => {
anki = { ...anki, templates };
},
dictionaries: () => dictionaries,
disableDictionary: (id: string) => {
dictionaries = dictionaries.map((entry) =>
entry.id === id ? { ...entry, enabled: false } : entry,
);
dictionaryRevision += 1;
},
setDuplicate: () => {
duplicate = true;
},
changeRevision: () => {
optionRevision += 1;
},
setLoadingStatusReplies: (count: number) => {
loadingStatusReplies = count;
},
setBusyEngineReplies: (count: number) => {
busyEngineReplies = count;
},
};
}
test('Hachidori runs the shared scanner with inflected offsets, headwords, names and frequencies', async () => {
const harness = await createHarness();
const tokens = await requestYomitanScanTokens(
'ミナト 食べた',
harness.deps,
{ error: assert.fail },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
},
);
assert.ok(tokens);
assert.equal(tokens[0]?.surface, 'ミナト');
assert.equal(tokens[0]?.isNameMatch, true);
assert.equal(tokens[1]?.surface, '食べた');
assert.equal(tokens[1]?.headword, '食べる');
assert.equal(tokens[1]?.startPos, 4);
assert.equal(tokens[1]?.endPos, 7);
assert.equal(tokens[1]?.frequencyRank, 42);
assert.deepEqual(tokens[1]?.wordClasses, ['v1']);
const frequencies = await requestYomitanTermFrequencies(
[{ term: '食べる', reading: 'たべる' }],
harness.deps,
{ error: assert.fail },
);
assert.equal(frequencies[0]?.frequency, 42);
assert.equal(frequencies[0]?.dictionary, 'Frequency');
});
test('Hachidori frequency lookups match API headwords, readings and requested dictionaries', async () => {
const harness = await createHarness();
const query = (term: string, reading: string | null, dictionaries = ['Frequency']) =>
harness.invoke('getTermFrequencies', { termReadingList: [{ term, reading }], dictionaries });
assert.deepEqual(await query('食べる', 'たべる'), [
{
term: '食べる',
reading: 'たべる',
hasReading: false,
dictionary: 'Frequency',
frequency: 42,
displayValue: null,
displayValueParsed: false,
},
]);
assert.deepEqual(await query('食べる', 'べつのよみ'), []);
assert.deepEqual(await query('食べる', null, ['Other frequency']), []);
assert.deepEqual(await query('食べるだけ', null), []);
assert.deepEqual(await query('頻度だけ', null), []);
assert.deepEqual(await query('食べる', null), await query('食べる', 'たべる'));
});
test('Hachidori syncs the Anki endpoint and every term template through revisioned writes', async () => {
const harness = await createHarness();
const synced = await syncYomitanDefaultAnkiServer(
'http://127.0.0.1:8766',
harness.deps,
{ error: assert.fail },
{ deck: 'Mining' },
);
assert.equal(synced, true);
assert.equal(harness.anki().url, 'http://127.0.0.1:8766');
assert.deepEqual(
harness.anki().templates.map((template) => template.deck),
['Mining', 'Mining'],
);
assert.equal(harness.proxyUrl(), null);
assert.equal(
await syncYomitanDefaultAnkiServer(
'http://127.0.0.1:8766',
harness.deps,
{ error: assert.fail },
{ forceOverride: true },
),
true,
);
assert.equal(harness.proxyUrl(), 'http://127.0.0.1:8766');
await assert.rejects(
harness.invoke('setAllSettings', {
value: {
hachidoriRevisions: { dictionaries: 99, options: 99 },
profiles: [{ options: { dictionaries: [], anki: { server: '', cardFormats: [] } } }],
},
}),
/settings changed/,
);
});
test('Hachidori applies only SubMiner changes when its settings moved on meanwhile', async () => {
const harness = await createHarness();
const projected = (await harness.invoke('optionsGetFull')) as {
profiles: Array<{ options: { dictionaries: Array<{ name: string; enabled: boolean }> } }>;
};
const jmdict = projected.profiles[0]!.options.dictionaries.find(
(entry) => entry.name === 'JMdict',
);
jmdict!.enabled = false;
// Hachidori changed both revisions after SubMiner read them.
harness.disableDictionary('names');
harness.setAnkiServer('http://192.168.1.10:8765');
harness.changeRevision();
assert.equal(await harness.invoke('setAllSettings', { value: projected }), true);
assert.deepEqual(
harness.dictionaries().map((entry) => [entry.id, entry.enabled]),
[
['terms', false],
['names', false],
['frequency', true],
],
);
assert.equal(harness.anki().url, 'http://192.168.1.10:8765');
assert.equal(harness.messages.filter((message) => message.type === 'hd_options_write').length, 0);
});
test('Hachidori settings writes survive an empty template list', async () => {
const harness = await createHarness();
harness.setAnkiTemplates([]);
assert.equal(
await syncYomitanDefaultAnkiServer(
'http://127.0.0.1:8766',
harness.deps,
{ error: assert.fail },
{
forceOverride: true,
},
),
true,
);
assert.equal(harness.anki().url, 'http://127.0.0.1:8766');
});
test('Hachidori initializes an empty library before the first dictionary import', async () => {
const harness = await createHarness(true);
assert.deepEqual(await harness.invoke('getDictionaryInfo'), []);
assert.equal(
await syncYomitanDefaultAnkiServer(
'http://127.0.0.1:8766',
harness.deps,
{ error: assert.fail },
{ forceOverride: true },
),
true,
);
});
test('Hachidori restores direct AnkiConnect when disabling its managed proxy', async () => {
const harness = await createHarness();
const logger = { error: assert.fail };
assert.equal(
await syncYomitanDefaultAnkiServer('http://127.0.0.1:8766', harness.deps, logger, {
forceOverride: true,
}),
true,
);
assert.equal(harness.anki().url, 'http://127.0.0.1:8766');
assert.equal(
await syncYomitanDefaultAnkiServer('http://127.0.0.1:8765', harness.deps, logger),
true,
);
assert.equal(harness.anki().url, 'http://127.0.0.1:8765');
assert.equal(harness.proxyUrl(), null);
});
test('Hachidori preserves a custom Anki endpoint when disabling its managed proxy', async () => {
const harness = await createHarness();
const logger = { error: assert.fail };
assert.equal(
await syncYomitanDefaultAnkiServer('http://127.0.0.1:8766', harness.deps, logger, {
forceOverride: true,
}),
true,
);
harness.setAnkiServer('http://192.168.1.10:8765');
assert.equal(
await syncYomitanDefaultAnkiServer('http://127.0.0.1:8765', harness.deps, logger),
false,
);
assert.equal(harness.anki().url, 'http://192.168.1.10:8765');
assert.equal(harness.proxyUrl(), null);
});
test('Hachidori fallback parsing retains token boundaries, inflected readings and headwords', async () => {
const harness = await createHarness();
const parsed = await requestYomitanParseResults('ミナト 食べた', harness.deps, {
error: assert.fail,
});
const tokens = selectYomitanParseTokens(parsed, () => false, 'headword');
assert.deepEqual(
tokens?.map((token) => ({
surface: token.surface,
headword: token.headword,
reading: token.reading,
start: token.startPos,
})),
[
{ surface: 'ミナト', headword: 'ミナト', reading: 'みなと', start: 0 },
{ surface: '食べた', headword: '食べる', reading: 'たべた', start: 4 },
],
);
});
test('Hachidori stats mining returns note IDs and prevents duplicate submissions', async () => {
const harness = await createHarness();
assert.deepEqual(await addYomitanNoteViaSearch('食べる', harness.deps, { error: assert.fail }), {
noteId: 19,
duplicateNoteIds: [],
});
harness.setDuplicate();
assert.deepEqual(await addYomitanNoteViaSearch('食べる', harness.deps, { error: assert.fail }), {
noteId: null,
duplicateNoteIds: [15],
});
assert.equal(harness.messages.filter((message) => message.type === 'hd_anki_submit').length, 1);
});
test('Hachidori mining requests render native dictionary aliases and frequency markers', async () => {
const harness = await createHarness();
await addYomitanNoteViaSearch('食べる', harness.deps, { error: assert.fail });
const request = harness.messages.find((message) => message.type === 'hd_anki_preflight')?.request;
assert.ok(request && typeof request === 'object');
assert.ok('subminerEnrich' in request && request.subminerEnrich === false);
const native: unknown = await import(
pathToFileURL(path.join(extensionPath, 'anki-values.js')).href
);
assert.ok(native && typeof native === 'object' && 'buildAnkiFields' in native);
assert.equal(typeof native.buildAnkiFields, 'function');
if (typeof native.buildAnkiFields !== 'function') assert.fail('Native renderer is unavailable');
const fields: unknown = await native.buildAnkiFields(
request,
{
Dictionary: { value: '{dictionary-alias}' },
Frequency: { value: '{single-frequency-frequency}' },
Rank: { value: '{frequency-harmonic-rank}' },
},
{},
);
assert.deepEqual(fields, {
Dictionary: 'Main dictionary',
Frequency: '<ul style="text-align: left;"><li>Frequency: </li></ul>',
Rank: '42',
});
assert.ok('dictionaryIds' in request);
assert.deepEqual(request.dictionaryIds, {
JMdict: 'terms',
[characterDictionary]: 'names',
Frequency: 'frequency',
});
assert.deepEqual(
harness.messages.find((message) => message.type === 'hd_anki_submit')?.request,
request,
);
});
test('Hachidori settings automation imports ZIP bytes and removes the matching dictionary ID', async () => {
const harness = await createHarness();
await harness.run(
"__subminerYomitanSettingsAutomation.importDictionaryArchiveBase64('UEs=', 'characters.zip')",
);
await harness.run(
`__subminerYomitanSettingsAutomation.deleteDictionary(${JSON.stringify(characterDictionary)})`,
);
const imported = harness.messages.find((message) => message.type === 'hd_import');
assert.equal(imported?.fileName, 'characters.zip');
assert.match(String(imported?.blobUrl), /^blob:/);
const removed = harness.messages.find((message) => message.type === 'hd_remove');
assert.equal(removed?.id, 'names');
});
test('Hachidori waits for a loading engine and retries busy requests before answering', async () => {
const harness = await createHarness();
harness.setLoadingStatusReplies(2);
harness.setBusyEngineReplies(1);
const dictionaries = (await harness.invoke('getDictionaryInfo')) as Array<{ title: string }>;
assert.equal(dictionaries.length, 3);
const statusCalls = harness.messages.filter((message) => message.type === 'hd_status').length;
assert.ok(statusCalls >= 3, `expected repeated status polls, saw ${statusCalls}`);
const tokens = await requestYomitanScanTokens('食べた', harness.deps, { error: assert.fail });
assert.equal(tokens?.[0]?.headword, '食べる');
assert.equal(tokens?.[0]?.frequencyRank, 42);
});
@@ -1,270 +0,0 @@
// Hachidori exposes its own storage and engine protocol. Adapt it only inside
// SubMiner's hidden extension windows so the shared scanner keeps its matching,
// character-name and frequency semantics without changing Hachidori's pages.
import { HACHIDORI_ANKI_SETTINGS_SCRIPT } from './hachidori-anki-settings';
export const HACHIDORI_SESSION_PARTITION = 'persist:hachidori';
export const HACHIDORI_PARSER_BRIDGE_SCRIPT = String.raw`
(async () => {
if (globalThis.__subminerDictionarySendMessage) return;
const { createApiHost } = await import('./api-host.js');
await import('./reader-options.js');
const send = async (type, fields = {}, target = 'hoshidicts-offscreen') => {
const reply = await chrome.runtime.sendMessage({
target, type, requestId: crypto.randomUUID(), ...fields,
});
if (!reply || reply.ok !== true) {
throw new Error(reply?.error || 'Hachidori returned an invalid response');
}
return reply;
};
// Hachidori reports an empty library while its dictionaries load and rejects
// engine requests during imports. Wait for a settled engine so SubMiner never
// caches an empty dictionary list or a token-less scan.
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
const waitForEngineReady = async () => {
const deadline = Date.now() + 120000;
for (;;) {
const reply = await send('hd_status').catch(() => null);
if (reply?.ready === true && reply.loading !== true) return;
if (Date.now() >= deadline) {
throw new Error(reply?.ready === true
? 'Hachidori is still importing dictionaries' : 'Hachidori dictionary engine is not ready');
}
await sleep(250);
}
};
const engine = async (type, fields = {}, target) => {
for (let attempt = 0; ; attempt += 1) {
await waitForEngineReady();
try {
return await send(type, fields, target);
} catch (error) {
if (attempt >= 3 || !/busy mutating/.test(error.message)) throw error;
await sleep(250);
}
}
};
const readState = async () => (await engine('hd_state_read', {}, 'hoshidicts-worker')).state
?? { revision: 0, dictionaries: [] };
const readOptions = async () => {
const stored = (await chrome.storage.local.get('options')).options;
return { ...globalThis.HDReaderOptions.normaliseOptions(stored), revision: stored?.revision ?? 0 };
};
globalThis.__subminerSetAnkiProxyUrl = async url => {
const previous = (await chrome.storage.local.get('subminerAnkiProxyUrl')).subminerAnkiProxyUrl;
await chrome.storage.local.set({ subminerAnkiProxyUrl: url });
return typeof previous === 'string' ? previous : null;
};
${HACHIDORI_ANKI_SETTINGS_SCRIPT}
const api = createApiHost({
engine: fields => engine(fields.type, fields),
render: fields => send(fields.type, fields, 'hachidori-anki-render'),
readDictionaries: async () => (await readState()).dictionaries,
readAudioSources: async () => (await readOptions()).audioSources.filter(source => source.enabled),
version: chrome.runtime.getManifest().version,
});
// Snapshot of the last projection SubMiner received, so a later write can
// be reduced to the fields SubMiner actually changed.
let lastProjection = null;
async function optionsGetFull() {
const [state, options] = await Promise.all([readState(), readOptions()]);
const revisions = { dictionaries: state.revision, options: options.revision };
lastProjection = {
revisions,
enabledByTitle: Object.fromEntries(state.dictionaries.map(entry => [entry.title, entry.enabled !== false])),
server: options.anki.url,
deckByFormat: Object.fromEntries(options.anki.templates.map(template => [template.id, template.deck])),
};
return {
profileCurrent: 0,
hachidoriRevisions: revisions,
profiles: [{ name: 'Hachidori', options: {
scanning: { length: options.scanLength },
dictionaries: state.dictionaries.map((entry, index) => ({
name: entry.title, alias: entry.displayName || entry.title,
id: index, enabled: entry.enabled !== false,
})),
anki: {
server: options.anki.url,
cardFormats: options.anki.templates.map(template => ({
id: template.id, type: 'term', enabled: true, deck: template.deck,
})),
},
} }],
};
}
async function setAllSettings(value) {
const projected = value.profiles?.[0]?.options;
const revisions = value.hachidoriRevisions;
if (!projected || !revisions) throw new Error('Invalid Hachidori settings projection');
const base = lastProjection
&& lastProjection.revisions.dictionaries === revisions.dictionaries
&& lastProjection.revisions.options === revisions.options ? lastProjection : null;
const enabledByTitle = Object.fromEntries(projected.dictionaries.map(item => [item.name, item.enabled === true]));
const deckByFormat = Object.fromEntries(projected.anki.cardFormats.map(format => [format.id, format.deck]));
for (let attempt = 0; ; attempt += 1) {
const [state, options] = await Promise.all([readState(), readOptions()]);
const current = state.revision === revisions.dictionaries && options.revision === revisions.options;
if (!current && !base) {
throw new Error('Hachidori settings changed while SubMiner was updating them; retry the action');
}
// Once Hachidori has moved on, apply only SubMiner's own changes on top
// of the newer settings instead of replaying the stale projection.
const enabledFor = title => enabledByTitle[title] === true;
const dictionaries = state.dictionaries.map(entry =>
current || (base.enabledByTitle[entry.title] === true) !== enabledFor(entry.title)
? { ...entry, enabled: enabledFor(entry.title) } : entry);
const templates = options.anki.templates.map(template => {
const deck = deckByFormat[template.id];
return deck !== undefined && (current || base.deckByFormat[template.id] !== deck)
? { ...template, deck } : template;
});
const url = current || base.server !== projected.anki.server ? projected.anki.server : options.anki.url;
const anki = { ...options.anki, url, templates, deck: templates[0]?.deck ?? options.anki.deck };
try {
if (JSON.stringify(dictionaries) !== JSON.stringify(state.dictionaries)) {
await engine('hd_apply_state', { baseRevision: state.revision, dictionaries });
}
if (JSON.stringify(anki) !== JSON.stringify(options.anki)) {
await engine('hd_options_write', { baseRevision: options.revision, options: { anki } }, 'hoshidicts-worker');
}
return true;
} catch (error) {
// A write raced another Hachidori change; re-read once and reapply the delta.
if (attempt > 0 || !base) throw error;
}
}
}
async function getTermFrequencies({ termReadingList, dictionaries }) {
const terms = [...new Set(termReadingList.map(pair => pair.term))];
const { results } = await api({ type: 'hd_api_term_entries', terms });
const frequencies = [];
for (const result of results) {
const term = terms[result.index];
const pairs = termReadingList.filter(pair => pair.term === term);
for (const entry of result.dictionaryEntries) {
for (const value of entry.frequencies) {
const headword = entry.headwords[value.headwordIndex];
if (!headword || headword.term !== term || !dictionaries.includes(value.dictionary)) continue;
if (!pairs.some(pair => pair.reading === null || pair.reading === headword.reading)) continue;
// Upstream does not expose the frequency entry's original reading.
// Keep its API flag and associate the value with the matched headword.
frequencies.push({ term, reading: headword.reading || null,
hasReading: value.hasReading, dictionary: value.dictionary,
frequency: value.frequency, displayValue: value.displayValue,
displayValueParsed: value.displayValueParsed });
}
}
}
return frequencies;
}
// Hachidori's public tokenize API emits display furigana without headwords.
// SubMiner's fallback requires one group per token and a dictionary form.
async function parseText({ text, scanLength }) {
const content = [];
let position = 0;
while (position < text.length) {
const rest = text.slice(position);
const reply = await engine('hd_lookup', { text: rest, maxResults: 1, scanLength });
const result = reply.results[0];
if (!result?.matched || !rest.startsWith(result.matched)) {
const character = String.fromCodePoint(rest.codePointAt(0));
content.push([{ text: character, reading: '' }]);
position += character.length;
continue;
}
const term = result.term;
// Keep the inflected ending in the reading, just as the main scanner does.
let stem = 0;
while (stem < term.expression.length && stem < result.matched.length && term.expression[stem] === result.matched[stem]) stem += 1;
const ending = term.expression.slice(stem);
const reading = stem > 0 && term.reading.endsWith(ending)
? term.reading.slice(0, term.reading.length - ending.length) + result.matched.slice(stem)
: term.reading;
content.push([{ text: result.matched, reading, headwords: [[{ term: term.expression }]] }]);
position += result.matched.length;
}
return [{ source: 'scanning-parser', index: 0, content }];
}
async function invoke(action, params) {
switch (action) {
case 'optionsGetFull': return optionsGetFull();
case 'setAllSettings': return setAllSettings(params.value);
case 'getDictionaryInfo': return (await readState()).dictionaries.map(entry => ({
title: entry.title, revision: entry.revision, frequencyMode: entry.frequencyMode,
}));
case 'termsFind': {
const reply = await api({ type: 'hd_api_term_entries', terms: [params.text] });
return reply.results[0];
}
case 'parseText': return parseText(params);
case 'getTermFrequencies': return getTermFrequencies(params);
default: throw new Error('Unsupported Hachidori parser action: ' + action);
}
}
globalThis.__subminerDictionarySendMessage = ({ action, params }, callback) => {
void invoke(action, params).then(result => callback({ result }), error => callback({ error: { message: error.message } }));
};
async function importArchive(blob, fileName) {
const blobUrl = URL.createObjectURL(blob);
try {
const reply = await engine('hd_import', { blobUrl, fileName });
if (reply.report?.success !== true) throw new Error(reply.report?.error || 'Hachidori dictionary import failed');
} finally {
URL.revokeObjectURL(blobUrl);
}
}
globalThis.__subminerYomitanSettingsAutomation = {
ready: true,
async importDictionaryArchiveUrl(url) {
const response = await fetch(url);
if (!response.ok) throw new Error('Could not read the dictionary archive');
await importArchive(await response.blob(), 'subminer-dictionary.zip');
},
async importDictionaryArchiveBase64(base64, fileName) {
const bytes = Uint8Array.from(atob(base64), character => character.charCodeAt(0));
await importArchive(new Blob([bytes], { type: 'application/zip' }), fileName);
},
async deleteDictionary(title) {
const dictionary = (await readState()).dictionaries.find(entry => entry.title === title);
if (dictionary) await engine('hd_remove', { id: dictionary.id, title });
},
};
globalThis.__subminerAddNote = async word => {
const lookup = await engine('hd_lookup', { text: word, maxResults: 1 });
const result = lookup.results[0];
if (!result) return { noteId: null, duplicateNoteIds: [] };
// Match the dictionary context supplied by Hachidori's popup to its
// native glossary, alias, and frequency template renderers.
const { dictionaries } = await readState();
const frequencyModes = new Map(dictionaries.map(entry => [entry.title, entry.frequencyMode]));
const term = { ...result.term, frequencies: result.term.frequencies.map(group =>
({ ...group, frequencyMode: frequencyModes.get(group.dictionary) })) };
const status = await send('hd_anki_status', {}, 'hachidori-anki');
const request = {
...result, term, generation: lookup.generation, sentence: word, searchQuery: word,
matchOffset: 0, documentTitle: 'SubMiner', popupSelectionText: '',
configKey: status.configKey, subminerEnrich: false,
dictionaryAliases: Object.fromEntries(dictionaries.filter(entry => entry.displayName)
.map(entry => [entry.title, entry.displayName])),
dictionaryIds: Object.fromEntries(dictionaries.map(entry => [entry.title, entry.id])),
frequencyDictionaries: dictionaries.filter(entry => entry.enabled !== false && entry.frequencyCount > 0)
.map(entry => entry.title),
captureUnavailable: ['screenshot', 'animation', 'audio'],
};
const preflight = await send('hd_anki_preflight', { request }, 'hachidori-anki');
if (preflight.canAdd !== true) {
if (preflight.state !== 'duplicate') throw new Error(preflight.error || 'Hachidori Anki mining is unavailable');
return { noteId: null, duplicateNoteIds: preflight.noteIds ?? [] };
}
const submitted = await send('hd_anki_submit', { request }, 'hachidori-anki');
if (submitted.state === 'added' || submitted.state === 'updated') {
return { noteId: submitted.noteId, duplicateNoteIds: [] };
}
if (submitted.state === 'duplicate') return { noteId: null, duplicateNoteIds: submitted.noteIds ?? [] };
throw new Error(submitted.error || 'Hachidori could not confirm the note write');
};
})();
`;
@@ -3,7 +3,6 @@ import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import test from 'node:test';
import * as vm from 'node:vm';
import {
countTermsFindLookups,
createDeps,
@@ -24,102 +23,6 @@ import {
upsertYomitanDictionarySettings,
} from './yomitan-parser-runtime';
test('Yomitan restores direct Anki after disabling its managed proxy without a page helper', async () => {
const options = { profiles: [{ options: { anki: { server: 'http://127.0.0.1:8765' } } }] };
let managedUrl: string | null = null;
const context = vm.createContext({
chrome: {
storage: {
local: {
get: async () => ({ subminerAnkiProxyUrl: managedUrl }),
set: async (value: { subminerAnkiProxyUrl: string | null }) => {
managedUrl = value.subminerAnkiProxyUrl;
},
},
},
runtime: {
sendMessage: (
message: { action: string },
callback: (response: { result: unknown }) => void,
) => callback({ result: message.action === 'optionsGetFull' ? options : null }),
},
},
});
const deps = createDeps(async (script) =>
structuredClone(await vm.runInContext(script, context)),
);
const logger = { error: assert.fail };
assert.equal(
await syncYomitanDefaultAnkiServer('http://127.0.0.1:8766', deps, logger, {
forceOverride: true,
}),
true,
);
assert.equal(managedUrl, 'http://127.0.0.1:8766');
assert.equal(await syncYomitanDefaultAnkiServer('http://127.0.0.1:8765', deps, logger), true);
assert.equal(options.profiles[0]?.options.anki.server, 'http://127.0.0.1:8765');
assert.equal(managedUrl, null);
});
for (const failure of ['optionsGetFull', 'setAllSettings', 'storageSet']) {
test(`Yomitan retries disabling its managed proxy after ${failure} fails`, async () => {
const proxyUrl = 'http://127.0.0.1:8766';
const directUrl = 'http://127.0.0.1:8765';
let options = { profiles: [{ options: { anki: { server: proxyUrl } } }] };
let managedUrl: string | null = proxyUrl;
let shouldFail = true;
const context = vm.createContext({
chrome: {
storage: {
local: {
get: async () => ({ subminerAnkiProxyUrl: managedUrl }),
set: async (value: { subminerAnkiProxyUrl: string | null }) => {
if (shouldFail && failure === 'storageSet') throw new Error('Storage unavailable');
managedUrl = value.subminerAnkiProxyUrl;
},
},
},
runtime: {
sendMessage: (
message: { action: string; params?: { value: typeof options } },
callback: (response: { result?: unknown; error?: { message: string } }) => void,
) => {
if (shouldFail && message.action === failure) {
callback({ error: { message: 'Settings unavailable' } });
} else if (message.action === 'optionsGetFull') {
callback({ result: structuredClone(options) });
} else if (message.action === 'setAllSettings' && message.params) {
options = structuredClone(message.params.value);
callback({ result: null });
} else {
assert.fail(`Unexpected action: ${message.action}`);
}
},
},
},
});
const deps = createDeps(async (script) =>
structuredClone(await vm.runInContext(script, context)),
);
const errors: string[] = [];
const logger = { error: (message: string) => errors.push(message) };
assert.equal(await syncYomitanDefaultAnkiServer(directUrl, deps, logger), false);
assert.equal(managedUrl, proxyUrl);
assert.equal(
options.profiles[0]?.options.anki.server,
failure === 'storageSet' ? directUrl : proxyUrl,
);
assert.equal(errors.length, 1);
shouldFail = false;
assert.equal(await syncYomitanDefaultAnkiServer(directUrl, deps, logger), true);
assert.equal(options.profiles[0]?.options.anki.server, directUrl);
assert.equal(managedUrl, null);
assert.equal(errors.length, 1);
});
}
test('syncYomitanDefaultAnkiServer updates default profile server when script reports update', async () => {
let scriptValue = '';
const deps = createDeps(async (script) => {
@@ -1,22 +1,7 @@
import type { BrowserWindow, Extension, Session } from 'electron';
import type { AnkiConnectConfig } from '../../../types';
import { buildHachidoriAnkiHints } from './hachidori-anki-settings';
import {
resolveHachidoriManagementUrl,
uploadHachidoriDictionary,
} from './hachidori-dictionary-import';
import {
buildHachidoriSharingScript,
parseHachidoriHostStatus,
type HachidoriSharingRequest,
} from '../../../shared/hachidori-sharing';
import * as fs from 'fs';
import * as http from 'http';
import * as path from 'path';
import {
HACHIDORI_PARSER_BRIDGE_SCRIPT,
HACHIDORI_SESSION_PARTITION,
} from './hachidori-parser-bridge';
import { selectYomitanParseTokens } from './parser-selection-stage';
import {
buildYomitanScanCallScript,
@@ -492,7 +477,7 @@ async function requestYomitanProfileMetadata(
(async () => {
const invoke = (action, params) =>
new Promise((resolve, reject) => {
(globalThis.__subminerDictionarySendMessage ?? chrome.runtime.sendMessage.bind(chrome.runtime))({ action, params }, (response) => {
chrome.runtime.sendMessage({ action, params }, (response) => {
if (chrome.runtime.lastError) {
reject(new Error(chrome.runtime.lastError.message));
return;
@@ -611,18 +596,6 @@ function logYomitanProfileDiagnostics(
logger.info?.('Yomitan active profile dictionaries loaded.', details);
}
function isHachidoriExtension(extension: Extension): boolean {
return extension.name === 'Hachidori';
}
// Without an explicit session, use the one the extension was loaded into:
// Hachidori lives in its own partition, Yomitan in the default session.
function resolveBackendSession(electron: typeof import('electron'), extension: Extension): Session {
return isHachidoriExtension(extension)
? electron.session.fromPartition(HACHIDORI_SESSION_PARTITION)
: electron.session.defaultSession;
}
async function ensureYomitanParserWindow(
deps: YomitanParserRuntimeDeps,
logger: LoggerLike,
@@ -633,20 +606,19 @@ async function ensureYomitanParserWindow(
return false;
}
const existingInitPromise = deps.getYomitanParserInitPromise();
if (existingInitPromise) {
return existingInitPromise;
}
const currentWindow = deps.getYomitanParserWindow();
if (currentWindow && !currentWindow.isDestroyed()) {
return true;
}
const existingInitPromise = deps.getYomitanParserInitPromise();
if (existingInitPromise) {
return existingInitPromise;
}
const initPromise = (async () => {
const { BrowserWindow } = electron;
const yomitanSession =
deps.getYomitanSession?.() ?? resolveBackendSession(electron, yomitanExt);
const { BrowserWindow, session } = electron;
const yomitanSession = deps.getYomitanSession?.() ?? session.defaultSession;
const parserWindow = new BrowserWindow({
show: false,
width: 800,
@@ -677,27 +649,11 @@ async function ensureYomitanParserWindow(
});
try {
const parserPage = isHachidoriExtension(yomitanExt) ? 'settings.html' : 'search.html';
await parserWindow.loadURL(`chrome-extension://${yomitanExt.id}/${parserPage}`);
await parserWindow.loadURL(`chrome-extension://${yomitanExt.id}/search.html`);
const readyPromise = deps.getYomitanParserReadyPromise();
if (readyPromise) {
await readyPromise;
}
if (isHachidoriExtension(yomitanExt)) {
await parserWindow.webContents.executeJavaScript(HACHIDORI_PARSER_BRIDGE_SCRIPT, true);
} else {
// did-finish-load precedes the search page's asynchronous backend initialization.
await parserWindow.webContents.executeJavaScript(
`(async () => {
const deadline = Date.now() + 10000;
while (typeof window.__subminerAddNote !== 'function') {
if (Date.now() >= deadline) throw new Error('Yomitan search page initialization timed out');
await new Promise(resolve => setTimeout(resolve, 50));
}
})()`,
true,
);
}
// Eagerly install the scan runtime so the first subtitle line does not
// pay the install round trip; failures fall back to the per-request
// install-and-retry path.
@@ -740,8 +696,8 @@ async function createYomitanExtensionWindow(
return null;
}
const { BrowserWindow } = electron;
const yomitanSession = deps.getYomitanSession?.() ?? resolveBackendSession(electron, yomitanExt);
const { BrowserWindow, session } = electron;
const yomitanSession = deps.getYomitanSession?.() ?? session.defaultSession;
const window = new BrowserWindow({
show: false,
width: 1200,
@@ -784,10 +740,6 @@ async function invokeYomitanSettingsAutomation<T>(
}
try {
const extension = deps.getYomitanExt();
if (extension && isHachidoriExtension(extension)) {
await settingsWindow.webContents.executeJavaScript(HACHIDORI_PARSER_BRIDGE_SCRIPT, true);
}
await settingsWindow.webContents.executeJavaScript(
`
(async () => {
@@ -940,7 +892,7 @@ export async function requestYomitanParseResults(
(async () => {
const invoke = (action, params) =>
new Promise((resolve, reject) => {
(globalThis.__subminerDictionarySendMessage ?? chrome.runtime.sendMessage.bind(chrome.runtime))({ action, params }, (response) => {
chrome.runtime.sendMessage({ action, params }, (response) => {
if (chrome.runtime.lastError) {
reject(new Error(chrome.runtime.lastError.message));
return;
@@ -970,7 +922,7 @@ export async function requestYomitanParseResults(
(async () => {
const invoke = (action, params) =>
new Promise((resolve, reject) => {
(globalThis.__subminerDictionarySendMessage ?? chrome.runtime.sendMessage.bind(chrome.runtime))({ action, params }, (response) => {
chrome.runtime.sendMessage({ action, params }, (response) => {
if (chrome.runtime.lastError) {
reject(new Error(chrome.runtime.lastError.message));
return;
@@ -1127,7 +1079,7 @@ async function fetchYomitanTermFrequencies(
(async () => {
const invoke = (action, params) =>
new Promise((resolve, reject) => {
(globalThis.__subminerDictionarySendMessage ?? chrome.runtime.sendMessage.bind(chrome.runtime))({ action, params }, (response) => {
chrome.runtime.sendMessage({ action, params }, (response) => {
if (chrome.runtime.lastError) {
reject(new Error(chrome.runtime.lastError.message));
return;
@@ -1170,7 +1122,7 @@ async function fetchYomitanTermFrequencies(
(async () => {
const invoke = (action, params) =>
new Promise((resolve, reject) => {
(globalThis.__subminerDictionarySendMessage ?? chrome.runtime.sendMessage.bind(chrome.runtime))({ action, params }, (response) => {
chrome.runtime.sendMessage({ action, params }, (response) => {
if (chrome.runtime.lastError) {
reject(new Error(chrome.runtime.lastError.message));
return;
@@ -1254,17 +1206,7 @@ function cacheFrequencyEntriesForPairs(
const key = makeTermReadingCacheKey(pair.term, pair.reading);
const exactEntries = groupedByPair.get(key);
const termEntries = groupedByTerm.get(pair.term) ?? [];
// Untagged frequency rows apply to every reading. A term-only query must
// retain all readings, rather than selecting only its untagged rows.
const untaggedEntries = groupedByPair.get(makeTermReadingCacheKey(pair.term, null)) ?? [];
frequencyCache.set(
key,
pair.reading === null
? termEntries
: exactEntries
? [...exactEntries, ...untaggedEntries]
: termEntries,
);
frequencyCache.set(key, exactEntries ?? termEntries);
}
}
@@ -1398,7 +1340,6 @@ export async function syncYomitanDefaultAnkiServer(
options?: {
forceOverride?: boolean;
deck?: string;
ankiConfig?: AnkiConnectConfig;
},
): Promise<boolean> {
const normalizedTargetServer = serverUrl.trim();
@@ -1418,7 +1359,7 @@ export async function syncYomitanDefaultAnkiServer(
(async () => {
const invoke = (action, params) =>
new Promise((resolve, reject) => {
(globalThis.__subminerDictionarySendMessage ?? chrome.runtime.sendMessage.bind(chrome.runtime))({ action, params }, (response) => {
chrome.runtime.sendMessage({ action, params }, (response) => {
if (chrome.runtime.lastError) {
reject(new Error(chrome.runtime.lastError.message));
return;
@@ -1438,13 +1379,6 @@ export async function syncYomitanDefaultAnkiServer(
const targetServer = ${JSON.stringify(normalizedTargetServer)};
const targetDeck = ${JSON.stringify(normalizedTargetDeck)};
const forceOverride = ${forceOverride ? 'true' : 'false'};
const hachidoriHints = ${JSON.stringify(options?.ankiConfig ? buildHachidoriAnkiHints(options.ankiConfig) : null)};
if (hachidoriHints && typeof globalThis.__subminerSyncAnkiSettings === 'function') {
return globalThis.__subminerSyncAnkiSettings({
server: targetServer, deck: targetDeck, forceOverride, hints: hachidoriHints,
});
}
const { subminerAnkiProxyUrl: previousManagedProxy } = await chrome.storage.local.get('subminerAnkiProxyUrl');
const optionsFull = await invoke("optionsGetFull", undefined);
const profiles = Array.isArray(optionsFull.profiles) ? optionsFull.profiles : [];
if (profiles.length === 0) {
@@ -1471,11 +1405,8 @@ export async function syncYomitanDefaultAnkiServer(
let changed = false;
if (currentServer !== targetServer) {
const canReplaceCurrent =
forceOverride || currentServer.length === 0 || currentServer === "http://127.0.0.1:8765" ||
(typeof previousManagedProxy === 'string' && currentServer === previousManagedProxy);
forceOverride || currentServer.length === 0 || currentServer === "http://127.0.0.1:8765";
if (!canReplaceCurrent) {
// A custom endpoint needs no settings change, but the proxy is no longer managed.
await chrome.storage.local.set({ subminerAnkiProxyUrl: null });
return { updated: false, matched: false, reason: "blocked-existing-server", currentServer, targetServer };
}
@@ -1513,28 +1444,17 @@ export async function syncYomitanDefaultAnkiServer(
}
}
if (changed) {
await invoke("setAllSettings", { value: optionsFull, source: "subminer" });
}
// Preserve the previous managed endpoint until settings are saved so failed switches can retry.
await chrome.storage.local.set({ subminerAnkiProxyUrl: forceOverride ? targetServer : null });
if (!changed) {
return { updated: false, matched: true, reason: "already-target", currentServer, targetServer, targetDeck };
}
await invoke("setAllSettings", { value: optionsFull, source: "subminer" });
return { updated: true, matched: true, currentServer, targetServer, targetDeck };
})();
`;
try {
const result = await parserWindow.webContents.executeJavaScript(script, true);
if (isObject(result) && result.pending === true) {
logger.info?.(
'Anki is unavailable; Hachidori field auto-population will retry when opened again',
);
return false;
}
const updated =
typeof result === 'object' &&
result !== null &&
@@ -1644,7 +1564,7 @@ function buildYomitanInvokeScript(actionLiteral: string, paramsLiteral: string):
(async () => {
const invoke = (action, params) =>
new Promise((resolve, reject) => {
(globalThis.__subminerDictionarySendMessage ?? chrome.runtime.sendMessage.bind(chrome.runtime))({ action, params }, (response) => {
chrome.runtime.sendMessage({ action, params }, (response) => {
if (chrome.runtime.lastError) {
reject(new Error(chrome.runtime.lastError.message));
return;
@@ -1666,25 +1586,6 @@ function buildYomitanInvokeScript(actionLiteral: string, paramsLiteral: string):
`;
}
export async function requestHachidoriSharing(
request: HachidoriSharingRequest,
deps: YomitanParserRuntimeDeps,
logger: LoggerLike,
) {
const extension = deps.getYomitanExt();
if (!extension || !isHachidoriExtension(extension)) throw new Error('Hachidori is not active.');
const ready = await ensureYomitanParserWindow(deps, logger);
const window = deps.getYomitanParserWindow();
if (!ready || !window || window.isDestroyed()) throw new Error('Hachidori is unavailable.');
const reply: unknown = await window.webContents.executeJavaScript(
buildHachidoriSharingScript(request),
true,
);
const status = parseHachidoriHostStatus(reply);
if (request.type !== 'hd_sharing_status') clearYomitanParserCachesForWindow(window);
return status;
}
async function invokeYomitanBackendAction<T>(
action: string,
params: unknown,
@@ -1803,7 +1704,6 @@ export async function importYomitanDictionaryFromZip(
zipPath: string,
deps: YomitanParserRuntimeDeps,
logger: LoggerLike,
hachidoriManagementUrl = '',
): Promise<boolean> {
const normalizedZipPath = zipPath.trim();
if (!normalizedZipPath || !fs.existsSync(normalizedZipPath)) {
@@ -1811,36 +1711,6 @@ export async function importYomitanDictionaryFromZip(
return false;
}
const extension = deps.getYomitanExt();
if (extension && isHachidoriExtension(extension)) {
try {
const host = await requestHachidoriSharing({ type: 'hd_sharing_status' }, deps, logger);
if (host.kind === 'disconnected' || host.kind === 'unavailable')
throw new Error(host.message);
if (host.kind === 'connected') {
const origin = resolveHachidoriManagementUrl(
host,
hachidoriManagementUrl,
normalizedZipPath,
logger.warn,
);
await uploadHachidoriDictionary(normalizedZipPath, origin);
const window = deps.getYomitanParserWindow();
if (window) clearYomitanParserCachesForWindow(window);
logger.info?.(
`Uploaded character dictionary to Hachidori host: ${path.basename(normalizedZipPath)}`,
);
return true;
}
} catch (error) {
logger.error(
'Hachidori character dictionary import failed:',
error instanceof Error ? error.message : String(error),
);
return false;
}
}
const supportsUrlImport = await invokeYomitanSettingsAutomation<boolean>(
`
(() => typeof globalThis.__subminerYomitanSettingsAutomation.importDictionaryArchiveUrl === "function")();
@@ -12,7 +12,7 @@ export type YomitanFrequencyMode = 'occurrence-based' | 'rank-based';
// Bump whenever the install script below changes so already-loaded parser
// windows re-install the new scan runtime instead of running the stale one.
export const YOMITAN_SCAN_RUNTIME_VERSION = 13;
export const YOMITAN_SCAN_RUNTIME_VERSION = 12;
export const YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL = '__subminer-yomitan-scan-runtime-missing__';
export interface YomitanScanRequestParams {
@@ -43,7 +43,7 @@ export const YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT = String.raw`
}
const invoke = (action, params) =>
new Promise((resolve, reject) => {
(globalThis.__subminerDictionarySendMessage ?? chrome.runtime.sendMessage.bind(chrome.runtime))({ action, params }, (response) => {
chrome.runtime.sendMessage({ action, params }, (response) => {
if (chrome.runtime.lastError) {
reject(new Error(chrome.runtime.lastError.message));
return;
Loaded 100 of 198 files, more files were not shown because too many files have changed in this diff. Show more