mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-20 00:15:27 -07:00
Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
80ffa26400
|
||
|
|
06b21a68fa
|
||
|
|
4ed878270f
|
||
|
|
1d1b0c7bb6
|
||
|
|
db61ce358d
|
||
|
|
e9778a945a | ||
|
|
7de73e16a1 | ||
|
|
273652f781 | ||
|
|
14c0086520
|
||
|
|
ff0bc0af6d | ||
|
|
00b1b79bf4 |
@@ -20,7 +20,7 @@
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.10.0",
|
||||
"@types/ws": "^8.18.1",
|
||||
"electron": "42.6.0",
|
||||
"electron": "43.4.1",
|
||||
"electron-builder": "26.15.3",
|
||||
"esbuild": "^0.25.12",
|
||||
"eslint": "^10.8.0",
|
||||
@@ -346,7 +346,7 @@
|
||||
|
||||
"ejs": ["ejs@3.1.10", "", { "dependencies": { "jake": "^10.8.5" }, "bin": { "ejs": "bin/cli.js" } }, "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA=="],
|
||||
|
||||
"electron": ["electron@42.6.0", "", { "dependencies": { "@electron-internal/extract-zip": "^1.0.1", "@electron/get": "^5.0.0", "@types/node": "^24.9.0" }, "bin": { "electron": "cli.js", "install-electron": "install.js" } }, "sha512-axGNgd+yCTg+vi1VEGrQqAj9WVWkePKwbICSAvMiT2eTaxhij9a/xhBHD6rXV8wrlW9ZfJzE5+xg752ImxrmTw=="],
|
||||
"electron": ["electron@43.4.1", "", { "dependencies": { "@electron-internal/extract-zip": "^1.0.1", "@electron/get": "^5.0.0", "@types/node": "^24.9.0" }, "bin": { "electron": "cli.js", "install-electron": "install.js" } }, "sha512-5b+EuiwkgG5iRcsEL34rimgRpkYp15SsfZOa0pC5kXs0Tb82TH4n95rpQzTZa7yRCbA7tm0WoEbuBL6NaAhAcA=="],
|
||||
|
||||
"electron-builder": ["electron-builder@26.15.3", "", { "dependencies": { "app-builder-lib": "26.15.3", "builder-util": "26.15.3", "builder-util-runtime": "9.7.0", "chalk": "^4.1.2", "ci-info": "^4.2.0", "dmg-builder": "26.15.3", "fs-extra": "^10.1.0", "lazy-val": "^1.0.5", "simple-update-notifier": "2.0.0", "yargs": "^17.6.2" }, "bin": { "electron-builder": "./cli.js", "install-app-deps": "./install-app-deps.js" } }, "sha512-a1KM5heqS3gQCZzizXEI8RjJy3QVogULPdeSknt76uLDpBIW/HDGsMg/XgP0riP6PI9COsRvFITKKGDqA8fJxA=="],
|
||||
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
type: fixed
|
||||
area: subtitles
|
||||
|
||||
- Typeset ASS karaoke and animated signs no longer flood the primary overlay, subtitle sidebar, immersion history, or sentence mining with repeated glyph fragments or full-line color phases. Matching timed comments and full-line boundary events recover the complete authored line without merging ordinary repeated dialogue or separately positioned signs, and dialogue spoken while a song's animation is on screen is kept intact instead of being replaced by the lyric. Entrance and exit frames that run past the authored line timing still resolve to the clean line during lyric transitions, and dialogue spoken while a song's animation is on screen enters immersion and subtitle history without the fragment lines beside it.
|
||||
- The secondary subtitle overlay drops layered duplicate lines from animated tracks, so a short stack of repeated words collapses to its distinct lines even when the full karaoke heuristic does not apply.
|
||||
@@ -0,0 +1,5 @@
|
||||
type: fixed
|
||||
area: dictionary
|
||||
|
||||
- Character dictionary generation, merged rebuilds, and imports no longer freeze the app (and trigger the compositor's "application not responding" dialog) on large dictionaries; snapshot reads/writes, archive building, and the character image/name lookup caches now do their heavy work off the UI's critical path.
|
||||
- Desktop progress notifications now update in place on Linux AppImage installs too: the AppImage's bundled libraries broke the system notify-send helper, which silently forced the flickering close-and-reopen notification fallback.
|
||||
@@ -0,0 +1,6 @@
|
||||
type: fixed
|
||||
area: dictionary
|
||||
|
||||
- Upgraded the desktop runtime to Electron 43.4.1 and added profile guards that block unsupported runtimes and Electron downgrades before Yomitan storage is loaded.
|
||||
- Development launches now use a separate `SubMiner-dev` profile unless production-profile access is explicitly requested.
|
||||
- Automatic character-dictionary changes now stop when a previously non-empty Yomitan profile suddenly reports zero dictionaries.
|
||||
@@ -0,0 +1,4 @@
|
||||
type: fixed
|
||||
area: overlay
|
||||
|
||||
- Fixed the overlay getting stuck on "Overlay loading" forever when startup stalls: mpv IPC connection attempts now time out and retry, switching sockets aborts obsolete attempts, and the plugin replaces its spinner with an actionable error if overlay content is still not ready after 30 seconds.
|
||||
@@ -0,0 +1,4 @@
|
||||
type: fixed
|
||||
area: subtitles
|
||||
|
||||
- Primary ASS subtitles now use the active parsed cue when it fully accounts for mpv's live text, preventing fill, border, blur, and shadow copies of the same full-span lyric from appearing repeatedly while preserving unmatched overlapping dialogue and signs.
|
||||
@@ -0,0 +1,4 @@
|
||||
type: fixed
|
||||
area: overlay
|
||||
|
||||
- Secondary subtitles now parse the selected ASS/SRT/VTT source with the primary subtitle deduplication pipeline, preventing layered animation text from appearing several times in the overlay, mined cards, and statistics. Live mpv text remains the fallback for unreadable tracks.
|
||||
@@ -0,0 +1,8 @@
|
||||
type: fixed
|
||||
area: stats
|
||||
|
||||
- Fixed Vocabulary totals and charts counting only the first browsing page instead of all tracked vocabulary, without delaying the rest of the page.
|
||||
- New-word history now uses permanent daily lexical rollups that apply the same vocabulary filters as the totals and normalize legacy second/millisecond timestamps; versioned background rebuilds repair existing history across legacy rollup-state schemas without dropping playback writes or clearing watch-time, activity, efficiency, and library charts.
|
||||
- Calendar-day chart labels now preserve the recorded local date in time zones west of UTC.
|
||||
- Vocabulary summary cards and charts refresh automatically after the word exclusion list changes, and failed or unfinished loads use bounded retries before showing an inline error with a Retry control.
|
||||
- Rapid exclusion edits no longer race each other; writes are sent in order so a slower earlier save cannot overwrite a newer list.
|
||||
@@ -64,6 +64,10 @@ make dev-watch # watch TS + renderer and launch E
|
||||
make dev-watch-macos # same as dev-watch, forcing --backend macos
|
||||
```
|
||||
|
||||
Development and debug launches use a separate `SubMiner-dev` profile so runtime experiments cannot modify the installed app's configuration or Yomitan dictionaries. To intentionally use the production profile for a development launch, set `SUBMINER_USE_PRODUCTION_PROFILE=1`. Use that override only after backing up the profile.
|
||||
|
||||
Always launch source builds through `bun run dev` or `bun run electron`. SubMiner refuses to load its profile when the running Electron major differs from the version pinned by the repository.
|
||||
|
||||
For mpv-plugin-driven testing without exporting `SUBMINER_BINARY_PATH` each run, set a one-time
|
||||
dev binary path with `mpv.subminerBinaryPath` in your SubMiner config. The launcher injects it into
|
||||
the mpv plugin at runtime:
|
||||
|
||||
@@ -82,7 +82,7 @@ Expandable session history with new-word activity, cumulative totals, and pause/
|
||||
|
||||
#### Vocabulary
|
||||
|
||||
Top repeated words (click a bar to open the word), new-word timeline, cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons.
|
||||
The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The word and kanji tables load first while those complete totals calculate separately. Top Repeated Words and New Words by Day use complete tracking history rather than the table's browsing page. New-word history is maintained as a permanent daily lexical rollup using the same token-visibility rules as the totals, including normalization of older timestamps stored in either seconds or milliseconds and retroactive corrections when tracked material is removed or reprocessed. On the first launch after an applicable upgrade, that history is version-rebuilt in the background and the chart refreshes when it is ready; if it remains unavailable, polling stops and an inline Retry control appears. The cards and charts also refresh automatically after the word exclusion list changes. The rest of the tab includes cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons.
|
||||
|
||||

|
||||
|
||||
@@ -138,6 +138,8 @@ Karaoke openings and animated signs are authored as one subtitle event per anima
|
||||
|
||||
Recording now collapses those runs as they happen, matching what the subtitle sidebar shows:
|
||||
|
||||
- When a typeset ASS file stores a clean lyric or sign in a timed authoring comment, or in full-line events surrounding generated fragments, the matching complete line is recorded once. The repeated glyph or clip-animation frames are not recorded. Dialogue spoken while such an animation is on screen records as itself, without the fragment lines beside it.
|
||||
- When karaoke styling redraws the same complete lyric across consecutive color or highlight phases, those phases are combined into one line with their full timing. Repeated ordinary dialogue remains separate.
|
||||
- When the active subtitle source has been parsed, its cue list has already had duplicate events and animation bursts merged. A line landing inside a surviving cue but after that cue's start is a frame the sidebar merged away, and is not recorded.
|
||||
- When no parsed cue covers the live timing, including while a subtitle source is changing or shifted, the strict metadata-free rule applies: a run of identical, contiguous lines each shorter than 0.1s stops being recorded after a few frames. Runs are tracked per line of text, so dual-line karaoke (a kanji and a romaji line frame-flipped together) collapses both lines. Ordinary repeated dialogue, and lines held for a normal beat, always record.
|
||||
|
||||
@@ -180,6 +182,7 @@ In practice:
|
||||
- Anime and episode pages keep lifetime totals from summary tables while session drill-down still reads retained sessions directly. With the current defaults, both are kept forever.
|
||||
- Trends can read the full available history because daily/monthly rollups are also kept forever by default.
|
||||
- Vocabulary and kanji totals are cumulative and not bounded by the raw session retention knobs.
|
||||
- New-word charts use their own permanent lexical daily rollups, which are not pruned by activity-rollup retention.
|
||||
|
||||
## Storage / Performance Model
|
||||
|
||||
@@ -349,6 +352,7 @@ Rollup tables:
|
||||
|
||||
- `imm_daily_rollups`
|
||||
- `imm_monthly_rollups`
|
||||
- `imm_lexical_daily_rollups` - permanent first-discovery counts for vocabulary and kanji chart history
|
||||
- `imm_rollup_state` - incremental rollup progress bookkeeping
|
||||
|
||||
Vocabulary tables:
|
||||
|
||||
@@ -12,6 +12,8 @@ When SubMiner parses the active subtitle source into a cue list, the sidebar bec
|
||||
- Clicking any cue seeks mpv to that timestamp.
|
||||
- The sidebar stays synchronized with the overlay - media transitions and subtitle source changes update both simultaneously.
|
||||
|
||||
For typeset ASS karaoke and animated signs, SubMiner collapses generated animation frames and repeated full-line color phases before they reach the sidebar. It recovers a clean complete line from a matching timed authoring comment or from full-line events surrounding generated fragments. Ordinary ASS comments, editor notes, alternate lines, repeated dialogue, and separately positioned signs remain distinct.
|
||||
|
||||
The sidebar only appears when a parsed cue list is available. External subtitle sources that SubMiner cannot parse (for example, embedded ASS tracks rendered directly by mpv) will not populate the sidebar.
|
||||
|
||||
## Layout Modes
|
||||
|
||||
@@ -109,6 +109,14 @@ If the overlay position is slightly off, right-click and drag on subtitle text t
|
||||
|
||||
If you haven't set up dictionaries yet, see [Yomitan setup](/usage#yomitan-setup) first.
|
||||
|
||||
**"Electron downgrade blocked" or "Unsupported Electron runtime"**
|
||||
|
||||
SubMiner refuses to load Yomitan storage when the current Electron major does not match the app build, or when the profile was previously opened by a newer Electron version. Launch the packaged app or use the repository's `bun run dev` command. Do not delete the runtime safety record merely to force an older Electron version to open the profile.
|
||||
|
||||
**"Yomitan reported zero dictionaries after previously reporting ..."**
|
||||
|
||||
SubMiner detected that a previously non-empty Yomitan profile suddenly appears empty. Automatic character-dictionary changes are blocked so they cannot normalize or overwrite the suspicious state. Close SubMiner, preserve the profile directory, and restore a known-good backup before importing or deleting dictionaries.
|
||||
|
||||
**"Yomitan extension not found in any search path"**
|
||||
|
||||
SubMiner bundles Yomitan and searches for it in these locations (in order):
|
||||
|
||||
@@ -155,6 +155,7 @@ The tray menu also includes `View Changelog`, which opens the in-app changelog m
|
||||
|
||||
- `--log-level` controls logger verbosity.
|
||||
- `--dev` and `--debug` are app/dev-mode switches; they are not log-level aliases.
|
||||
- `--dev` and `--debug` use a separate `SubMiner-dev` profile. They do not read or modify dictionaries and configuration from the installed app unless `SUBMINER_USE_PRODUCTION_PROFILE=1` is explicitly set.
|
||||
- `--background` starts at the default quieter logging level (`warn`), then follows `logging.level` after config loads. An explicit `--log-level` remains the override.
|
||||
- `--background` launched from a terminal detaches and returns the prompt; stop it with tray Quit or `SubMiner.AppImage --stop` (`SubMiner.exe --stop` on Windows).
|
||||
- Linux desktop launcher starts SubMiner with `--background` by default (via electron-builder `linux.executableArgs`).
|
||||
|
||||
@@ -70,18 +70,25 @@ interface SubtitleCue {
|
||||
startTime: number; // seconds
|
||||
endTime: number; // seconds
|
||||
text: string; // plain text, decoded from the source format
|
||||
source?: 'canonical-ass'; // recovered authored text for generated ASS animation
|
||||
animationStartTime?: number; // full generated-frame envelope; entrance/exit frames
|
||||
animationEndTime?: number; // run past the authored timing, live matching uses this
|
||||
}
|
||||
```
|
||||
|
||||
**Supported formats:**
|
||||
|
||||
- SRT/VTT: Regex-based parsing of timing lines + text content between timing blocks.
|
||||
- ASS: Parse `[Events]` section, extract `Dialogue:` lines, read the field order from the `Format:` row, and take everything after the Text field index as the text (Text can itself contain commas).
|
||||
- ASS: Parse the `[Events]` section, read the field order from the `Format:` row, and extract timed `Dialogue:` lines. Timed `Comment:` lines are normally ignored, but can supply canonical authored text when they match a nearby generated animation from the same style and actor. Text can itself contain commas.
|
||||
|
||||
**ASS decoding.** The parser is where ASS text is decoded, once, via `assToPlainText()` in `src/core/services/ass-text.ts`. That decoder mirrors mpv's `ass_to_plaintext` so a cue read from a file reads identically to the same line arriving live on `sub-text`: `{...}` override blocks are markup, `\pN … \p0` vector drawing runs are dropped rather than shown as text, `\N`/`\n`/`\h` are the only escapes (`\{`, `\}` and `\\` are not), and an unclosed `{` is rendered verbatim. Every layer downstream — renderer, timing tracker, tokenizer, tokenization cache keys — receives plain text and uses `normalizePlainSubtitleText()` for whitespace only, so nothing decodes the same string twice and one authored line always maps to one cache key.
|
||||
|
||||
**Duplicate collapsing.** Typeset scripts emit one `Dialogue:` event per animation frame, plus layered copies of the same line. The parser collapses identical text over an identical span unconditionally, and collapses contiguous same-text runs of at least three events when the run looks like an animation. For ASS that means shared style and actor plus authoring evidence: a temporal tag (`\t`, `\move`, `\k`/`\kf`/`\ko`/`\K`, or anything wrapped in `\t(...)`), an animated `Effect` column (`Karaoke`, `Banner`, `Scroll`), or override values that change across the run. Static tags shared by every event (`\pos`, an identical `\clip`) are not evidence. SRT/VTT carry no such metadata, so there collapsing needs at least five contiguous events all under 0.1s — the frame timing left behind by ASS-to-SRT conversion. The parser keeps this authoring metadata (style, actor, layer, `Effect`, parsed override commands, source order) private; `parseSubtitleCues()` returns only `SubtitleCue`.
|
||||
|
||||
ASS scripts can also redraw one complete lyric for two or more long color/highlight phases. Those flush-timed phases collapse separately from short animation frames when they share text, style, actor, and layer and carry direct animation evidence, such as temporal tags or changing non-spatial overrides. Spatial command changes do not prove a phase, so separately positioned signs remain distinct.
|
||||
|
||||
**Canonical animation recovery.** Some ASS producers keep the readable lyric or sign as a timed `Comment:` and generate hundreds of `Dialogue:` frames containing repeated glyphs or changing clip regions. Others retain the complete line as brief `Dialogue:` events around the generated fragments. A complete event is promoted only when nearby dialogue from the same style and actor forms a proven animation cluster and reconstructs its entire text in source order. The generated frames are then replaced by one cue marked `source: 'canonical-ass'`. This source marker lets the live primary-subtitle path prefer the clean authored text and timing for display, sidebar history, immersion recording, and mining, while unmatched editor notes and alternative translations remain ignored.
|
||||
|
||||
#### Prefetch Service Lifecycle
|
||||
|
||||
1. **Activation trigger:** When a subtitle track is activated (or changes), check if it's external via MPV's `track-list` property. If `external === true`, read the file via `external-filename` using the existing `loadSubtitleSourceText` infrastructure.
|
||||
|
||||
@@ -23,7 +23,9 @@ Trend charts now consume one chart-oriented backend payload from `/api/stats/tre
|
||||
- lookup rate trends
|
||||
- watch-time by day-of-week/hour
|
||||
- vocabulary-backed:
|
||||
- new-words trend
|
||||
- new-words trend reads permanent daily lexical rollups
|
||||
- rollup rows count only vocabulary-visible tokens and normalize mixed legacy timestamp units
|
||||
- a persisted rollup version invalidates stale materializations and triggers an atomic background rebuild
|
||||
|
||||
## Metric Semantics
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# Subtitle Overlay Priming
|
||||
|
||||
Status: active
|
||||
Last verified: 2026-08-04
|
||||
Last verified: 2026-08-18
|
||||
Owner: Kyle Yasuda
|
||||
Read when: debugging subtitle state or blank Linux/X11 overlay windows when the visible overlay is shown or recreated
|
||||
|
||||
@@ -69,6 +69,11 @@ coming and prefetching would otherwise idle for the rest of the cue.
|
||||
|
||||
## Live Cue Delivery
|
||||
|
||||
- Primary live text first resolves recovered canonical ASS animations. Otherwise, when
|
||||
every live mpv line matches an active parsed cue, it uses the parsed cue text so exact
|
||||
full-span style layers appear once instead of repeating for fill, border, blur, and
|
||||
shadow events. Any unmatched live line keeps the complete live stack, preserving
|
||||
dialogue or signs that overlap a lyric.
|
||||
- A tokenization cache miss emits the plain cue synchronously. Tokenization remains serialized so
|
||||
live work does not contend for Yomitan state.
|
||||
- If a newer cue arrives while an older line is still tokenizing, the newer plain cue or empty
|
||||
@@ -77,6 +82,25 @@ coming and prefetching would otherwise idle for the rest of the cue.
|
||||
- The current cue upgrades in place when its tokens and annotations are ready. This can reflow text
|
||||
or character images, but cue visibility does not wait for that work.
|
||||
|
||||
## Secondary Subtitle Flow
|
||||
|
||||
- `secondary-sub-text` remains the immediate fallback, so unreadable and remote subtitle sources
|
||||
still appear without waiting for file resolution.
|
||||
- `secondary-subtitle-track.ts` resolves `secondary-sid` against mpv's track list. External tracks
|
||||
are read directly; supported embedded text tracks are extracted through the same ffmpeg-backed
|
||||
source resolver used by primary subtitle prefetching.
|
||||
- The selected source is parsed with `parseSubtitleCues()`, including metadata-aware ASS duplicate
|
||||
and animation collapse. Playback `time-pos` selects the active parsed cue after applying
|
||||
`secondary-sub-delay`.
|
||||
- The resolved text is stored in `mpvClient.currentSecondarySubText` before it is broadcast. The
|
||||
overlay, mining, timing tracker, and immersion statistics therefore consume the same secondary
|
||||
text when a readable source is available.
|
||||
- Media and `secondary-sid` changes clear the previous parsed state before refreshing the source;
|
||||
track-list changes refresh without discarding an unchanged source. Observed
|
||||
`secondary-sub-delay` changes retime the active parsed cue without rereading the file. If loading,
|
||||
extraction, or parsing fails, the controller returns to live mpv text and the renderer's
|
||||
conservative short stack heuristic remains the final display fallback.
|
||||
|
||||
## Emitted State
|
||||
|
||||
- `emitSubtitle(payload)` maps to `emitSubtitlePayload(payload)`. Overlay windows and annotation
|
||||
@@ -84,8 +108,8 @@ coming and prefetching would otherwise idle for the rest of the cue.
|
||||
- The basic subtitle websocket receives the immediate plain cue only. Because its serialized
|
||||
payload discards annotations, the later upgrade would be an identical duplicate and is skipped
|
||||
when text and cue timing match.
|
||||
- Secondary priming reads mpv `secondary-sub-text`, stores it in
|
||||
`mpvClient.currentSecondarySubText`, and broadcasts `secondary-subtitle:set` to overlay windows.
|
||||
- Secondary priming reads mpv `secondary-sub-text` and routes it through the secondary track
|
||||
controller. A parsed active cue replaces the live text when the selected source is readable.
|
||||
- If secondary `requestProperty` fails, the primary flow stays complete and only a debug line is
|
||||
written.
|
||||
|
||||
|
||||
@@ -18,6 +18,9 @@ export function createImmersionDbFixture(dbPath: string): void {
|
||||
db.prepare(
|
||||
`INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('last_rollup_sample_ms', 0)`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('lexical_daily_rollups_version', 0)`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`INSERT INTO imm_lifetime_global(global_id, CREATED_DATE, LAST_UPDATE_DATE) VALUES (1, ?, ?)`,
|
||||
).run(String(Date.now()), String(Date.now()));
|
||||
|
||||
@@ -108,6 +108,36 @@ test('fixture schema stays aligned with production sync-touched tables and index
|
||||
}
|
||||
});
|
||||
|
||||
test('fixture leaves lexical rollups pending when their table is absent', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-fixture-rollup-state-'));
|
||||
const fixturePath = path.join(dir, 'fixture.sqlite');
|
||||
try {
|
||||
createImmersionDbFixture(fixturePath);
|
||||
const db = new BunDatabase(fixturePath, { readonly: true });
|
||||
try {
|
||||
const state = db
|
||||
.query<{ state_value: string }>(
|
||||
`SELECT state_value FROM imm_rollup_state
|
||||
WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||
)
|
||||
.get();
|
||||
const rollupTable = db
|
||||
.query<{ name: string }>(
|
||||
`SELECT name FROM sqlite_schema
|
||||
WHERE type = 'table' AND name = 'imm_lexical_daily_rollups'`,
|
||||
)
|
||||
.get();
|
||||
|
||||
assert.equal(state?.state_value, '0');
|
||||
assert.equal(rollupTable, null);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('fixture session inserts enforce foreign keys', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-fixture-foreign-keys-'));
|
||||
const fixturePath = path.join(dir, 'fixture.sqlite');
|
||||
|
||||
@@ -154,6 +154,7 @@ export const IMMERSION_DB_FIXTURE_DDL = `
|
||||
last_seen REAL,
|
||||
frequency INTEGER,
|
||||
frequency_rank INTEGER,
|
||||
vocabulary_visible INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1)),
|
||||
UNIQUE(headword, word, reading)
|
||||
);
|
||||
CREATE TABLE imm_kanji(
|
||||
|
||||
+1
-1
@@ -125,7 +125,7 @@
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.10.0",
|
||||
"@types/ws": "^8.18.1",
|
||||
"electron": "42.6.0",
|
||||
"electron": "43.4.1",
|
||||
"electron-builder": "26.15.3",
|
||||
"undici": "7.29.0",
|
||||
"esbuild": "^0.25.12",
|
||||
|
||||
@@ -7,6 +7,8 @@ local OVERLAY_RESTART_PING_MAX_ATTEMPTS = 20
|
||||
local OVERLAY_LOADING_OSD_PREFIX = "Overlay loading "
|
||||
local OVERLAY_LOADING_OSD_FRAMES = { "|", "/", "-", "\\" }
|
||||
local OVERLAY_LOADING_OSD_REFRESH_SECONDS = 0.18
|
||||
local OVERLAY_LOADING_OSD_DEADLINE_SECONDS = 30
|
||||
local OVERLAY_LOADING_OSD_TIMEOUT_MESSAGE = "Overlay did not become ready; check SubMiner logs"
|
||||
local AUTO_PLAY_READY_LOADING_OSD = "Loading subtitle tokenization..."
|
||||
local AUTO_PLAY_READY_READY_OSD = "Subtitle tokenization ready"
|
||||
local DEFAULT_AUTO_PLAY_READY_TIMEOUT_SECONDS = 30
|
||||
@@ -265,10 +267,19 @@ function M.create(ctx)
|
||||
state.overlay_loading_osd_timer = nil
|
||||
end
|
||||
|
||||
local function clear_overlay_loading_osd_deadline()
|
||||
local timeout = state.overlay_loading_osd_deadline
|
||||
if timeout and timeout.kill then
|
||||
timeout:kill()
|
||||
end
|
||||
state.overlay_loading_osd_deadline = nil
|
||||
end
|
||||
|
||||
local function stop_overlay_loading_osd()
|
||||
state.overlay_loading_osd_active = false
|
||||
state.overlay_loading_osd_frame = 1
|
||||
clear_overlay_loading_osd_timer()
|
||||
clear_overlay_loading_osd_deadline()
|
||||
end
|
||||
|
||||
local function start_overlay_loading_osd()
|
||||
@@ -291,6 +302,21 @@ function M.create(ctx)
|
||||
end
|
||||
end)
|
||||
end
|
||||
if type(mp.add_timeout) == "function" then
|
||||
state.overlay_loading_osd_deadline = mp.add_timeout(OVERLAY_LOADING_OSD_DEADLINE_SECONDS, function()
|
||||
if not state.overlay_loading_osd_active then
|
||||
return
|
||||
end
|
||||
state.overlay_loading_osd_deadline = nil
|
||||
stop_overlay_loading_osd()
|
||||
subminer_log(
|
||||
"warn",
|
||||
"process",
|
||||
"Overlay loading deadline expired before the app reported content ready"
|
||||
)
|
||||
show_osd(OVERLAY_LOADING_OSD_TIMEOUT_MESSAGE, { force = true })
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
||||
local function disarm_auto_play_ready_gate(options)
|
||||
|
||||
@@ -26,6 +26,7 @@ function M.new()
|
||||
auto_play_ready_initial_pause_ownership_consumed = false,
|
||||
overlay_loading_osd_active = false,
|
||||
overlay_loading_osd_timer = nil,
|
||||
overlay_loading_osd_deadline = nil,
|
||||
overlay_loading_osd_frame = 1,
|
||||
pending_visible_overlay_hide_timer = nil,
|
||||
pending_visible_overlay_hide_generation = 0,
|
||||
|
||||
@@ -130,7 +130,9 @@ local function run_plugin_scenario(config)
|
||||
|
||||
function mp.add_timeout(seconds, callback)
|
||||
recorded.timeouts[#recorded.timeouts + 1] = seconds
|
||||
local delay = tonumber(seconds) or 0
|
||||
local timeout = {
|
||||
seconds = delay,
|
||||
killed = false,
|
||||
callback = callback,
|
||||
}
|
||||
@@ -138,7 +140,6 @@ local function run_plugin_scenario(config)
|
||||
self.killed = true
|
||||
end
|
||||
|
||||
local delay = tonumber(seconds) or 0
|
||||
if callback and delay < 5 and not config.defer_timeouts then
|
||||
callback()
|
||||
end
|
||||
@@ -514,6 +515,15 @@ local function has_timeout(timeouts, target)
|
||||
return false
|
||||
end
|
||||
|
||||
local function find_timeout_handle(recorded, target)
|
||||
for _, timeout in ipairs(recorded.timeout_handles) do
|
||||
if math.abs(timeout.seconds - target) < 0.0001 then
|
||||
return timeout
|
||||
end
|
||||
end
|
||||
return nil
|
||||
end
|
||||
|
||||
local function env_has(call, target)
|
||||
local env = (call and call.env) or {}
|
||||
for _, value in ipairs(env) do
|
||||
@@ -1636,6 +1646,8 @@ do
|
||||
#recorded.periodic_timers == 1,
|
||||
"auto-start visible overlay should refresh the early overlay loading OSD"
|
||||
)
|
||||
local overlay_loading_deadline = find_timeout_handle(recorded, 30)
|
||||
assert_true(overlay_loading_deadline ~= nil, "overlay loading OSD should have a bounded deadline")
|
||||
local overlay_loading_timer = recorded.periodic_timers[1]
|
||||
recorded.periodic_timers[1].callback()
|
||||
assert_true(
|
||||
@@ -1670,6 +1682,46 @@ do
|
||||
recorded.periodic_timers[1].killed == true,
|
||||
"overlay loading ready should stop the early overlay loading OSD refresher"
|
||||
)
|
||||
assert_true(
|
||||
overlay_loading_deadline.killed == true,
|
||||
"overlay loading ready should cancel the bounded loading deadline"
|
||||
)
|
||||
end
|
||||
|
||||
do
|
||||
local recorded, err = run_plugin_scenario({
|
||||
defer_timeouts = true,
|
||||
process_list = "",
|
||||
option_overrides = {
|
||||
binary_path = binary_path,
|
||||
auto_start = "yes",
|
||||
auto_start_visible_overlay = "yes",
|
||||
osd_messages = false,
|
||||
socket_path = "/tmp/subminer-socket",
|
||||
},
|
||||
input_ipc_server = "/tmp/subminer-socket",
|
||||
media_title = "Random Movie",
|
||||
files = {
|
||||
[binary_path] = true,
|
||||
},
|
||||
})
|
||||
assert_true(recorded ~= nil, "plugin failed to load for overlay loading deadline scenario: " .. tostring(err))
|
||||
fire_event(recorded, "start-file")
|
||||
local overlay_loading_deadline = find_timeout_handle(recorded, 30)
|
||||
assert_true(overlay_loading_deadline ~= nil, "overlay loading deadline should be scheduled")
|
||||
overlay_loading_deadline.callback()
|
||||
assert_true(
|
||||
recorded.periodic_timers[1].killed == true,
|
||||
"overlay loading deadline should stop the loading spinner"
|
||||
)
|
||||
assert_true(
|
||||
has_osd_message(recorded.osd, "SubMiner: Overlay did not become ready; check SubMiner logs"),
|
||||
"overlay loading deadline should replace the spinner with actionable feedback"
|
||||
)
|
||||
assert_true(
|
||||
has_log_containing(recorded.logs, "Overlay loading deadline expired"),
|
||||
"overlay loading deadline should leave a diagnostic log entry"
|
||||
)
|
||||
end
|
||||
|
||||
do
|
||||
|
||||
@@ -284,6 +284,22 @@ function createMockTracker(
|
||||
getSessionTimeline: async () => [],
|
||||
getSessionEvents: async () => [],
|
||||
getVocabularyStats: async () => VOCABULARY_STATS,
|
||||
getVocabularySummary: async () => ({
|
||||
uniqueWords: 501,
|
||||
uniqueWordsWithoutNames: 500,
|
||||
uniqueKanji: 201,
|
||||
newThisWeek: 7,
|
||||
newThisWeekWithoutNames: 6,
|
||||
knownWordCount: 250,
|
||||
knownWordCountWithoutNames: 249,
|
||||
}),
|
||||
getVocabularyChartData: async () => ({
|
||||
ready: true,
|
||||
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
}),
|
||||
getStatsExcludedWords: async () => [],
|
||||
replaceStatsExcludedWords: async () => {},
|
||||
getKanjiStats: async () => KANJI_STATS,
|
||||
@@ -711,6 +727,38 @@ describe('stats server API routes', () => {
|
||||
assert.equal(body[0].headword, 'する');
|
||||
});
|
||||
|
||||
it('GET /api/stats/vocabulary/summary returns database-wide card totals', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
|
||||
const res = await app.request('/api/stats/vocabulary/summary');
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(await res.json(), {
|
||||
uniqueWords: 501,
|
||||
uniqueWordsWithoutNames: 500,
|
||||
uniqueKanji: 201,
|
||||
newThisWeek: 7,
|
||||
newThisWeekWithoutNames: 6,
|
||||
knownWordCount: 250,
|
||||
knownWordCountWithoutNames: 249,
|
||||
});
|
||||
});
|
||||
|
||||
it('GET /api/stats/vocabulary/charts returns complete chart datasets', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
|
||||
const res = await app.request('/api/stats/vocabulary/charts');
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(await res.json(), {
|
||||
ready: true,
|
||||
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
});
|
||||
});
|
||||
|
||||
it('GET /api/stats/kanji returns kanji frequency data', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
const res = await app.request('/api/stats/kanji');
|
||||
|
||||
@@ -47,7 +47,7 @@ test('stripFilenameTags normalizes common media-title formats', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('fetchIfMissing backfills a missing blob from an existing cover URL', async () => {
|
||||
async function backfillMissingCoverBlob(): Promise<void> {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
ensureSchema(db);
|
||||
@@ -103,9 +103,15 @@ test('fetchIfMissing backfills a missing blob from an existing cover URL', async
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('fetchIfMissing reuses cached cover art from another video in the same anime', async () => {
|
||||
test(
|
||||
'fetchIfMissing backfills a missing blob from an existing cover URL',
|
||||
{ timeout: 15_000 },
|
||||
backfillMissingCoverBlob,
|
||||
);
|
||||
|
||||
async function reuseCachedAnimeCoverArt(): Promise<void> {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
ensureSchema(db);
|
||||
@@ -179,7 +185,13 @@ test('fetchIfMissing reuses cached cover art from another video in the same anim
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test(
|
||||
'fetchIfMissing reuses cached cover art from another video in the same anime',
|
||||
{ timeout: 15_000 },
|
||||
reuseCachedAnimeCoverArt,
|
||||
);
|
||||
|
||||
function createJsonResponse(payload: unknown): Response {
|
||||
return new Response(JSON.stringify(payload), {
|
||||
|
||||
@@ -559,6 +559,241 @@ test('fresh tracker DB creates lifetime summary tables', async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('fresh tracker DB skips lexical rollup backfill work', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let backfillRuns = 0;
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor({ dbPath }, {
|
||||
runLexicalRollupBackfillTask: async () => {
|
||||
backfillRuns += 1;
|
||||
},
|
||||
} as never);
|
||||
|
||||
assert.equal(backfillRuns, 0);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('tracker starts the injected lexical rollup backfill when it is pending', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let backfillRuns = 0;
|
||||
|
||||
try {
|
||||
const setupDb = new Database(dbPath);
|
||||
const { ensureSchema } = await import('./immersion-tracker/storage');
|
||||
ensureSchema(setupDb);
|
||||
setupDb
|
||||
.prepare(
|
||||
`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||
)
|
||||
.run();
|
||||
setupDb.close();
|
||||
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor({ dbPath }, {
|
||||
runLexicalRollupBackfillTask: async () => {
|
||||
backfillRuns += 1;
|
||||
},
|
||||
} as never);
|
||||
|
||||
assert.equal(backfillRuns, 1);
|
||||
await waitForCondition(
|
||||
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
|
||||
);
|
||||
assert.equal(
|
||||
(tracker as unknown as { preserveWriteQueueUntilDrained: boolean })
|
||||
.preserveWriteQueueUntilDrained,
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('tracker runs startup session-rollup maintenance before lexical backfill locks writes', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let releaseBackfill = (): void => {};
|
||||
const heldBackfill = new Promise<void>((resolve) => {
|
||||
releaseBackfill = resolve;
|
||||
});
|
||||
|
||||
try {
|
||||
const startedAtMs = trackerNowMs() - 60_000;
|
||||
const endedAtMs = trackerNowMs();
|
||||
const setupDb = new Database(dbPath);
|
||||
const { ensureSchema } = await import('./immersion-tracker/storage');
|
||||
ensureSchema(setupDb);
|
||||
setupDb.exec(`
|
||||
INSERT INTO imm_videos (
|
||||
video_id, video_key, canonical_title, source_type, duration_ms, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (1, 'local:/tmp/rollup-recovery.mkv', 'Rollup Recovery', 1, 0, '1', '1');
|
||||
INSERT INTO imm_sessions (
|
||||
session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status,
|
||||
active_watched_ms, lines_seen, tokens_seen, cards_mined, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (
|
||||
1, 'rollup-recovery', 1, '${startedAtMs}', '${endedAtMs}', 2,
|
||||
60000, 10, 20, 2, '${startedAtMs}', '${endedAtMs}'
|
||||
);
|
||||
INSERT INTO imm_session_telemetry (
|
||||
session_id, sample_ms, total_watched_ms, active_watched_ms, lines_seen,
|
||||
tokens_seen, cards_mined, lookup_count, lookup_hits, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (
|
||||
1, '${endedAtMs}', 60000, 60000, 10, 20, 2, 0, 0,
|
||||
'${endedAtMs}', '${endedAtMs}'
|
||||
);
|
||||
DELETE FROM imm_daily_rollups;
|
||||
DELETE FROM imm_monthly_rollups;
|
||||
UPDATE imm_rollup_state SET state_value = '0';
|
||||
`);
|
||||
setupDb.close();
|
||||
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor({ dbPath }, {
|
||||
runLexicalRollupBackfillTask: async () => heldBackfill,
|
||||
} as never);
|
||||
|
||||
const privateApi = tracker as unknown as {
|
||||
db: DatabaseSync;
|
||||
writeLock: { locked: boolean };
|
||||
};
|
||||
assert.equal(privateApi.writeLock.locked, true);
|
||||
assert.equal(
|
||||
(
|
||||
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_daily_rollups').get() as {
|
||||
total: number;
|
||||
}
|
||||
).total,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_monthly_rollups').get() as {
|
||||
total: number;
|
||||
}
|
||||
).total,
|
||||
1,
|
||||
);
|
||||
} finally {
|
||||
releaseBackfill();
|
||||
if (tracker) {
|
||||
await waitForCondition(
|
||||
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
|
||||
);
|
||||
}
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('tracker queues playback writes until lexical rollup backfill settles', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let startBackfill = (): void => {};
|
||||
let releaseBackfill = (): void => {};
|
||||
let markBackfillStarted = (): void => {};
|
||||
const backfillStartGate = new Promise<void>((resolve) => {
|
||||
startBackfill = resolve;
|
||||
});
|
||||
const heldBackfill = new Promise<void>((resolve) => {
|
||||
releaseBackfill = resolve;
|
||||
});
|
||||
const backfillStarted = new Promise<void>((resolve) => {
|
||||
markBackfillStarted = resolve;
|
||||
});
|
||||
|
||||
try {
|
||||
const setupDb = new Database(dbPath);
|
||||
const { ensureSchema } = await import('./immersion-tracker/storage');
|
||||
ensureSchema(setupDb);
|
||||
setupDb
|
||||
.prepare(
|
||||
`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||
)
|
||||
.run();
|
||||
setupDb.close();
|
||||
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath, policy: { queueCap: 100 } },
|
||||
{
|
||||
runLexicalRollupBackfillTask: async (workerDbPath) => {
|
||||
await backfillStartGate;
|
||||
const workerDb = new Database(workerDbPath);
|
||||
try {
|
||||
workerDb.exec('BEGIN IMMEDIATE');
|
||||
markBackfillStarted();
|
||||
await heldBackfill;
|
||||
workerDb.exec('COMMIT');
|
||||
} catch (error) {
|
||||
try {
|
||||
workerDb.exec('ROLLBACK');
|
||||
} catch {
|
||||
// Preserve the original worker failure.
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
workerDb.close();
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
tracker.handleMediaChange('https://example.com/backfill-test.mp4', 'Backfill Test');
|
||||
startBackfill();
|
||||
await backfillStarted;
|
||||
for (let index = 0; index < 125; index += 1) tracker.recordCardsMined(1);
|
||||
|
||||
const privateApi = tracker as unknown as {
|
||||
db: DatabaseSync;
|
||||
queue: unknown[];
|
||||
droppedWriteCount: number;
|
||||
flushNow: () => void;
|
||||
writeLock: { locked: boolean };
|
||||
};
|
||||
assert.equal(privateApi.writeLock.locked, true);
|
||||
privateApi.flushNow();
|
||||
|
||||
assert.ok(privateApi.queue.length > 100, 'the protected queue may grow past its normal cap');
|
||||
assert.equal(privateApi.droppedWriteCount, 0, 'backfill must not discard playback writes');
|
||||
assert.equal(
|
||||
(
|
||||
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
|
||||
total: number;
|
||||
}
|
||||
).total,
|
||||
0,
|
||||
);
|
||||
|
||||
releaseBackfill();
|
||||
await waitForCondition(() => privateApi.queue.length === 0, 5_000);
|
||||
assert.equal(
|
||||
(
|
||||
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
|
||||
total: number;
|
||||
}
|
||||
).total,
|
||||
125,
|
||||
);
|
||||
} finally {
|
||||
releaseBackfill();
|
||||
if (tracker) {
|
||||
await waitForCondition(
|
||||
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
|
||||
5_000,
|
||||
);
|
||||
}
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('startup backfills lifetime summaries when retained sessions exist but summary tables are empty', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
@@ -4909,3 +5144,149 @@ test('ensureAnimeCoverArt fetches art via the latest video of the anime', async
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary coalesces concurrent requests into one worker task', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let taskRuns = 0;
|
||||
let releaseTask: (() => void) | null = null;
|
||||
const seenKnownWords: Array<ReadonlySet<string> | null> = [];
|
||||
const summary = {
|
||||
uniqueWords: 1,
|
||||
uniqueWordsWithoutNames: 1,
|
||||
uniqueKanji: 0,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: null,
|
||||
knownWordCountWithoutNames: null,
|
||||
};
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runVocabularySummaryTask: async (_dbPath, knownWords) => {
|
||||
taskRuns += 1;
|
||||
seenKnownWords.push(knownWords);
|
||||
await new Promise<void>((resolve) => {
|
||||
releaseTask = resolve;
|
||||
});
|
||||
return summary;
|
||||
},
|
||||
destroyVocabularySummaryRunner: () => {},
|
||||
},
|
||||
);
|
||||
|
||||
const knownWordsSnapshot = new Set(['猫']);
|
||||
const first = tracker.getVocabularySummary(knownWordsSnapshot);
|
||||
const second = tracker.getVocabularySummary(knownWordsSnapshot);
|
||||
await waitForCondition(() => releaseTask !== null);
|
||||
let release = releaseTask as (() => void) | null;
|
||||
assert.ok(release);
|
||||
release();
|
||||
assert.deepEqual(await first, summary);
|
||||
assert.equal(await second, await first);
|
||||
assert.equal(taskRuns, 1);
|
||||
assert.deepEqual(seenKnownWords, [knownWordsSnapshot]);
|
||||
|
||||
releaseTask = null;
|
||||
const third = tracker.getVocabularySummary(null);
|
||||
await waitForCondition(() => releaseTask !== null);
|
||||
release = releaseTask as (() => void) | null;
|
||||
assert.ok(release);
|
||||
release();
|
||||
assert.deepEqual(await third, summary);
|
||||
assert.equal(taskRuns, 2);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary coalesces equivalent known-word snapshots by value', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let taskRuns = 0;
|
||||
const releases: Array<() => void> = [];
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runVocabularySummaryTask: async () => {
|
||||
taskRuns += 1;
|
||||
await new Promise<void>((resolve) => releases.push(resolve));
|
||||
return {
|
||||
uniqueWords: 2,
|
||||
uniqueWordsWithoutNames: 2,
|
||||
uniqueKanji: 2,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: 2,
|
||||
knownWordCountWithoutNames: 2,
|
||||
};
|
||||
},
|
||||
destroyVocabularySummaryRunner: () => {},
|
||||
},
|
||||
);
|
||||
|
||||
const first = tracker.getVocabularySummary(new Set(['猫', '犬']));
|
||||
const second = tracker.getVocabularySummary(new Set(['犬', '猫']));
|
||||
await waitForCondition(() => releases.length > 0);
|
||||
const observedTaskRuns = taskRuns;
|
||||
for (const release of releases) release();
|
||||
await Promise.all([first, second]);
|
||||
|
||||
assert.equal(observedTaskRuns, 1);
|
||||
|
||||
const third = tracker.getVocabularySummary(new Set(['猫', '犬']));
|
||||
await waitForCondition(() => releases.length === 2);
|
||||
releases[1]!();
|
||||
await third;
|
||||
assert.equal(taskRuns, 2, 'a settled snapshot must be evicted from the in-flight map');
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary keeps different known-word snapshots independent', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
const releases: Array<() => void> = [];
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runVocabularySummaryTask: async (_dbPath, knownWords) => {
|
||||
await new Promise<void>((resolve) => releases.push(resolve));
|
||||
return {
|
||||
uniqueWords: 1,
|
||||
uniqueWordsWithoutNames: 1,
|
||||
uniqueKanji: 0,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: knownWords?.size ?? null,
|
||||
knownWordCountWithoutNames: knownWords?.size ?? null,
|
||||
};
|
||||
},
|
||||
destroyVocabularySummaryRunner: () => {},
|
||||
},
|
||||
);
|
||||
|
||||
const withoutKnownWords = tracker.getVocabularySummary(null);
|
||||
const withKnownWords = tracker.getVocabularySummary(new Set(['猫']));
|
||||
await waitForCondition(() => releases.length === 2);
|
||||
for (const release of releases) release();
|
||||
|
||||
assert.equal((await withoutKnownWords).knownWordCount, null);
|
||||
assert.equal((await withKnownWords).knownWordCount, 1);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -58,6 +58,7 @@ import {
|
||||
getSessionEvents,
|
||||
getSimilarWords,
|
||||
getStatsExcludedWords,
|
||||
getVocabularyChartData,
|
||||
getVocabularyStats,
|
||||
replaceStatsExcludedWords,
|
||||
searchSubtitleSentences,
|
||||
@@ -96,6 +97,12 @@ import {
|
||||
DeleteMaintenanceWorkerRuntime,
|
||||
type RunDeleteMaintenanceTask,
|
||||
} from './immersion-tracker/delete-maintenance-worker-runtime';
|
||||
import {
|
||||
VocabularySummaryWorkerRuntime,
|
||||
type RunVocabularySummaryTask,
|
||||
} from './immersion-tracker/vocabulary-summary-worker-runtime';
|
||||
import { LexicalRollupWorkerRuntime } from './immersion-tracker/lexical-rollup-worker-runtime';
|
||||
import { areLexicalDailyRollupsReady } from './immersion-tracker/lexical-rollups';
|
||||
import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
|
||||
import {
|
||||
cleanupDuplicateSubtitleLines,
|
||||
@@ -185,6 +192,7 @@ import {
|
||||
type StatsExcludedWordRow,
|
||||
type StreakCalendarRow,
|
||||
type VocabularyCleanupSummary,
|
||||
type VocabularyStatsSummary,
|
||||
type WatchTimePerAnimeRow,
|
||||
type WordAnimeAppearanceRow,
|
||||
type WordDetailRow,
|
||||
@@ -405,13 +413,24 @@ export class ImmersionTrackerService {
|
||||
private readonly monthlyRollupRetentionMs: number;
|
||||
private readonly vacuumIntervalMs: number;
|
||||
private readonly dbPath: string;
|
||||
private readonly writeLock = { locked: false };
|
||||
private readonly writeLock = {
|
||||
locked: false,
|
||||
reasons: new Set<'flush' | 'delete-maintenance' | 'lexical-rollup-backfill'>(),
|
||||
};
|
||||
private readonly destroyDeleteMaintenanceRunner: () => void;
|
||||
private readonly runVocabularySummaryTask: (
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
) => Promise<VocabularyStatsSummary>;
|
||||
private readonly vocabularySummariesInFlight = new Map<string, Promise<VocabularyStatsSummary>>();
|
||||
private readonly destroyVocabularySummaryRunner: () => void;
|
||||
private readonly runLexicalRollupBackfillTask: () => Promise<void>;
|
||||
private readonly destroyLexicalRollupBackfillRunner: () => void;
|
||||
private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
|
||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private flushScheduled = false;
|
||||
private droppedWriteCount = 0;
|
||||
private preserveWriteQueueUntilDrained = false;
|
||||
private lastVacuumMs = 0;
|
||||
private isDestroyed = false;
|
||||
private sessionState: SessionState | null = null;
|
||||
@@ -434,6 +453,10 @@ export class ImmersionTrackerService {
|
||||
dependencies: {
|
||||
runDeleteMaintenanceTask?: RunDeleteMaintenanceTask;
|
||||
destroyDeleteMaintenanceRunner?: () => void;
|
||||
runVocabularySummaryTask?: RunVocabularySummaryTask;
|
||||
destroyVocabularySummaryRunner?: () => void;
|
||||
runLexicalRollupBackfillTask?: (dbPath: string) => Promise<void>;
|
||||
destroyLexicalRollupBackfillRunner?: () => void;
|
||||
} = {},
|
||||
) {
|
||||
this.dbPath = options.dbPath;
|
||||
@@ -453,13 +476,34 @@ export class ImmersionTrackerService {
|
||||
runTask: (task) => runDeleteMaintenanceTask(this.dbPath, task),
|
||||
onBusy: () => {
|
||||
this.requireWriteQueueDrained('delete maintenance');
|
||||
this.writeLock.locked = true;
|
||||
this.setWriteLock('delete-maintenance', true);
|
||||
},
|
||||
onIdle: () => {
|
||||
this.writeLock.locked = false;
|
||||
this.setWriteLock('delete-maintenance', false);
|
||||
if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
|
||||
},
|
||||
});
|
||||
if (dependencies.runVocabularySummaryTask) {
|
||||
this.runVocabularySummaryTask = (knownWords) =>
|
||||
dependencies.runVocabularySummaryTask!(this.dbPath, knownWords);
|
||||
this.destroyVocabularySummaryRunner =
|
||||
dependencies.destroyVocabularySummaryRunner ?? (() => {});
|
||||
} else {
|
||||
const vocabularySummaryRuntime = new VocabularySummaryWorkerRuntime();
|
||||
this.runVocabularySummaryTask = (knownWords) =>
|
||||
vocabularySummaryRuntime.run(this.dbPath, knownWords);
|
||||
this.destroyVocabularySummaryRunner = () => vocabularySummaryRuntime.destroy();
|
||||
}
|
||||
if (dependencies.runLexicalRollupBackfillTask) {
|
||||
this.runLexicalRollupBackfillTask = () =>
|
||||
dependencies.runLexicalRollupBackfillTask!(this.dbPath);
|
||||
this.destroyLexicalRollupBackfillRunner =
|
||||
dependencies.destroyLexicalRollupBackfillRunner ?? (() => {});
|
||||
} else {
|
||||
const lexicalRollupRuntime = new LexicalRollupWorkerRuntime();
|
||||
this.runLexicalRollupBackfillTask = () => lexicalRollupRuntime.run(this.dbPath);
|
||||
this.destroyLexicalRollupBackfillRunner = () => lexicalRollupRuntime.destroy();
|
||||
}
|
||||
const parentDir = path.dirname(this.dbPath);
|
||||
if (!fs.existsSync(parentDir)) {
|
||||
fs.mkdirSync(parentDir, { recursive: true });
|
||||
@@ -548,6 +592,7 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
this.preparedStatements = createTrackerPreparedStatements(this.db);
|
||||
this.scheduleMaintenance();
|
||||
if (!areLexicalDailyRollupsReady(this.db)) this.startLexicalRollupBackfill();
|
||||
this.scheduleFlush();
|
||||
}
|
||||
|
||||
@@ -565,6 +610,8 @@ export class ImmersionTrackerService {
|
||||
this.isDestroyed = true;
|
||||
this.deleteMaintenanceScheduler.destroy();
|
||||
this.destroyDeleteMaintenanceRunner();
|
||||
this.destroyVocabularySummaryRunner();
|
||||
this.destroyLexicalRollupBackfillRunner();
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
@@ -634,6 +681,25 @@ export class ImmersionTrackerService {
|
||||
return getVocabularyStats(this.db, limit, excludePos);
|
||||
}
|
||||
|
||||
async getVocabularySummary(knownWords: ReadonlySet<string> | null) {
|
||||
const key = knownWords ? JSON.stringify([...knownWords].sort()) : 'null';
|
||||
const inFlight = this.vocabularySummariesInFlight.get(key);
|
||||
if (inFlight) return inFlight;
|
||||
const task = this.runVocabularySummaryTask(knownWords);
|
||||
this.vocabularySummariesInFlight.set(key, task);
|
||||
try {
|
||||
return await task;
|
||||
} finally {
|
||||
if (this.vocabularySummariesInFlight.get(key) === task) {
|
||||
this.vocabularySummariesInFlight.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async getVocabularyChartData() {
|
||||
return getVocabularyChartData(this.db);
|
||||
}
|
||||
|
||||
async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {
|
||||
return getStatsExcludedWords(this.db);
|
||||
}
|
||||
@@ -910,6 +976,33 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
}
|
||||
|
||||
private setWriteLock(
|
||||
reason: 'flush' | 'delete-maintenance' | 'lexical-rollup-backfill',
|
||||
active: boolean,
|
||||
): void {
|
||||
if (active) this.writeLock.reasons.add(reason);
|
||||
else this.writeLock.reasons.delete(reason);
|
||||
this.writeLock.locked = this.writeLock.reasons.size > 0;
|
||||
}
|
||||
|
||||
private startLexicalRollupBackfill(): void {
|
||||
this.requireWriteQueueDrained('lexical rollup backfill');
|
||||
this.preserveWriteQueueUntilDrained = true;
|
||||
this.setWriteLock('lexical-rollup-backfill', true);
|
||||
void this.runLexicalRollupBackfillTask()
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
'Lexical daily rollup backfill failed; it will retry on next startup',
|
||||
error,
|
||||
);
|
||||
})
|
||||
.finally(() => {
|
||||
this.setWriteLock('lexical-rollup-backfill', false);
|
||||
if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
|
||||
else if (!this.isDestroyed) this.scheduleFlush(0);
|
||||
});
|
||||
}
|
||||
|
||||
async reassignAnimeAnilist(
|
||||
animeId: number,
|
||||
info: {
|
||||
@@ -1906,7 +1999,12 @@ export class ImmersionTrackerService {
|
||||
|
||||
private recordWrite(write: QueuedWrite): void {
|
||||
if (this.isDestroyed) return;
|
||||
const { dropped } = enqueueWrite(this.queue, write, this.queueCap);
|
||||
// A lexical migration owns the database write lock, so dropping the oldest
|
||||
// entry cannot relieve pressure: nothing can flush until the worker exits.
|
||||
// Preserve that finite startup burst and drain it as soon as the lock lifts.
|
||||
const { dropped } = this.preserveWriteQueueUntilDrained
|
||||
? (this.queue.push(write), { dropped: 0 })
|
||||
: enqueueWrite(this.queue, write, this.queueCap);
|
||||
if (dropped > 0) {
|
||||
this.droppedWriteCount += dropped;
|
||||
this.logger.warn(`Immersion tracker queue overflow; dropped ${dropped} oldest writes`);
|
||||
@@ -1954,6 +2052,7 @@ export class ImmersionTrackerService {
|
||||
private flushNow(): void {
|
||||
if (this.writeLock.locked || this.isDestroyed) return;
|
||||
if (this.queue.length === 0) {
|
||||
this.preserveWriteQueueUntilDrained = false;
|
||||
this.flushScheduled = false;
|
||||
return;
|
||||
}
|
||||
@@ -1965,7 +2064,7 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
|
||||
const batch = this.queue.splice(0, Math.min(this.batchSize, this.queue.length));
|
||||
this.writeLock.locked = true;
|
||||
this.setWriteLock('flush', true);
|
||||
try {
|
||||
this.db.exec('BEGIN IMMEDIATE');
|
||||
for (const write of batch) {
|
||||
@@ -1977,8 +2076,9 @@ export class ImmersionTrackerService {
|
||||
this.queue.unshift(...batch);
|
||||
this.logger.warn('Immersion tracker flush failed, retrying later', error as Error);
|
||||
} finally {
|
||||
this.writeLock.locked = false;
|
||||
this.setWriteLock('flush', false);
|
||||
this.flushScheduled = false;
|
||||
if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
|
||||
if (this.queue.length > 0) {
|
||||
this.scheduleFlush(this.flushIntervalMs);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { Database } from '../sqlite.js';
|
||||
import type { DatabaseSync } from '../sqlite.js';
|
||||
@@ -21,17 +18,6 @@ interface SeedLine {
|
||||
createdMs?: number;
|
||||
}
|
||||
|
||||
function makeDbPath(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-duplicate-line-test-'));
|
||||
return path.join(dir, 'immersion.sqlite');
|
||||
}
|
||||
|
||||
function cleanupDbPath(dbPath: string): void {
|
||||
const dir = path.dirname(dbPath);
|
||||
if (!fs.existsSync(dir)) return;
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
/** One episode, two sessions of it, and one word occurrence per seeded line. */
|
||||
function seed(db: DatabaseSync, lines: SeedLine[]): void {
|
||||
db.exec(`
|
||||
@@ -82,12 +68,16 @@ function seed(db: DatabaseSync, lines: SeedLine[]): void {
|
||||
`);
|
||||
}
|
||||
|
||||
function createDb(lines: SeedLine[]): { db: DatabaseSync; dbPath: string } {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
/**
|
||||
* These tests exercise the cleanup SQL, not durability. A fresh on-disk database per
|
||||
* test pays a schema-creation fsync that is cheap on a local NVMe but slow enough on CI
|
||||
* runners to blow the 5s per-test timeout, so the database stays in memory.
|
||||
*/
|
||||
function createDb(lines: SeedLine[]): { db: DatabaseSync } {
|
||||
const db = new Database(':memory:');
|
||||
ensureSchema(db);
|
||||
seed(db, lines);
|
||||
return { db, dbPath };
|
||||
return { db };
|
||||
}
|
||||
|
||||
/** A typeset line mpv reported once per animation frame. */
|
||||
@@ -119,7 +109,7 @@ function wordFrequency(db: DatabaseSync): number {
|
||||
}
|
||||
|
||||
test('a karaoke burst collapses to one line and gives back its word counts', () => {
|
||||
const { db, dbPath } = createDb([
|
||||
const { db } = createDb([
|
||||
...karaokeFrames(1, '飛び上がる', 10_000, 40, 40),
|
||||
{ session: 1, text: 'おはよう', startMs: 20_000, endMs: 22_000 },
|
||||
]);
|
||||
@@ -148,7 +138,6 @@ test('a karaoke burst collapses to one line and gives back its word counts', ()
|
||||
assert.equal(summary.samples[0]!.videoTitle, 'Ep 1');
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -160,7 +149,7 @@ test('ordinary repeated dialogue survives', () => {
|
||||
startMs: 5_000 + index * 800,
|
||||
endMs: 5_000 + (index + 1) * 800,
|
||||
}));
|
||||
const { db, dbPath } = createDb(lines);
|
||||
const { db } = createDb(lines);
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
@@ -171,14 +160,13 @@ test('ordinary repeated dialogue survives', () => {
|
||||
assert.equal(wordFrequency(db), 6);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a long run of quarter-second frames is still a burst', () => {
|
||||
// Between the timing-only bound (0.1s) and the animation-frame bound (0.3s): heavier
|
||||
// typesetting lands here, and the run length is what makes it conclusive.
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
|
||||
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
@@ -189,12 +177,11 @@ test('a long run of quarter-second frames is still a burst', () => {
|
||||
assert.equal(wordFrequency(db), 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a qualifying short-frame burst may end with one long hold frame', () => {
|
||||
const { db, dbPath } = createDb([
|
||||
const { db } = createDb([
|
||||
...karaokeFrames(1, '飛び上がる', 10_000, 8, 40),
|
||||
{ session: 1, text: '飛び上がる', startMs: 10_320, endMs: 12_320 },
|
||||
]);
|
||||
@@ -208,12 +195,11 @@ test('a qualifying short-frame burst may end with one long hold frame', () => {
|
||||
assert.equal(wordFrequency(db), 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a long event before the final frame prevents burst cleanup', () => {
|
||||
const { db, dbPath } = createDb([
|
||||
const { db } = createDb([
|
||||
...karaokeFrames(1, '飛び上がる', 10_000, 5, 40),
|
||||
{ session: 1, text: '飛び上がる', startMs: 10_200, endMs: 12_200 },
|
||||
{ session: 1, text: '飛び上がる', startMs: 12_200, endMs: 12_240 },
|
||||
@@ -226,12 +212,11 @@ test('a long event before the final frame prevents burst cleanup', () => {
|
||||
assert.equal(countLines(db), 7);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a run of frames longer than the animation bound survives', () => {
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 400));
|
||||
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 400));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
@@ -240,7 +225,6 @@ test('a run of frames longer than the animation bound survives', () => {
|
||||
assert.equal(countLines(db), 6);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -248,7 +232,7 @@ test('the four-frame residue the live gate stores is cleaned up', () => {
|
||||
// The streaming gate records the first four frames of a burst before the run is long
|
||||
// enough to recognise. Four contiguous identical events under the strict timing-only
|
||||
// bound are that residue, and no real dialogue.
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 40));
|
||||
const { db } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 40));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
@@ -259,14 +243,13 @@ test('the four-frame residue the live gate stores is cleaned up', () => {
|
||||
assert.equal(wordFrequency(db), 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a four-frame run above the strict frame bound survives', () => {
|
||||
// Long enough per event to be plausible dialogue; only a five-event run may use the
|
||||
// looser animation-frame bound.
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 250));
|
||||
const { db } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 250));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
@@ -275,14 +258,13 @@ test('a four-frame run above the strict frame bound survives', () => {
|
||||
assert.equal(countLines(db), 4);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('an explicit minRunLength raises the bar', () => {
|
||||
// Five quarter-second frames qualify under the defaults; a cautious run asking for six
|
||||
// leaves them alone. Above the strict bound, so the residue rule stays out of it.
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 5, 250));
|
||||
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 5, 250));
|
||||
|
||||
try {
|
||||
const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
|
||||
@@ -293,12 +275,11 @@ test('an explicit minRunLength raises the bar', () => {
|
||||
assert.equal(countLines(db), 5);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('an explicit maxFrameSeconds tightens the frame bound', () => {
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
|
||||
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db, { maxFrameSeconds: 0.2 });
|
||||
@@ -307,13 +288,12 @@ test('an explicit maxFrameSeconds tightens the frame bound', () => {
|
||||
assert.equal(countLines(db), 6);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a non-finite maxFrameSeconds falls back to the default bound', () => {
|
||||
// Six normal-beat lines: Infinity must not turn every event into a "short frame".
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 800));
|
||||
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 800));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db, { maxFrameSeconds: Infinity });
|
||||
@@ -322,12 +302,11 @@ test('a non-finite maxFrameSeconds falls back to the default bound', () => {
|
||||
assert.equal(countLines(db), 6);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('sampleLimit zero removes bursts but reports no samples', () => {
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
|
||||
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db, { sampleLimit: 0 });
|
||||
@@ -337,12 +316,11 @@ test('sampleLimit zero removes bursts but reports no samples', () => {
|
||||
assert.equal(countLines(db), 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a short run below every threshold survives', () => {
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 3, 40));
|
||||
const { db } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 3, 40));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
@@ -351,7 +329,6 @@ test('a short run below every threshold survives', () => {
|
||||
assert.equal(countLines(db), 3);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -361,7 +338,7 @@ test('interleaved dual-line karaoke collapses each line to one row', () => {
|
||||
const kanji = karaokeFrames(1, '飛び上がる', 10_000, 20, 60);
|
||||
const romaji = karaokeFrames(1, 'tobiagaru', 10_001, 20, 60);
|
||||
const interleaved = [...kanji, ...romaji].sort((a, b) => a.startMs - b.startMs);
|
||||
const { db, dbPath } = createDb(interleaved);
|
||||
const { db } = createDb(interleaved);
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
@@ -372,12 +349,11 @@ test('interleaved dual-line karaoke collapses each line to one row', () => {
|
||||
assert.equal(wordFrequency(db), 2);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('the same line in a rewatch session is never merged into the first watch', () => {
|
||||
const { db, dbPath } = createDb([
|
||||
const { db } = createDb([
|
||||
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
|
||||
...karaokeFrames(2, '飛び上がる', 10_000, 6, 40),
|
||||
]);
|
||||
@@ -392,12 +368,11 @@ test('the same line in a rewatch session is never merged into the first watch',
|
||||
assert.equal(wordFrequency(db), 2);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a gap between runs splits them', () => {
|
||||
const { db, dbPath } = createDb([
|
||||
const { db } = createDb([
|
||||
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
|
||||
...karaokeFrames(1, '飛び上がる', 60_000, 6, 40),
|
||||
]);
|
||||
@@ -409,12 +384,11 @@ test('a gap between runs splits them', () => {
|
||||
assert.equal(countLines(db), 2);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a dry run reports what an apply would do and writes nothing', () => {
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
|
||||
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
|
||||
|
||||
try {
|
||||
const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
|
||||
@@ -430,14 +404,13 @@ test('a dry run reports what an apply would do and writes nothing', () => {
|
||||
assert.equal(countLines(db), 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('the lookback window leaves older bursts alone', () => {
|
||||
const recentMs = BASE_MS;
|
||||
const oldMs = BASE_MS - 40 * DAY_MS;
|
||||
const { db, dbPath } = createDb([
|
||||
const { db } = createDb([
|
||||
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40).map((line) => ({
|
||||
...line,
|
||||
createdMs: oldMs,
|
||||
@@ -462,6 +435,5 @@ test('the lookback window leaves older bursts alone', () => {
|
||||
} finally {
|
||||
globalThis.__subminerTestNowMs = undefined;
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
getKanjiOccurrences,
|
||||
getSessionSummaries,
|
||||
getVocabularyStats,
|
||||
getVocabularySummary,
|
||||
getKanjiStats,
|
||||
getSessionEvents,
|
||||
getSessionTimeline,
|
||||
@@ -1875,6 +1876,115 @@ test('getVocabularyStats returns rows ordered by frequency descending', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary counts every tracked vocabulary row instead of a display page', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const nowSec = Math.floor(Date.now() / 1000);
|
||||
const insertWord = db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', ?, ?, 1)
|
||||
`);
|
||||
const insertKanji = db.prepare(`
|
||||
INSERT INTO imm_kanji (kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)
|
||||
`);
|
||||
|
||||
for (let index = 0; index < 501; index += 1) {
|
||||
insertWord.run(`単語${index}`, `単語${index}`, nowSec - 8 * 86_400, nowSec - 8 * 86_400);
|
||||
}
|
||||
for (let index = 0; index < 201; index += 1) {
|
||||
insertKanji.run(
|
||||
String.fromCodePoint(0x4e00 + index),
|
||||
nowSec - 8 * 86_400,
|
||||
nowSec - 8 * 86_400,
|
||||
);
|
||||
}
|
||||
insertWord.run('今週', '今週', nowSec - 86_400, nowSec - 86_400);
|
||||
|
||||
assert.deepEqual(getVocabularySummary(db, new Set(['単語0', '今週']), nowSec * 1000), {
|
||||
uniqueWords: 502,
|
||||
uniqueWordsWithoutNames: 502,
|
||||
uniqueKanji: 201,
|
||||
newThisWeek: 1,
|
||||
newThisWeekWithoutNames: 1,
|
||||
knownWordCount: 2,
|
||||
knownWordCountWithoutNames: 2,
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary applies vocabulary exclusions and Hide Names totals', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, '', 'noun', '名詞', ?, '', 1, 1, 1)
|
||||
`);
|
||||
insertWord.run('猫', '猫', '一般');
|
||||
insertWord.run('太郎', '太郎', '固有名詞');
|
||||
insertWord.run('東京', '東京都', '一般');
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_stats_excluded_words (headword, word, reading)
|
||||
VALUES ('東京', '東京', '')
|
||||
`,
|
||||
).run();
|
||||
|
||||
assert.deepEqual(getVocabularySummary(db, new Set(['猫', '太郎', '東京']), 9 * 86_400_000), {
|
||||
uniqueWords: 2,
|
||||
uniqueWordsWithoutNames: 1,
|
||||
uniqueKanji: 0,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: 2,
|
||||
knownWordCountWithoutNames: 1,
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary counts identically across id-keyed scan batches', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', 1, 1, 1)
|
||||
`);
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
insertWord.run(`単語${index}`, `単語${index}`);
|
||||
}
|
||||
|
||||
const fullScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000);
|
||||
const batchedScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000, 2);
|
||||
|
||||
assert.equal(fullScan.uniqueWords, 5);
|
||||
assert.deepEqual(batchedScan, fullScan);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
LexicalRollupWorkerRuntime,
|
||||
resolveLexicalRollupWorkerPath,
|
||||
} from './lexical-rollup-worker-runtime';
|
||||
import { areLexicalDailyRollupsReady } from './lexical-rollups';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas, ensureSchema } from './storage';
|
||||
|
||||
test('lexical rollup worker backfills without using the tracker connection', async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-runtime-'));
|
||||
const dbPath = path.join(directory, 'immersion.sqlite');
|
||||
const runtime = new LexicalRollupWorkerRuntime();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
applyPragmas(db);
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES ('鳥', '鳥', 'とり', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||
db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
|
||||
'lexical_daily_rollups_version',
|
||||
);
|
||||
db.close();
|
||||
|
||||
await runtime.run(dbPath);
|
||||
|
||||
const checkDb = new Database(dbPath);
|
||||
try {
|
||||
assert.equal(areLexicalDailyRollupsReady(checkDb), true);
|
||||
} finally {
|
||||
checkDb.close();
|
||||
}
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
try {
|
||||
db.close();
|
||||
} catch {
|
||||
// Closed before the worker starts.
|
||||
}
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup worker module resolves in the current layout', () => {
|
||||
const workerPath = resolveLexicalRollupWorkerPath();
|
||||
assert.ok(workerPath, 'expected the lexical rollup worker module to resolve');
|
||||
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
|
||||
});
|
||||
|
||||
test('lexical rollup worker leaves a backfill pending when no worker can start', async () => {
|
||||
const runtime = new LexicalRollupWorkerRuntime({
|
||||
resolveWorkerPath: () => null,
|
||||
warn: () => {},
|
||||
} as never);
|
||||
|
||||
try {
|
||||
await assert.doesNotReject(runtime.run('/tmp/not-used.sqlite'));
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup worker absorbs termination failures after settling', async () => {
|
||||
let sendMessage: ((message: { ok: boolean }) => void) | null = null;
|
||||
const runtime = new LexicalRollupWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/fake-worker.js',
|
||||
createWorker: async () => ({
|
||||
once(event: string, listener: (value: never) => void) {
|
||||
if (event === 'message') sendMessage = listener as (message: { ok: boolean }) => void;
|
||||
return this;
|
||||
},
|
||||
terminate: async () => {
|
||||
throw new Error('termination failed');
|
||||
},
|
||||
}),
|
||||
warn: () => {},
|
||||
} as never);
|
||||
|
||||
const unhandled: unknown[] = [];
|
||||
const captureUnhandled = (reason: unknown) => unhandled.push(reason);
|
||||
process.on('unhandledRejection', captureUnhandled);
|
||||
try {
|
||||
const task = runtime.run('/tmp/not-used.sqlite');
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
const notify = sendMessage as ((message: { ok: boolean }) => void) | null;
|
||||
assert.ok(notify);
|
||||
notify({ ok: true });
|
||||
await task;
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.deepEqual(unhandled, []);
|
||||
} finally {
|
||||
process.off('unhandledRejection', captureUnhandled);
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup worker times out when it never responds', async () => {
|
||||
let terminated = false;
|
||||
const runtime = new LexicalRollupWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/fake-worker.js',
|
||||
createWorker: async () => ({
|
||||
once() {
|
||||
return this;
|
||||
},
|
||||
terminate: async () => {
|
||||
terminated = true;
|
||||
return 0;
|
||||
},
|
||||
}),
|
||||
timeoutMs: 1,
|
||||
warn: () => {},
|
||||
} as never);
|
||||
|
||||
try {
|
||||
const outcome = await Promise.race([
|
||||
runtime.run('/tmp/not-used.sqlite').then(
|
||||
() => 'resolved',
|
||||
(error: unknown) => String(error),
|
||||
),
|
||||
new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
|
||||
]);
|
||||
|
||||
assert.match(outcome, /timed out/);
|
||||
assert.equal(terminated, true);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createLogger } from '../../../logger';
|
||||
|
||||
interface WorkerResponse {
|
||||
ok?: boolean;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
interface WorkerHandle {
|
||||
once(event: 'message', listener: (message: WorkerResponse) => void): this;
|
||||
once(event: 'error', listener: (error: Error) => void): this;
|
||||
once(event: 'exit', listener: (code: number) => void): this;
|
||||
terminate(): Promise<number>;
|
||||
}
|
||||
|
||||
interface LexicalRollupWorkerRuntimeOptions {
|
||||
resolveWorkerPath?: () => string | null;
|
||||
createWorker?: (workerPath: string, workerData: { dbPath: string }) => Promise<WorkerHandle>;
|
||||
timeoutMs?: number;
|
||||
warn?: (message: string, ...meta: unknown[]) => void;
|
||||
}
|
||||
|
||||
const logger = createLogger('main:immersion-tracker:lexical-rollup-worker');
|
||||
const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
|
||||
|
||||
export function resolveLexicalRollupWorkerPath(): string | null {
|
||||
const fileName = __filename.endsWith('.ts')
|
||||
? 'lexical-rollup-worker-thread.ts'
|
||||
: 'lexical-rollup-worker-thread.js';
|
||||
const workerPath = path.join(__dirname, fileName);
|
||||
return fs.existsSync(workerPath) ? workerPath : null;
|
||||
}
|
||||
|
||||
export class LexicalRollupWorkerRuntime {
|
||||
private readonly activeWorkers = new Set<WorkerHandle>();
|
||||
private destroyed = false;
|
||||
|
||||
constructor(private readonly options: LexicalRollupWorkerRuntimeOptions = {}) {}
|
||||
|
||||
async run(dbPath: string): Promise<void> {
|
||||
if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
|
||||
let worker: WorkerHandle;
|
||||
try {
|
||||
const workerPath = (this.options.resolveWorkerPath ?? resolveLexicalRollupWorkerPath)();
|
||||
if (!workerPath) throw new Error('Emitted lexical rollup worker module was not found');
|
||||
const createWorker =
|
||||
this.options.createWorker ??
|
||||
(async (resolvedPath, workerData) => {
|
||||
const { Worker } = await import('node:worker_threads');
|
||||
return new Worker(resolvedPath, { workerData });
|
||||
});
|
||||
worker = await createWorker(workerPath, { dbPath });
|
||||
} catch (error) {
|
||||
if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
|
||||
(this.options.warn ?? logger.warn)(
|
||||
'Lexical rollup worker unavailable; leaving backfill pending for a later startup',
|
||||
error,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.destroyed) {
|
||||
await worker.terminate().catch(() => undefined);
|
||||
throw new Error('Lexical rollup worker is shut down');
|
||||
}
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
let settled = false;
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
this.activeWorkers.add(worker);
|
||||
const settle = (error?: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timeout) clearTimeout(timeout);
|
||||
this.activeWorkers.delete(worker);
|
||||
void worker.terminate().catch(() => undefined);
|
||||
if (error) reject(error);
|
||||
else resolve();
|
||||
};
|
||||
timeout = setTimeout(
|
||||
() => settle(new Error('Lexical rollup worker timed out')),
|
||||
this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
|
||||
);
|
||||
worker.once('message', (message) => {
|
||||
if (message.ok) settle();
|
||||
else
|
||||
settle(
|
||||
new Error(
|
||||
`Lexical rollup backfill failed: ${String(message.error ?? 'unknown error')}`,
|
||||
),
|
||||
);
|
||||
});
|
||||
worker.once('error', (error) => settle(error));
|
||||
worker.once('exit', (code) => {
|
||||
if (!settled) {
|
||||
settle(
|
||||
new Error(
|
||||
code === 0
|
||||
? 'Lexical rollup worker exited without a response'
|
||||
: `Lexical rollup worker exited with code ${code}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
for (const worker of this.activeWorkers) {
|
||||
void worker.terminate().catch(() => undefined);
|
||||
}
|
||||
this.activeWorkers.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
|
||||
|
||||
if (!parentPort) throw new Error('lexical rollup worker missing parent port');
|
||||
|
||||
try {
|
||||
executeLexicalRollupBackfillTask((workerData as { dbPath: string }).dbPath);
|
||||
parentPort.postMessage({ ok: true });
|
||||
} catch (error) {
|
||||
parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
|
||||
import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
|
||||
import { Database } from './sqlite';
|
||||
import { ensureSchema } from './storage';
|
||||
|
||||
test('lexical rollup backfill materializes pre-existing vocabulary off the caller DB connection', () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-worker-'));
|
||||
const dbPath = path.join(directory, 'immersion.sqlite');
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||
).run('犬', '犬', 'いぬ', 1_700_000_000, 1_700_000_000);
|
||||
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||
db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
|
||||
'lexical_daily_rollups_version',
|
||||
);
|
||||
|
||||
executeLexicalRollupBackfillTask(dbPath);
|
||||
|
||||
assert.equal(areLexicalDailyRollupsReady(db), true);
|
||||
assert.equal(getLexicalDailyRollups(db)[0]?.wordCount, 1);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
import { areLexicalDailyRollupsReady, rebuildLexicalDailyRollups } from './lexical-rollups';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas } from './storage';
|
||||
|
||||
export function executeLexicalRollupBackfillTask(dbPath: string): void {
|
||||
const db = new Database(dbPath);
|
||||
try {
|
||||
applyPragmas(db);
|
||||
if (!areLexicalDailyRollupsReady(db)) {
|
||||
rebuildLexicalDailyRollups(db);
|
||||
}
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,421 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
areLexicalDailyRollupsReady,
|
||||
getLexicalDailyRollups,
|
||||
rebuildLexicalDailyRollups,
|
||||
} from './lexical-rollups';
|
||||
import { getTrendsDashboard } from './query-trends';
|
||||
import {
|
||||
getVocabularyChartData,
|
||||
getVocabularySummary,
|
||||
replaceStatsExcludedWords,
|
||||
} from './query-lexical';
|
||||
import { Database } from './sqlite';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { ensureSchema } from './storage';
|
||||
|
||||
function makeDbPath(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollups-'));
|
||||
return path.join(dir, 'immersion.sqlite');
|
||||
}
|
||||
|
||||
test('lexical daily rollups follow first-seen corrections and deletions', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const firstDay = 19_500;
|
||||
const correctedDay = firstDay + 2;
|
||||
const firstSeen = firstDay * 86_400 + 43_200;
|
||||
const correctedSeen = correctedDay * 86_400 + 43_200;
|
||||
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||
).run('猫', '猫', 'ねこ', firstSeen, firstSeen);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)`,
|
||||
).run('猫', firstSeen, firstSeen);
|
||||
|
||||
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||
{ epochDay: firstDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 1 },
|
||||
]);
|
||||
|
||||
db.prepare(`UPDATE imm_words SET first_seen = ? WHERE headword = ?`).run(correctedSeen, '猫');
|
||||
db.prepare(`DELETE FROM imm_kanji WHERE kanji = ?`).run('猫');
|
||||
|
||||
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||
{ epochDay: correctedDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
|
||||
]);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical daily rollups normalize second and millisecond timestamps', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const epochDay = 19_500;
|
||||
const timestampSeconds = epochDay * 86_400 + 43_200;
|
||||
const timestampMilliseconds = timestampSeconds * 1_000;
|
||||
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||
).run('猫', '猫', 'ねこ', timestampSeconds, timestampSeconds);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||
).run('犬', '犬', 'いぬ', timestampMilliseconds, timestampMilliseconds);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)`,
|
||||
).run('猫', timestampSeconds, timestampSeconds);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)`,
|
||||
).run('犬', timestampMilliseconds, timestampMilliseconds);
|
||||
|
||||
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||
{ epochDay, wordCount: 2, wordCountWithoutNames: 2, kanjiCount: 2 },
|
||||
]);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup rebuild excludes rows hidden by vocabulary persistence rules', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const epochDay = 19_500;
|
||||
const firstSeen = epochDay * 86_400 + 43_200;
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(
|
||||
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||
).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(
|
||||
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||
).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
|
||||
|
||||
rebuildLexicalDailyRollups(db);
|
||||
|
||||
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||
{ epochDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
|
||||
]);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup rebuild tolerates nullable legacy vocabulary text', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (NULL, NULL, NULL, 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (NULL, '猫', 'ねこ', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
|
||||
assert.doesNotThrow(() => rebuildLexicalDailyRollups(db));
|
||||
assert.equal(areLexicalDailyRollupsReady(db), true);
|
||||
assert.equal(getVocabularySummary(db, null).uniqueWords, 1);
|
||||
assert.equal(getVocabularySummary(db, new Set(['猫'])).knownWordCount, 1);
|
||||
assert.equal(getVocabularyChartData(db).topWords[0]?.headword, '猫');
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup rebuild scans vocabulary visibility in bounded id batches', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
const expectedBatchSize = 5_000;
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, '', 1700000000, 1700000000, 1)`,
|
||||
);
|
||||
db.exec('BEGIN');
|
||||
for (let index = 0; index <= expectedBatchSize; index += 1) {
|
||||
insertWord.run(`語${index}`, `語${index}`);
|
||||
}
|
||||
db.exec('COMMIT');
|
||||
|
||||
const scanPageSizes: number[] = [];
|
||||
const instrumentedDb: DatabaseSync = {
|
||||
prepare(source) {
|
||||
const statement = db.prepare(source);
|
||||
if (!source.includes('WHERE id > ?') || !source.includes('ORDER BY id')) {
|
||||
return statement;
|
||||
}
|
||||
return {
|
||||
run: (...params) => statement.run(...params),
|
||||
get: (...params) => statement.get(...params),
|
||||
all: (...params) => {
|
||||
const rows = statement.all(...params);
|
||||
scanPageSizes.push(rows.length);
|
||||
return rows;
|
||||
},
|
||||
};
|
||||
},
|
||||
exec(source) {
|
||||
db.exec(source);
|
||||
return instrumentedDb;
|
||||
},
|
||||
close() {
|
||||
return instrumentedDb;
|
||||
},
|
||||
};
|
||||
|
||||
rebuildLexicalDailyRollups(instrumentedDb);
|
||||
|
||||
assert.deepEqual(scanPageSizes, [expectedBatchSize, 1]);
|
||||
assert.equal(getVocabularySummary(db, null).uniqueWords, expectedBatchSize + 1);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('chart exclusions do not subtract vocabulary rows already hidden from the rollup', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const epochDay = 19_500;
|
||||
const firstSeen = epochDay * 86_400 + 43_200;
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(
|
||||
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||
).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(
|
||||
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||
).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
|
||||
rebuildLexicalDailyRollups(db);
|
||||
replaceStatsExcludedWords(db, [{ headword: 'は', word: 'は', reading: 'は' }]);
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.deepEqual(charts.newWordsTimeline, [{ epochDay, wordCount: 1 }]);
|
||||
assert.deepEqual(charts.newWordsTimelineWithoutNames, [{ epochDay, wordCount: 1 }]);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('legacy lexical rollup readiness does not satisfy the current rollup version', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_rollup_state(state_key, state_value)
|
||||
VALUES ('lexical_daily_rollups_ready', '1')
|
||||
ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`DELETE FROM imm_rollup_state WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||
).run();
|
||||
|
||||
assert.equal(areLexicalDailyRollupsReady(db), false);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('current lexical rollup readiness accepts legacy integer state storage', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
db.exec(`
|
||||
CREATE TABLE imm_rollup_state(
|
||||
state_key TEXT PRIMARY KEY,
|
||||
state_value INTEGER NOT NULL
|
||||
);
|
||||
INSERT INTO imm_rollup_state(state_key, state_value)
|
||||
VALUES ('lexical_daily_rollups_version', 2);
|
||||
`);
|
||||
|
||||
assert.equal(areLexicalDailyRollupsReady(db), true);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('imm_words persists vocabulary visibility for rollup maintenance', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const columns = db.prepare(`PRAGMA table_info(imm_words)`).all() as Array<{ name: string }>;
|
||||
|
||||
assert.equal(
|
||||
columns.some((column) => column.name === 'vocabulary_visible'),
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary charts use complete top-word and lexical rollup data', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, '', 1700000000, 1700000000, ?)`,
|
||||
);
|
||||
for (let index = 0; index < 501; index += 1) {
|
||||
insertWord.run(`語${index}`, `語${index}`, index === 500 ? 10_000 : 1);
|
||||
}
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.equal(charts.topWords[0]?.headword, '語500');
|
||||
assert.equal(charts.topWords[0]?.frequency, 10_000);
|
||||
assert.equal(charts.newWordsTimeline[0]?.wordCount, 501);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary charts find full top-word sets beyond excluded and name rows', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, pos2, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, '', ?, 1700000000, 1700000000, ?)`,
|
||||
);
|
||||
const exclusions = [];
|
||||
for (let index = 0; index < 100; index += 1) {
|
||||
const headword = `語${index}`;
|
||||
insertWord.run(
|
||||
headword,
|
||||
headword,
|
||||
index < 80 && index >= 60 ? '固有名詞' : '一般',
|
||||
100 - index,
|
||||
);
|
||||
if (index < 60) exclusions.push({ headword, word: headword, reading: '' });
|
||||
}
|
||||
replaceStatsExcludedWords(db, exclusions);
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.equal(charts.topWords.length, 12);
|
||||
assert.equal(charts.topWords[0]?.headword, '語60');
|
||||
assert.equal(charts.topWordsWithoutNames.length, 12);
|
||||
assert.equal(charts.topWordsWithoutNames[0]?.headword, '語80');
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary charts handle exclusion lists above one SQLite variable batch', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES ('語0', '語0', '', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
const exclusions = Array.from({ length: 10_923 }, (_, index) => ({
|
||||
headword: `語${index}`,
|
||||
word: `語${index}`,
|
||||
reading: '',
|
||||
}));
|
||||
replaceStatsExcludedWords(db, exclusions);
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.deepEqual(charts.topWords, []);
|
||||
assert.deepEqual(charts.newWordsTimeline, []);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup rebuild preserves the original error when rollback also fails', () => {
|
||||
const originalError = new Error('rebuild failed');
|
||||
const db = {
|
||||
exec(sql: string) {
|
||||
if (sql === 'BEGIN IMMEDIATE') return;
|
||||
if (sql === 'ROLLBACK') throw new Error('rollback failed');
|
||||
throw originalError;
|
||||
},
|
||||
prepare() {
|
||||
return { all: () => [], run: () => undefined };
|
||||
},
|
||||
} as unknown as DatabaseSync;
|
||||
|
||||
assert.throws(() => rebuildLexicalDailyRollups(db), originalError);
|
||||
});
|
||||
|
||||
test('trends read historical new-word buckets from lexical rollups', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES ('海', '海', 'うみ', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
db.prepare(`UPDATE imm_lexical_daily_rollups SET word_count = 9`).run();
|
||||
|
||||
const dashboard = getTrendsDashboard(db, 'all', 'day', false);
|
||||
|
||||
assert.equal(dashboard.progress.newWords[0]?.value, 9);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,229 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility';
|
||||
|
||||
export interface LexicalDailyRollup {
|
||||
epochDay: number;
|
||||
wordCount: number;
|
||||
wordCountWithoutNames: number;
|
||||
kanjiCount: number;
|
||||
}
|
||||
|
||||
const LOCAL_EPOCH_DAY_SQL = `
|
||||
CAST(julianday(
|
||||
CASE
|
||||
WHEN ABS(CAST(%VALUE% AS REAL)) >= 10000000000 THEN CAST(%VALUE% AS REAL) / 1000
|
||||
ELSE CAST(%VALUE% AS REAL)
|
||||
END,
|
||||
'unixepoch', 'localtime'
|
||||
) - 2440587.5 AS INTEGER)
|
||||
`;
|
||||
|
||||
const LEXICAL_DAILY_ROLLUP_VERSION = '2';
|
||||
const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
|
||||
const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
|
||||
|
||||
export function localEpochDaySql(value: string): string {
|
||||
return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
|
||||
}
|
||||
|
||||
function createWordRollupTriggers(db: DatabaseSync): void {
|
||||
const dayForNew = localEpochDaySql('NEW.first_seen');
|
||||
const dayForOld = localEpochDaySql('OLD.first_seen');
|
||||
|
||||
db.exec(`
|
||||
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_insert;
|
||||
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_delete;
|
||||
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_first_seen_update;
|
||||
|
||||
CREATE TRIGGER imm_words_lexical_rollup_insert
|
||||
AFTER INSERT ON imm_words
|
||||
WHEN NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
VALUES (${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0)
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||
word_count = word_count + 1,
|
||||
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER imm_words_lexical_rollup_delete
|
||||
AFTER DELETE ON imm_words
|
||||
WHEN OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
VALUES (${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0)
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||
word_count = word_count - 1,
|
||||
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||
DELETE FROM imm_lexical_daily_rollups
|
||||
WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER imm_words_lexical_rollup_first_seen_update
|
||||
AFTER UPDATE OF first_seen, pos2, vocabulary_visible ON imm_words
|
||||
WHEN OLD.first_seen IS NOT NEW.first_seen
|
||||
OR OLD.pos2 IS NOT NEW.pos2
|
||||
OR OLD.vocabulary_visible IS NOT NEW.vocabulary_visible
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0
|
||||
WHERE OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||
word_count = word_count - 1,
|
||||
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0
|
||||
WHERE NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||
word_count = word_count + 1,
|
||||
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||
DELETE FROM imm_lexical_daily_rollups
|
||||
WHERE word_count = 0 AND kanji_count = 0;
|
||||
END;
|
||||
`);
|
||||
}
|
||||
|
||||
function createKanjiRollupTriggers(db: DatabaseSync): void {
|
||||
const dayForNew = localEpochDaySql('NEW.first_seen');
|
||||
const dayForOld = localEpochDaySql('OLD.first_seen');
|
||||
db.exec(`
|
||||
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_insert;
|
||||
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_delete;
|
||||
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_first_seen_update;
|
||||
|
||||
CREATE TRIGGER imm_kanji_lexical_rollup_insert
|
||||
AFTER INSERT ON imm_kanji WHEN NEW.first_seen IS NOT NULL
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
VALUES (${dayForNew}, 0, 0, 1)
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
|
||||
END;
|
||||
CREATE TRIGGER imm_kanji_lexical_rollup_delete
|
||||
AFTER DELETE ON imm_kanji WHEN OLD.first_seen IS NOT NULL
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
VALUES (${dayForOld}, 0, 0, -1)
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
|
||||
DELETE FROM imm_lexical_daily_rollups
|
||||
WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
|
||||
END;
|
||||
CREATE TRIGGER imm_kanji_lexical_rollup_first_seen_update
|
||||
AFTER UPDATE OF first_seen ON imm_kanji WHEN OLD.first_seen IS NOT NEW.first_seen
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${dayForOld}, 0, 0, -1 WHERE OLD.first_seen IS NOT NULL
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${dayForNew}, 0, 0, 1 WHERE NEW.first_seen IS NOT NULL
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
|
||||
DELETE FROM imm_lexical_daily_rollups WHERE word_count = 0 AND kanji_count = 0;
|
||||
END;
|
||||
`);
|
||||
}
|
||||
|
||||
export function ensureLexicalDailyRollupTables(db: DatabaseSync): void {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS imm_lexical_daily_rollups(
|
||||
epoch_day INTEGER PRIMARY KEY,
|
||||
word_count INTEGER NOT NULL DEFAULT 0,
|
||||
word_count_without_names INTEGER NOT NULL DEFAULT 0,
|
||||
kanji_count INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
INSERT INTO imm_rollup_state(state_key, state_value)
|
||||
VALUES ('${LEXICAL_DAILY_ROLLUP_VERSION_KEY}', '0')
|
||||
ON CONFLICT(state_key) DO NOTHING;
|
||||
`);
|
||||
createWordRollupTriggers(db);
|
||||
createKanjiRollupTriggers(db);
|
||||
}
|
||||
|
||||
export function areLexicalDailyRollupsReady(db: DatabaseSync): boolean {
|
||||
const row = db
|
||||
.prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
|
||||
.get(LEXICAL_DAILY_ROLLUP_VERSION_KEY) as { value: string | number } | undefined;
|
||||
// Older databases created this column with INTEGER affinity, while current
|
||||
// databases use TEXT. SQLite returns the same persisted version with a
|
||||
// different JS type depending on that legacy schema.
|
||||
return row !== undefined && String(row.value) === LEXICAL_DAILY_ROLLUP_VERSION;
|
||||
}
|
||||
|
||||
export function markLexicalDailyRollupsReady(db: DatabaseSync): void {
|
||||
db.prepare(
|
||||
`INSERT INTO imm_rollup_state(state_key, state_value)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
|
||||
).run(LEXICAL_DAILY_ROLLUP_VERSION_KEY, LEXICAL_DAILY_ROLLUP_VERSION);
|
||||
}
|
||||
|
||||
/** Rebuild from the first-seen source of truth; run off the UI/main DB thread. */
|
||||
export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
|
||||
let transactionStarted = false;
|
||||
try {
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
transactionStarted = true;
|
||||
const scanVocabulary = db.prepare(
|
||||
`SELECT id, word, headword, reading, part_of_speech AS partOfSpeech,
|
||||
pos1, pos2, pos3, frequency_rank AS frequencyRank
|
||||
FROM imm_words
|
||||
WHERE id > ?
|
||||
ORDER BY id
|
||||
LIMIT ?`,
|
||||
);
|
||||
const updateVisibility = db.prepare(
|
||||
`UPDATE imm_words SET vocabulary_visible = ? WHERE id = ? AND vocabulary_visible IS NOT ?`,
|
||||
);
|
||||
let lastId = Number.MIN_SAFE_INTEGER;
|
||||
for (;;) {
|
||||
const vocabularyRows = scanVocabulary.all(
|
||||
lastId,
|
||||
VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE,
|
||||
) as Array<VocabularyVisibilityRow & { id: number }>;
|
||||
if (vocabularyRows.length === 0) break;
|
||||
for (const row of vocabularyRows) {
|
||||
const visible = isVocabularyStatsRowVisible(row) ? 1 : 0;
|
||||
updateVisibility.run(visible, row.id, visible);
|
||||
}
|
||||
lastId = vocabularyRows[vocabularyRows.length - 1]!.id;
|
||||
if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break;
|
||||
}
|
||||
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||
db.exec(`
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${localEpochDaySql('first_seen')}, COUNT(*),
|
||||
SUM(CASE WHEN pos2 = '固有名詞' THEN 0 ELSE 1 END), 0
|
||||
FROM imm_words
|
||||
WHERE first_seen IS NOT NULL AND vocabulary_visible = 1
|
||||
GROUP BY ${localEpochDaySql('first_seen')};
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${localEpochDaySql('first_seen')}, 0, 0, COUNT(*)
|
||||
FROM imm_kanji
|
||||
WHERE first_seen IS NOT NULL
|
||||
GROUP BY ${localEpochDaySql('first_seen')}
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + excluded.kanji_count;
|
||||
`);
|
||||
markLexicalDailyRollupsReady(db);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
if (transactionStarted) {
|
||||
try {
|
||||
db.exec('ROLLBACK');
|
||||
} catch {
|
||||
// Preserve the rebuild failure; it is the actionable cause.
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function getLexicalDailyRollups(db: DatabaseSync): LexicalDailyRollup[] {
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT epoch_day AS epochDay, word_count AS wordCount,
|
||||
word_count_without_names AS wordCountWithoutNames, kanji_count AS kanjiCount
|
||||
FROM imm_lexical_daily_rollups
|
||||
ORDER BY epoch_day ASC
|
||||
`,
|
||||
)
|
||||
.all() as LexicalDailyRollup[];
|
||||
}
|
||||
@@ -1,6 +1,4 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { PartOfSpeech, type MergedToken } from '../../../types';
|
||||
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
|
||||
import type {
|
||||
KanjiAnimeAppearanceRow,
|
||||
KanjiDetailRow,
|
||||
@@ -13,19 +11,38 @@ import type {
|
||||
SimilarWordRow,
|
||||
StatsExcludedWordRow,
|
||||
VocabularyStatsRow,
|
||||
VocabularyStatsSummary,
|
||||
WordAnimeAppearanceRow,
|
||||
WordDetailRow,
|
||||
WordOccurrenceRow,
|
||||
} from './types';
|
||||
import { fromDbTimestamp, toDbTimestamp } from './query-shared';
|
||||
import { nowMs } from './time';
|
||||
import {
|
||||
areLexicalDailyRollupsReady,
|
||||
getLexicalDailyRollups,
|
||||
localEpochDaySql,
|
||||
} from './lexical-rollups';
|
||||
import { isVocabularyStatsRowVisible } from './vocabulary-visibility';
|
||||
|
||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
|
||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
|
||||
const VOCABULARY_CHART_LIMIT = 12;
|
||||
const VOCABULARY_CHART_PAGE_SIZE = 100;
|
||||
const EXCLUSION_ALIAS_BATCH_SIZE = 300;
|
||||
const VOCABULARY_SUMMARY_SCAN_BATCH_SIZE = 5_000;
|
||||
const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
|
||||
const SENTENCE_SEARCH_MAX_LIMIT = 100;
|
||||
const KANJI_PATTERN = /\p{Script=Han}/gu;
|
||||
|
||||
export interface VocabularyChartData {
|
||||
ready: boolean;
|
||||
topWords: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||
topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||
newWordsTimeline: Array<{ epochDay: number; wordCount: number }>;
|
||||
newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>;
|
||||
}
|
||||
|
||||
function resolveSentenceSearchLimit(limit: number): number {
|
||||
if (!Number.isFinite(limit)) return SENTENCE_SEARCH_DEFAULT_LIMIT;
|
||||
const normalized = Math.floor(limit);
|
||||
@@ -73,33 +90,6 @@ function uniqueKanji(text: string): string[] {
|
||||
return Array.from(new Set(text.match(KANJI_PATTERN) ?? []));
|
||||
}
|
||||
|
||||
function toVocabularyToken(row: VocabularyStatsRow): MergedToken {
|
||||
const partOfSpeech =
|
||||
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
|
||||
? (row.partOfSpeech as PartOfSpeech)
|
||||
: PartOfSpeech.other;
|
||||
|
||||
return {
|
||||
surface: row.word,
|
||||
reading: row.reading ?? '',
|
||||
headword: row.headword,
|
||||
startPos: 0,
|
||||
endPos: row.word.length,
|
||||
partOfSpeech,
|
||||
pos1: row.pos1 ?? '',
|
||||
pos2: row.pos2 ?? '',
|
||||
pos3: row.pos3 ?? '',
|
||||
frequencyRank: row.frequencyRank ?? undefined,
|
||||
isMerged: false,
|
||||
isKnown: false,
|
||||
isNPlusOneTarget: false,
|
||||
};
|
||||
}
|
||||
|
||||
function isVocabularyStatsRowVisible(row: VocabularyStatsRow): boolean {
|
||||
return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
|
||||
}
|
||||
|
||||
export function getVocabularyStats(
|
||||
db: DatabaseSync,
|
||||
limit = 100,
|
||||
@@ -153,6 +143,198 @@ export function getVocabularyStats(
|
||||
return visibleRows.slice(0, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Chart data is intentionally independent of the paginated vocabulary tables.
|
||||
* Top words use the frequency index; new-word history reads permanent daily
|
||||
* lexical rollups rather than loading every vocabulary row into the dashboard.
|
||||
*/
|
||||
export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData {
|
||||
const ready = areLexicalDailyRollupsReady(db);
|
||||
const excludedAliases = new Set(
|
||||
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
|
||||
);
|
||||
const isExcluded = (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>): boolean =>
|
||||
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias));
|
||||
const topWords = getTopVocabularyChartWords(db, isExcluded);
|
||||
const rollups = ready ? getLexicalDailyRollups(db) : [];
|
||||
const timeline = new Map(rollups.map((row) => [row.epochDay, { ...row }]));
|
||||
if (excludedAliases.size > 0 && ready) {
|
||||
const aliases = [...excludedAliases];
|
||||
const excludedRows = new Map<
|
||||
number,
|
||||
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
|
||||
wordId: number;
|
||||
epochDay: number;
|
||||
}
|
||||
>();
|
||||
for (let offset = 0; offset < aliases.length; offset += EXCLUSION_ALIAS_BATCH_SIZE) {
|
||||
const batch = aliases.slice(offset, offset + EXCLUSION_ALIAS_BATCH_SIZE);
|
||||
const placeholders = batch.map(() => '?').join(', ');
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id AS wordId, headword, word, reading, pos2,
|
||||
${localEpochDaySql('first_seen')} AS epochDay
|
||||
FROM imm_words
|
||||
WHERE vocabulary_visible = 1
|
||||
AND (headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}))
|
||||
`,
|
||||
)
|
||||
.all(...batch, ...batch, ...batch) as Array<
|
||||
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
|
||||
wordId: number;
|
||||
epochDay: number;
|
||||
}
|
||||
>;
|
||||
for (const row of rows) excludedRows.set(row.wordId, row);
|
||||
}
|
||||
for (const word of excludedRows.values()) {
|
||||
if (!isExcluded(word)) continue;
|
||||
const rollup = timeline.get(word.epochDay);
|
||||
if (!rollup) continue;
|
||||
rollup.wordCount -= 1;
|
||||
if (word.pos2 !== '固有名詞') rollup.wordCountWithoutNames -= 1;
|
||||
}
|
||||
}
|
||||
return {
|
||||
ready,
|
||||
topWords: topWords.all.map((word) => ({
|
||||
wordId: word.wordId,
|
||||
headword: vocabularyDisplayHeadword(word),
|
||||
frequency: word.frequency,
|
||||
})),
|
||||
topWordsWithoutNames: topWords.withoutNames.map((word) => ({
|
||||
wordId: word.wordId,
|
||||
headword: vocabularyDisplayHeadword(word),
|
||||
frequency: word.frequency,
|
||||
})),
|
||||
newWordsTimeline: [...timeline.values()]
|
||||
.filter((row) => row.wordCount > 0)
|
||||
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCount })),
|
||||
newWordsTimelineWithoutNames: [...timeline.values()]
|
||||
.filter((row) => row.wordCountWithoutNames > 0)
|
||||
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCountWithoutNames })),
|
||||
};
|
||||
}
|
||||
|
||||
function getTopVocabularyChartWords(
|
||||
db: DatabaseSync,
|
||||
isExcluded: (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>) => boolean,
|
||||
): { all: VocabularyStatsRow[]; withoutNames: VocabularyStatsRow[] } {
|
||||
const stmt = db.prepare(`
|
||||
SELECT id AS wordId, headword, word, reading,
|
||||
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
|
||||
frequency, frequency_rank AS frequencyRank,
|
||||
first_seen AS firstSeen, last_seen AS lastSeen,
|
||||
0 AS animeCount
|
||||
FROM imm_words
|
||||
ORDER BY frequency DESC, id
|
||||
LIMIT ? OFFSET ?
|
||||
`);
|
||||
const all: VocabularyStatsRow[] = [];
|
||||
const withoutNames: VocabularyStatsRow[] = [];
|
||||
let offset = 0;
|
||||
|
||||
while (all.length < VOCABULARY_CHART_LIMIT || withoutNames.length < VOCABULARY_CHART_LIMIT) {
|
||||
const page = stmt.all(VOCABULARY_CHART_PAGE_SIZE, offset) as VocabularyStatsRow[];
|
||||
if (page.length === 0) break;
|
||||
for (const word of page) {
|
||||
if (!isVocabularyStatsRowVisible(word) || isExcluded(word)) continue;
|
||||
if (all.length < VOCABULARY_CHART_LIMIT) all.push(word);
|
||||
if (word.pos2 !== '固有名詞' && withoutNames.length < VOCABULARY_CHART_LIMIT) {
|
||||
withoutNames.push(word);
|
||||
}
|
||||
}
|
||||
offset += page.length;
|
||||
}
|
||||
|
||||
return { all, withoutNames };
|
||||
}
|
||||
|
||||
function excludedVocabularyAliases(
|
||||
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
|
||||
): string[] {
|
||||
const aliases = [word.headword?.trim() ?? '', word.word?.trim() ?? ''].filter(Boolean);
|
||||
if (aliases.length === 0) aliases.push(word.reading?.trim() ?? '');
|
||||
return [...new Set(aliases)];
|
||||
}
|
||||
|
||||
function vocabularyDisplayHeadword(
|
||||
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
|
||||
): string {
|
||||
return word.headword?.trim() || word.word?.trim() || word.reading?.trim() || '';
|
||||
}
|
||||
|
||||
function timestampSeconds(timestamp: number): number {
|
||||
return timestamp < 10_000_000_000 ? timestamp : Math.floor(timestamp / 1000);
|
||||
}
|
||||
|
||||
export function getVocabularySummary(
|
||||
db: DatabaseSync,
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
nowMs: number = Date.now(),
|
||||
scanBatchSize: number = VOCABULARY_SUMMARY_SCAN_BATCH_SIZE,
|
||||
): VocabularyStatsSummary {
|
||||
// Visibility and exclusion rules live in JS, so rows are scanned in id-keyed
|
||||
// batches to keep memory bounded on large vocabularies.
|
||||
const scanStmt = db.prepare(`
|
||||
SELECT id AS wordId, headword, word, reading,
|
||||
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
|
||||
frequency, frequency_rank AS frequencyRank,
|
||||
first_seen AS firstSeen, last_seen AS lastSeen,
|
||||
0 AS animeCount
|
||||
FROM imm_words
|
||||
WHERE id > ?
|
||||
ORDER BY id
|
||||
LIMIT ?
|
||||
`);
|
||||
const excludedAliases = new Set(
|
||||
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
|
||||
);
|
||||
const weekAgoSec = nowMs / 1000 - 7 * 86_400;
|
||||
const summary: VocabularyStatsSummary = {
|
||||
uniqueWords: 0,
|
||||
uniqueWordsWithoutNames: 0,
|
||||
uniqueKanji: (db.prepare('SELECT COUNT(*) AS count FROM imm_kanji').get() as { count: number })
|
||||
.count,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: knownWords ? 0 : null,
|
||||
knownWordCountWithoutNames: knownWords ? 0 : null,
|
||||
};
|
||||
|
||||
let lastId = Number.MIN_SAFE_INTEGER;
|
||||
for (;;) {
|
||||
const words = scanStmt.all(lastId, scanBatchSize) as VocabularyStatsRow[];
|
||||
if (words.length === 0) break;
|
||||
lastId = words[words.length - 1]!.wordId;
|
||||
for (const word of words) {
|
||||
if (
|
||||
!isVocabularyStatsRowVisible(word) ||
|
||||
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const isName = word.pos2 === '固有名詞';
|
||||
const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec;
|
||||
const isKnown = knownWords?.has(vocabularyDisplayHeadword(word)) ?? false;
|
||||
summary.uniqueWords += 1;
|
||||
if (!isName) summary.uniqueWordsWithoutNames += 1;
|
||||
if (isNewThisWeek) {
|
||||
summary.newThisWeek += 1;
|
||||
if (!isName) summary.newThisWeekWithoutNames += 1;
|
||||
}
|
||||
if (isKnown) {
|
||||
summary.knownWordCount! += 1;
|
||||
if (!isName) summary.knownWordCountWithoutNames! += 1;
|
||||
}
|
||||
}
|
||||
if (words.length < scanBatchSize) break;
|
||||
}
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
export function getStatsExcludedWords(db: DatabaseSync): StatsExcludedWordRow[] {
|
||||
return db
|
||||
.prepare(
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
toDbTimestamp,
|
||||
} from './query-shared';
|
||||
import { getDailyRollups, getMonthlyRollups } from './query-sessions';
|
||||
import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
|
||||
|
||||
type TrendRange = '7d' | '30d' | '90d' | '365d' | 'all';
|
||||
type TrendGroupBy = 'day' | 'month';
|
||||
@@ -660,6 +661,16 @@ function buildNewWordsPerDay(
|
||||
cutoffMs: string | null,
|
||||
axis: number[] | null,
|
||||
): TrendChartPoint[] {
|
||||
if (areLexicalDailyRollupsReady(db)) {
|
||||
// A trend range is defined in calendar buckets, so the rollup includes the
|
||||
// complete local cutoff day rather than applying a time-of-day boundary.
|
||||
const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
|
||||
const rows = getLexicalDailyRollups(db).filter(
|
||||
(row) => cutoffDay === null || row.epochDay >= cutoffDay,
|
||||
);
|
||||
return fillAxisPoints(axis, new Map(rows.map((row) => [row.epochDay, row.wordCount])));
|
||||
}
|
||||
|
||||
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
||||
const prepared = db.prepare(`
|
||||
SELECT
|
||||
@@ -691,6 +702,18 @@ function buildNewWordsPerMonth(
|
||||
cutoffMs: string | null,
|
||||
axis: number[] | null,
|
||||
): TrendChartPoint[] {
|
||||
if (areLexicalDailyRollupsReady(db)) {
|
||||
const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
|
||||
const byMonth = new Map<number, number>();
|
||||
for (const row of getLexicalDailyRollups(db)) {
|
||||
if (cutoffDay !== null && row.epochDay < cutoffDay) continue;
|
||||
const { year, month } = dayPartsFromEpochDay(row.epochDay);
|
||||
const monthKey = year * 100 + month;
|
||||
byMonth.set(monthKey, (byMonth.get(monthKey) ?? 0) + row.wordCount);
|
||||
}
|
||||
return fillAxisPoints(axis, byMonth);
|
||||
}
|
||||
|
||||
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
||||
const prepared = db.prepare(`
|
||||
SELECT
|
||||
|
||||
@@ -184,6 +184,51 @@ test('ensureSchema adds manual assignment locks when upgrading the previous sche
|
||||
}
|
||||
});
|
||||
|
||||
test('ensureSchema preserves durable session rollups across unrelated schema upgrades', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.exec(`
|
||||
INSERT INTO imm_videos (
|
||||
video_id, video_key, canonical_title, source_type, duration_ms, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (1, 'local:/tmp/preserved.mkv', 'Preserved', 1, 0, '1', '1');
|
||||
INSERT INTO imm_daily_rollups (
|
||||
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards
|
||||
) VALUES (20000, 1, 2, 30, 40, 50, 3);
|
||||
INSERT INTO imm_monthly_rollups (
|
||||
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards
|
||||
) VALUES (202410, 1, 2, 30, 40, 50, 3);
|
||||
UPDATE imm_rollup_state
|
||||
SET state_value = '123'
|
||||
WHERE state_key = 'last_rollup_sample_ms';
|
||||
UPDATE imm_schema_version SET schema_version = 21;
|
||||
`);
|
||||
|
||||
ensureSchema(db);
|
||||
|
||||
const daily = db
|
||||
.prepare('SELECT total_sessions AS totalSessions FROM imm_daily_rollups')
|
||||
.get() as { totalSessions: number } | null;
|
||||
const monthly = db
|
||||
.prepare('SELECT total_sessions AS totalSessions FROM imm_monthly_rollups')
|
||||
.get() as { totalSessions: number } | null;
|
||||
const rollupState = db
|
||||
.prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
|
||||
.get('last_rollup_sample_ms') as { value: string } | null;
|
||||
|
||||
assert.equal(daily?.totalSessions, 2);
|
||||
assert.equal(monthly?.totalSessions, 2);
|
||||
assert.equal(rollupState?.value, '123');
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('stats excluded words are replaced and read from sqlite storage', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
@@ -4,6 +4,7 @@ import { parseMediaInfo } from '../../../jimaku/utils';
|
||||
import { normalizeTitleIdentity } from '../../utils/title-normalization';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { nowMs } from './time';
|
||||
import { ensureLexicalDailyRollupTables, markLexicalDailyRollupsReady } from './lexical-rollups';
|
||||
import { SCHEMA_VERSION } from './types';
|
||||
import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types';
|
||||
import { toDbMs, toDbTimestamp } from './query-shared';
|
||||
@@ -890,11 +891,11 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
VALUES ('last_rollup_sample_ms', 0)
|
||||
ON CONFLICT(state_key) DO NOTHING
|
||||
`);
|
||||
|
||||
const currentVersion = db
|
||||
.prepare('SELECT schema_version FROM imm_schema_version ORDER BY schema_version DESC LIMIT 1')
|
||||
.get() as { schema_version: number } | null;
|
||||
if (currentVersion?.schema_version === SCHEMA_VERSION) {
|
||||
ensureLexicalDailyRollupTables(db);
|
||||
ensureLifetimeSummaryTables(db);
|
||||
ensureStatsExcludedWordsTable(db);
|
||||
ensureAnimeMergeTables(db);
|
||||
@@ -1068,6 +1069,7 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
last_seen REAL,
|
||||
frequency INTEGER,
|
||||
frequency_rank INTEGER,
|
||||
vocabulary_visible INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1)),
|
||||
UNIQUE(headword, word, reading)
|
||||
);
|
||||
`);
|
||||
@@ -1451,8 +1453,18 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
addColumnIfMissing(db, 'imm_sessions', 'ended_media_ms', 'INTEGER');
|
||||
}
|
||||
|
||||
if (currentVersion?.schema_version && currentVersion.schema_version < 23) {
|
||||
addColumnIfMissing(
|
||||
db,
|
||||
'imm_words',
|
||||
'vocabulary_visible',
|
||||
'INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1))',
|
||||
);
|
||||
}
|
||||
|
||||
migrateSessionEventTimestampsToText(db);
|
||||
|
||||
ensureLexicalDailyRollupTables(db);
|
||||
ensureLifetimeSummaryTables(db);
|
||||
ensureStatsExcludedWordsTable(db);
|
||||
|
||||
@@ -1572,19 +1584,21 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
ON imm_youtube_videos(youtube_video_id)
|
||||
`);
|
||||
|
||||
if (currentVersion?.schema_version && currentVersion.schema_version < SCHEMA_VERSION) {
|
||||
db.exec('DELETE FROM imm_daily_rollups');
|
||||
db.exec('DELETE FROM imm_monthly_rollups');
|
||||
db.exec(
|
||||
`UPDATE imm_rollup_state SET state_value = 0 WHERE state_key = 'last_rollup_sample_ms'`,
|
||||
);
|
||||
}
|
||||
// Session rollups intentionally outlive raw session and telemetry retention.
|
||||
// Preserve them across unrelated schema upgrades because deleted historical
|
||||
// buckets cannot be rebuilt after their source rows have been pruned.
|
||||
|
||||
db.exec(`
|
||||
INSERT INTO imm_schema_version(schema_version, applied_at_ms)
|
||||
VALUES (${SCHEMA_VERSION}, ${toDbTimestamp(nowMs())})
|
||||
ON CONFLICT DO NOTHING
|
||||
`);
|
||||
|
||||
// A new database has no history to materialize. Upgrades are populated by the
|
||||
// background worker so startup never scans the existing vocabulary table.
|
||||
if (!currentVersion) {
|
||||
markLexicalDailyRollupsReady(db);
|
||||
}
|
||||
}
|
||||
|
||||
export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPreparedStatements {
|
||||
@@ -1617,9 +1631,10 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
|
||||
`),
|
||||
wordUpsertStmt: db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency, frequency_rank
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen,
|
||||
frequency, frequency_rank, vocabulary_visible
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, 1
|
||||
)
|
||||
ON CONFLICT(headword, word, reading) DO UPDATE SET
|
||||
frequency = COALESCE(frequency, 0) + 1,
|
||||
@@ -1632,6 +1647,7 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
|
||||
pos1 = COALESCE(NULLIF(imm_words.pos1, ''), excluded.pos1),
|
||||
pos2 = COALESCE(NULLIF(imm_words.pos2, ''), excluded.pos2),
|
||||
pos3 = COALESCE(NULLIF(imm_words.pos3, ''), excluded.pos3),
|
||||
vocabulary_visible = 1,
|
||||
first_seen = MIN(COALESCE(first_seen, excluded.first_seen), excluded.first_seen),
|
||||
last_seen = MAX(COALESCE(last_seen, excluded.last_seen), excluded.last_seen),
|
||||
frequency_rank = CASE
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export const SCHEMA_VERSION = 21;
|
||||
export const SCHEMA_VERSION = 23;
|
||||
export const DEFAULT_QUEUE_CAP = 1_000;
|
||||
export const DEFAULT_BATCH_SIZE = 25;
|
||||
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
|
||||
@@ -306,6 +306,16 @@ export interface VocabularyStatsRow {
|
||||
lastSeen: number;
|
||||
}
|
||||
|
||||
export interface VocabularyStatsSummary {
|
||||
uniqueWords: number;
|
||||
uniqueWordsWithoutNames: number;
|
||||
uniqueKanji: number;
|
||||
newThisWeek: number;
|
||||
newThisWeekWithoutNames: number;
|
||||
knownWordCount: number | null;
|
||||
knownWordCountWithoutNames: number | null;
|
||||
}
|
||||
|
||||
export interface StatsExcludedWordRow {
|
||||
headword: string;
|
||||
word: string;
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
resolveVocabularySummaryWorkerPath,
|
||||
VocabularySummaryWorkerRuntime,
|
||||
} from './vocabulary-summary-worker-runtime';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas, ensureSchema } from './storage';
|
||||
|
||||
test('vocabulary summary worker reads the database from a separate connection', async () => {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-vocabulary-summary-worker-'));
|
||||
const dbPath = path.join(tempDir, 'immersion.sqlite');
|
||||
const runtime = new VocabularySummaryWorkerRuntime();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
applyPragmas(db);
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES ('猫', '猫', 'ねこ', 'noun', '名詞', '一般', '', 1, 1, 1)
|
||||
`,
|
||||
).run();
|
||||
db.close();
|
||||
|
||||
const summary = await runtime.run(dbPath, new Set(['猫']));
|
||||
|
||||
assert.equal(summary.uniqueWords, 1);
|
||||
assert.equal(summary.knownWordCount, 1);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
try {
|
||||
db.close();
|
||||
} catch {
|
||||
// The worker needs the setup connection closed before it starts.
|
||||
}
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary summary worker module resolves in the current layout', () => {
|
||||
const workerPath = resolveVocabularySummaryWorkerPath();
|
||||
assert.ok(workerPath, 'expected the vocabulary summary worker module to resolve');
|
||||
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
|
||||
});
|
||||
|
||||
test('vocabulary summary worker never falls back to the caller thread', async () => {
|
||||
const runtime = new VocabularySummaryWorkerRuntime({
|
||||
resolveWorkerPath: () => null,
|
||||
warn: () => {},
|
||||
});
|
||||
|
||||
try {
|
||||
await assert.rejects(
|
||||
runtime.run('/tmp/subminer-summary-worker-not-used.sqlite', null),
|
||||
/worker unavailable/i,
|
||||
);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary summary worker times out when it never responds', async () => {
|
||||
let terminated = false;
|
||||
const runtime = new VocabularySummaryWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/fake-worker.js',
|
||||
createWorker: async () => ({
|
||||
once() {
|
||||
return this;
|
||||
},
|
||||
terminate: async () => {
|
||||
terminated = true;
|
||||
return 0;
|
||||
},
|
||||
}),
|
||||
timeoutMs: 1,
|
||||
warn: () => {},
|
||||
} as never);
|
||||
|
||||
try {
|
||||
const outcome = await Promise.race([
|
||||
runtime.run('/tmp/not-used.sqlite', null).then(
|
||||
() => 'resolved',
|
||||
(error: unknown) => String(error),
|
||||
),
|
||||
new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
|
||||
]);
|
||||
|
||||
assert.match(outcome, /timed out/);
|
||||
assert.equal(terminated, true);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createLogger } from '../../../logger';
|
||||
import type { VocabularyStatsSummary } from './types';
|
||||
|
||||
interface VocabularySummaryWorkerResponse {
|
||||
summary?: VocabularyStatsSummary;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
interface VocabularySummaryWorkerHandle {
|
||||
once(event: 'message', listener: (message: VocabularySummaryWorkerResponse) => void): this;
|
||||
once(event: 'error', listener: (error: Error) => void): this;
|
||||
once(event: 'exit', listener: (code: number) => void): this;
|
||||
terminate(): Promise<number>;
|
||||
}
|
||||
|
||||
interface VocabularySummaryWorkerRuntimeOptions {
|
||||
resolveWorkerPath?: () => string | null;
|
||||
createWorker?: (
|
||||
workerPath: string,
|
||||
workerData: { dbPath: string; knownWords: string[] | null },
|
||||
) => Promise<VocabularySummaryWorkerHandle>;
|
||||
timeoutMs?: number;
|
||||
warn?: (message: string, ...meta: unknown[]) => void;
|
||||
}
|
||||
|
||||
export type RunVocabularySummaryTask = (
|
||||
dbPath: string,
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
) => Promise<VocabularyStatsSummary>;
|
||||
|
||||
export function resolveVocabularySummaryWorkerPath(): string | null {
|
||||
const fileName = __filename.endsWith('.ts')
|
||||
? 'vocabulary-summary-worker-thread.ts'
|
||||
: 'vocabulary-summary-worker-thread.js';
|
||||
const workerPath = path.join(__dirname, fileName);
|
||||
return fs.existsSync(workerPath) ? workerPath : null;
|
||||
}
|
||||
|
||||
const logger = createLogger('main:immersion-tracker:vocabulary-summary-worker');
|
||||
const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
|
||||
|
||||
export class VocabularySummaryWorkerRuntime {
|
||||
private readonly activeWorkers = new Set<VocabularySummaryWorkerHandle>();
|
||||
private destroyed = false;
|
||||
|
||||
constructor(private readonly options: VocabularySummaryWorkerRuntimeOptions = {}) {}
|
||||
|
||||
async run(
|
||||
dbPath: string,
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
): Promise<VocabularyStatsSummary> {
|
||||
if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
|
||||
const workerData = { dbPath, knownWords: knownWords ? [...knownWords] : null };
|
||||
let worker: VocabularySummaryWorkerHandle;
|
||||
try {
|
||||
const workerPath = (this.options.resolveWorkerPath ?? resolveVocabularySummaryWorkerPath)();
|
||||
if (!workerPath) throw new Error('Emitted vocabulary summary worker module was not found');
|
||||
const createWorker =
|
||||
this.options.createWorker ??
|
||||
(async (resolvedPath, data) => {
|
||||
const { Worker } = await import('node:worker_threads');
|
||||
return new Worker(resolvedPath, { workerData: data });
|
||||
});
|
||||
worker = await createWorker(workerPath, workerData);
|
||||
} catch (error) {
|
||||
if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
|
||||
(this.options.warn ?? logger.warn)(
|
||||
'Vocabulary summary worker unavailable; refusing to scan vocabulary on the current thread',
|
||||
error,
|
||||
);
|
||||
throw new Error('Vocabulary summary worker unavailable');
|
||||
}
|
||||
|
||||
if (this.destroyed) {
|
||||
await worker.terminate().catch(() => undefined);
|
||||
throw new Error('Vocabulary summary worker is shut down');
|
||||
}
|
||||
|
||||
return new Promise<VocabularyStatsSummary>((resolve, reject) => {
|
||||
let settled = false;
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
this.activeWorkers.add(worker);
|
||||
const settle = (result: VocabularyStatsSummary | Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timeout) clearTimeout(timeout);
|
||||
this.activeWorkers.delete(worker);
|
||||
void worker.terminate().catch(() => undefined);
|
||||
if (result instanceof Error) reject(result);
|
||||
else resolve(result);
|
||||
};
|
||||
timeout = setTimeout(
|
||||
() => settle(new Error('Vocabulary summary worker timed out')),
|
||||
this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
worker.once('message', (message) => {
|
||||
if (message.summary) {
|
||||
settle(message.summary);
|
||||
return;
|
||||
}
|
||||
settle(
|
||||
new Error(
|
||||
`Vocabulary summary failed: ${String(message.error ?? 'unknown worker error')}`,
|
||||
),
|
||||
);
|
||||
});
|
||||
worker.once('error', (error) => settle(error));
|
||||
worker.once('exit', (code) => {
|
||||
if (!settled) {
|
||||
settle(
|
||||
new Error(
|
||||
code === 0
|
||||
? 'Vocabulary summary worker exited without a response'
|
||||
: `Vocabulary summary worker exited with code ${code}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
for (const worker of this.activeWorkers) {
|
||||
void worker.terminate().catch(() => undefined);
|
||||
}
|
||||
this.activeWorkers.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
import { executeVocabularySummaryTask } from './vocabulary-summary-worker';
|
||||
|
||||
interface VocabularySummaryWorkerData {
|
||||
dbPath: string;
|
||||
knownWords: string[] | null;
|
||||
}
|
||||
|
||||
if (!parentPort) throw new Error('vocabulary summary worker missing parent port');
|
||||
|
||||
const request = workerData as VocabularySummaryWorkerData;
|
||||
|
||||
try {
|
||||
parentPort.postMessage({
|
||||
summary: executeVocabularySummaryTask(request.dbPath, request.knownWords),
|
||||
});
|
||||
} catch (error) {
|
||||
parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { getVocabularySummary } from './query-lexical';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas } from './storage';
|
||||
import type { VocabularyStatsSummary } from './types';
|
||||
|
||||
export function executeVocabularySummaryTask(
|
||||
dbPath: string,
|
||||
knownWords: string[] | null,
|
||||
): VocabularyStatsSummary {
|
||||
const db = new Database(dbPath);
|
||||
try {
|
||||
applyPragmas(db);
|
||||
return getVocabularySummary(db, knownWords ? new Set(knownWords) : null);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { PartOfSpeech, type MergedToken } from '../../../types';
|
||||
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
|
||||
|
||||
export interface VocabularyVisibilityRow {
|
||||
word: string | null;
|
||||
headword: string | null;
|
||||
reading?: string | null;
|
||||
partOfSpeech?: string | null;
|
||||
pos1?: string | null;
|
||||
pos2?: string | null;
|
||||
pos3?: string | null;
|
||||
frequencyRank?: number | null;
|
||||
}
|
||||
|
||||
function toVocabularyToken(row: VocabularyVisibilityRow): MergedToken {
|
||||
const word = row.word ?? '';
|
||||
const headword = row.headword ?? word;
|
||||
const partOfSpeech =
|
||||
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
|
||||
? (row.partOfSpeech as PartOfSpeech)
|
||||
: PartOfSpeech.other;
|
||||
|
||||
return {
|
||||
surface: word,
|
||||
reading: row.reading ?? '',
|
||||
headword,
|
||||
startPos: 0,
|
||||
endPos: word.length,
|
||||
partOfSpeech,
|
||||
pos1: row.pos1 ?? '',
|
||||
pos2: row.pos2 ?? '',
|
||||
pos3: row.pos3 ?? '',
|
||||
frequencyRank: row.frequencyRank ?? undefined,
|
||||
isMerged: false,
|
||||
isKnown: false,
|
||||
isNPlusOneTarget: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function isVocabularyStatsRowVisible(row: VocabularyVisibilityRow): boolean {
|
||||
if (!(row.word?.trim() || row.headword?.trim())) return false;
|
||||
return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
|
||||
}
|
||||
@@ -125,9 +125,83 @@ test('mineSentenceCard creates sentence card from mpv subtitle state', async ()
|
||||
]);
|
||||
});
|
||||
|
||||
test('mineSentenceCard refreshes secondary subtitle text before creating card', async () => {
|
||||
test('mineSentenceCard prefers a canonical primary subtitle snapshot', async () => {
|
||||
const created: Array<{
|
||||
sentence: string;
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
secondarySub?: string;
|
||||
}> = [];
|
||||
|
||||
await mineSentenceCard({
|
||||
ankiIntegration: {
|
||||
updateLastAddedFromClipboard: async () => {},
|
||||
triggerFieldGroupingForLastAddedCard: async () => {},
|
||||
markLastCardAsAudioCard: async () => {},
|
||||
createSentenceCard: async (sentence, startTime, endTime, secondarySub) => {
|
||||
created.push({ sentence, startTime, endTime, secondarySub });
|
||||
return true;
|
||||
},
|
||||
},
|
||||
mpvClient: {
|
||||
connected: true,
|
||||
currentSubText: '今今今手手手',
|
||||
currentSubStart: 11.4,
|
||||
currentSubEnd: 11.8,
|
||||
currentSecondarySubText: 'English subtitle',
|
||||
},
|
||||
primarySubtitle: {
|
||||
text: '今 手にある物差しでは',
|
||||
startTime: 11.13,
|
||||
endTime: 13.83,
|
||||
},
|
||||
showMpvOsd: () => {},
|
||||
});
|
||||
|
||||
assert.deepEqual(created, [
|
||||
{
|
||||
sentence: '今 手にある物差しでは',
|
||||
startTime: 11.13,
|
||||
endTime: 13.83,
|
||||
secondarySub: 'English subtitle',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('mineSentenceCard uses normalized secondary subtitle state instead of raw mpv text', async () => {
|
||||
const created: Array<{ sentence: string; secondarySub?: string }> = [];
|
||||
let requestedRawSecondaryText = false;
|
||||
|
||||
await mineSentenceCard({
|
||||
ankiIntegration: {
|
||||
updateLastAddedFromClipboard: async () => {},
|
||||
triggerFieldGroupingForLastAddedCard: async () => {},
|
||||
markLastCardAsAudioCard: async () => {},
|
||||
createSentenceCard: async (sentence, _startTime, _endTime, secondarySub) => {
|
||||
created.push({ sentence, secondarySub });
|
||||
return true;
|
||||
},
|
||||
},
|
||||
mpvClient: {
|
||||
connected: true,
|
||||
currentSubText: '日本語字幕',
|
||||
currentSubStart: 10,
|
||||
currentSubEnd: 12,
|
||||
currentSecondarySubText: 'Your\nmosaic',
|
||||
requestProperty: async () => {
|
||||
requestedRawSecondaryText = true;
|
||||
return 'Your\nYour\nYour\nYour\nmosaic';
|
||||
},
|
||||
},
|
||||
showMpvOsd: () => {},
|
||||
});
|
||||
|
||||
assert.equal(requestedRawSecondaryText, false);
|
||||
assert.deepEqual(created, [{ sentence: '日本語字幕', secondarySub: 'Your\nmosaic' }]);
|
||||
});
|
||||
|
||||
test('mineSentenceCard omits normalized secondary text that matches the primary subtitle', async () => {
|
||||
const created: Array<{ sentence: string; secondarySub?: string }> = [];
|
||||
const requestedProperties: string[] = [];
|
||||
|
||||
await mineSentenceCard({
|
||||
ankiIntegration: {
|
||||
@@ -145,43 +219,6 @@ test('mineSentenceCard refreshes secondary subtitle text before creating card',
|
||||
currentSubStart: 10,
|
||||
currentSubEnd: 12,
|
||||
currentSecondarySubText: '日本語字幕',
|
||||
requestProperty: async (name: string) => {
|
||||
requestedProperties.push(name);
|
||||
return name === 'secondary-sub-text' ? 'English subtitle' : null;
|
||||
},
|
||||
},
|
||||
showMpvOsd: () => {},
|
||||
});
|
||||
|
||||
assert.deepEqual(requestedProperties, ['secondary-sub-text']);
|
||||
assert.deepEqual(created, [{ sentence: '日本語字幕', secondarySub: 'English subtitle' }]);
|
||||
});
|
||||
|
||||
test('mineSentenceCard does not fall back to stale cached secondary subtitle after successful refresh', async () => {
|
||||
const created: Array<{ sentence: string; secondarySub?: string }> = [];
|
||||
|
||||
await mineSentenceCard({
|
||||
ankiIntegration: {
|
||||
updateLastAddedFromClipboard: async () => {},
|
||||
triggerFieldGroupingForLastAddedCard: async () => {},
|
||||
markLastCardAsAudioCard: async () => {},
|
||||
createSentenceCard: async (sentence, _startTime, _endTime, secondarySub) => {
|
||||
created.push({ sentence, secondarySub });
|
||||
return true;
|
||||
},
|
||||
},
|
||||
mpvClient: {
|
||||
connected: true,
|
||||
currentSubText: '日本語字幕',
|
||||
currentSubStart: 10,
|
||||
currentSubEnd: 12,
|
||||
currentSecondarySubText: 'stale cached subtitle',
|
||||
requestProperty: async (name: string) => {
|
||||
if (name === 'secondary-sub-text') {
|
||||
return '';
|
||||
}
|
||||
return null;
|
||||
},
|
||||
},
|
||||
showMpvOsd: () => {},
|
||||
});
|
||||
|
||||
+10
-16
@@ -129,18 +129,10 @@ function normalizeSecondarySubText(text: unknown, primaryText: string): string |
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
async function getCurrentSecondarySubTextForSentenceCard(
|
||||
function getCurrentSecondarySubTextForSentenceCard(
|
||||
mpvClient: MpvClientLike,
|
||||
): Promise<string | undefined> {
|
||||
const primaryText = mpvClient.currentSubText;
|
||||
if (mpvClient.requestProperty) {
|
||||
try {
|
||||
const latestSecondaryText = await mpvClient.requestProperty('secondary-sub-text');
|
||||
return normalizeSecondarySubText(latestSecondaryText, primaryText);
|
||||
} catch {
|
||||
// Fall back to the cached secondary subtitle below.
|
||||
}
|
||||
}
|
||||
primaryText: string,
|
||||
): string | undefined {
|
||||
return normalizeSecondarySubText(mpvClient.currentSecondarySubText, primaryText);
|
||||
}
|
||||
|
||||
@@ -175,6 +167,7 @@ export async function markLastCardAsAudioCard(deps: {
|
||||
export async function mineSentenceCard(deps: {
|
||||
ankiIntegration: AnkiIntegrationLike | null;
|
||||
mpvClient: MpvClientLike | null;
|
||||
primarySubtitle?: Pick<SubtitleMiningContext, 'text' | 'startTime' | 'endTime'>;
|
||||
showMpvOsd: (text: string) => void;
|
||||
}): Promise<boolean> {
|
||||
const anki = requireAnkiIntegration(deps.ankiIntegration, deps.showMpvOsd);
|
||||
@@ -185,16 +178,17 @@ export async function mineSentenceCard(deps: {
|
||||
deps.showMpvOsd('MPV not connected');
|
||||
return false;
|
||||
}
|
||||
if (!mpvClient.currentSubText) {
|
||||
const primaryText = deps.primarySubtitle?.text ?? mpvClient.currentSubText;
|
||||
if (!primaryText) {
|
||||
deps.showMpvOsd('No current subtitle');
|
||||
return false;
|
||||
}
|
||||
|
||||
const secondarySubText = await getCurrentSecondarySubTextForSentenceCard(mpvClient);
|
||||
const secondarySubText = getCurrentSecondarySubTextForSentenceCard(mpvClient, primaryText);
|
||||
return await anki.createSentenceCard(
|
||||
mpvClient.currentSubText,
|
||||
mpvClient.currentSubStart,
|
||||
mpvClient.currentSubEnd,
|
||||
primaryText,
|
||||
deps.primarySubtitle?.startTime ?? mpvClient.currentSubStart,
|
||||
deps.primarySubtitle?.endTime ?? mpvClient.currentSubEnd,
|
||||
secondarySubText,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -65,6 +65,8 @@ const MPV_SUBTITLE_PROPERTY_OBSERVATIONS: string[] = [
|
||||
'secondary-sub-visibility',
|
||||
'sub-visibility',
|
||||
'sid',
|
||||
'secondary-sid',
|
||||
'secondary-sub-delay',
|
||||
'track-list',
|
||||
];
|
||||
|
||||
|
||||
@@ -63,6 +63,8 @@ function createDeps(overrides: Partial<MpvProtocolHandleMessageDeps> = {}): {
|
||||
emitSubtitleTiming: (payload) => state.events.push(payload),
|
||||
emitSecondarySubtitleChange: (payload) => state.events.push(payload),
|
||||
emitSubtitleTrackChange: (payload) => state.events.push(payload),
|
||||
emitSecondarySubtitleTrackChange: (payload) => state.events.push(payload),
|
||||
emitSecondarySubtitleDelayChange: (payload) => state.events.push(payload),
|
||||
emitSubtitleTrackListChange: (payload) => state.events.push(payload),
|
||||
getCurrentSubText: () => state.subText,
|
||||
setCurrentSubText: (text) => {
|
||||
@@ -158,12 +160,42 @@ test('dispatchMpvProtocolMessage emits subtitle track changes', async () => {
|
||||
});
|
||||
|
||||
await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sid', data: '3' }, deps);
|
||||
await dispatchMpvProtocolMessage(
|
||||
{ event: 'property-change', name: 'secondary-sid', data: '4' },
|
||||
deps,
|
||||
);
|
||||
await dispatchMpvProtocolMessage(
|
||||
{ event: 'property-change', name: 'secondary-sub-delay', data: '0.5' },
|
||||
deps,
|
||||
);
|
||||
await dispatchMpvProtocolMessage(
|
||||
{ event: 'property-change', name: 'track-list', data: [{ type: 'sub', id: 3 }] },
|
||||
deps,
|
||||
);
|
||||
|
||||
assert.deepEqual(state.events, [{ sid: 3 }, { trackList: [{ type: 'sub', id: 3 }] }]);
|
||||
assert.deepEqual(state.events, [
|
||||
{ sid: 3 },
|
||||
{ sid: 4 },
|
||||
{ delay: 0.5 },
|
||||
{ trackList: [{ type: 'sub', id: 3 }] },
|
||||
]);
|
||||
});
|
||||
|
||||
test('dispatchMpvProtocolMessage rejects decimal subtitle track IDs', async () => {
|
||||
const { deps, state } = createDeps();
|
||||
|
||||
await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sid', data: '4.5' }, deps);
|
||||
await dispatchMpvProtocolMessage(
|
||||
{ event: 'property-change', name: 'secondary-sid', data: '4.5' },
|
||||
deps,
|
||||
);
|
||||
await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sid', data: 4.5 }, deps);
|
||||
await dispatchMpvProtocolMessage(
|
||||
{ event: 'property-change', name: 'secondary-sid', data: 4.5 },
|
||||
deps,
|
||||
);
|
||||
|
||||
assert.deepEqual(state.events, [{ sid: null }, { sid: null }, { sid: null }, { sid: null }]);
|
||||
});
|
||||
|
||||
test('dispatchMpvProtocolMessage enforces sub-visibility hidden when overlay suppression is enabled', async () => {
|
||||
|
||||
@@ -54,6 +54,8 @@ export interface MpvProtocolHandleMessageDeps {
|
||||
emitSubtitleTiming: (payload: { text: string; start: number; end: number }) => void;
|
||||
emitSecondarySubtitleChange: (payload: { text: string }) => void;
|
||||
emitSubtitleTrackChange: (payload: { sid: number | null }) => void;
|
||||
emitSecondarySubtitleTrackChange: (payload: { sid: number | null }) => void;
|
||||
emitSecondarySubtitleDelayChange: (payload: { delay: number }) => void;
|
||||
emitSubtitleTrackListChange: (payload: { trackList: unknown[] | null }) => void;
|
||||
getCurrentSubText: () => string;
|
||||
setCurrentSubText: (text: string) => void;
|
||||
@@ -281,7 +283,25 @@ export async function dispatchMpvProtocolMessage(
|
||||
: typeof msg.data === 'string'
|
||||
? Number(msg.data)
|
||||
: null;
|
||||
deps.emitSubtitleTrackChange({ sid: sid !== null && Number.isFinite(sid) ? sid : null });
|
||||
deps.emitSubtitleTrackChange({ sid: sid !== null && Number.isInteger(sid) ? sid : null });
|
||||
} else if (msg.name === 'secondary-sid') {
|
||||
const sid =
|
||||
typeof msg.data === 'number'
|
||||
? msg.data
|
||||
: typeof msg.data === 'string'
|
||||
? Number(msg.data)
|
||||
: null;
|
||||
deps.emitSecondarySubtitleTrackChange({
|
||||
sid: sid !== null && Number.isInteger(sid) ? sid : null,
|
||||
});
|
||||
} else if (msg.name === 'secondary-sub-delay') {
|
||||
const delay =
|
||||
typeof msg.data === 'number'
|
||||
? msg.data
|
||||
: typeof msg.data === 'string'
|
||||
? Number(msg.data)
|
||||
: 0;
|
||||
deps.emitSecondarySubtitleDelayChange({ delay: Number.isFinite(delay) ? delay : 0 });
|
||||
} else if (msg.name === 'track-list') {
|
||||
deps.emitSubtitleTrackListChange({
|
||||
trackList: Array.isArray(msg.data) ? (msg.data as unknown[]) : null,
|
||||
|
||||
@@ -38,7 +38,15 @@ class ManualCloseSocket extends FakeSocket {
|
||||
}
|
||||
}
|
||||
|
||||
const wait = () => new Promise((resolve) => setTimeout(resolve, 0));
|
||||
class HangingSocket extends FakeSocket {
|
||||
override connect(path: string): void {
|
||||
this.connectedPaths.push(path);
|
||||
// Never emits 'connect', 'error', or 'close' on its own: models a named
|
||||
// pipe dial that stalls indefinitely.
|
||||
}
|
||||
}
|
||||
|
||||
const wait = (ms = 0) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
test('getMpvReconnectDelay follows existing reconnect ramp', () => {
|
||||
assert.equal(getMpvReconnectDelay(0, true), 1000);
|
||||
@@ -232,6 +240,75 @@ test('MpvSocketTransport.shutdown clears socket and lifecycle flags', async () =
|
||||
assert.deepEqual(events, []);
|
||||
});
|
||||
|
||||
test('MpvSocketTransport aborts a hung connect after the timeout and allows a fresh dial', async () => {
|
||||
const events: string[] = [];
|
||||
const errors: Error[] = [];
|
||||
const sockets: HangingSocket[] = [];
|
||||
const transport = new MpvSocketTransport({
|
||||
socketPath: '/tmp/mpv.sock',
|
||||
connectTimeoutMs: 5,
|
||||
onConnect: () => {
|
||||
events.push('connect');
|
||||
},
|
||||
onData: () => {},
|
||||
onError: (error) => {
|
||||
events.push('error');
|
||||
errors.push(error);
|
||||
},
|
||||
onClose: () => {
|
||||
events.push('close');
|
||||
},
|
||||
socketFactory: () => {
|
||||
const socket = new HangingSocket();
|
||||
sockets.push(socket);
|
||||
return socket as unknown as net.Socket;
|
||||
},
|
||||
});
|
||||
|
||||
transport.connect();
|
||||
assert.equal(transport.isConnecting, true);
|
||||
|
||||
await wait(20);
|
||||
|
||||
assert.deepEqual(events, ['error', 'close']);
|
||||
assert.match(errors[0]!.message, /connect timed out/);
|
||||
assert.equal(sockets[0]!.destroyed, true);
|
||||
assert.equal(transport.isConnecting, false);
|
||||
assert.equal(transport.isConnected, false);
|
||||
|
||||
transport.connect();
|
||||
assert.equal(transport.isConnecting, true);
|
||||
assert.equal(sockets.length, 2);
|
||||
assert.equal(sockets[1]!.connectedPaths.at(0), '/tmp/mpv.sock');
|
||||
|
||||
transport.shutdown();
|
||||
});
|
||||
|
||||
test('MpvSocketTransport does not fire the connect timeout after a successful connect', async () => {
|
||||
const events: string[] = [];
|
||||
const transport = new MpvSocketTransport({
|
||||
socketPath: '/tmp/mpv.sock',
|
||||
connectTimeoutMs: 5,
|
||||
onConnect: () => {
|
||||
events.push('connect');
|
||||
},
|
||||
onData: () => {},
|
||||
onError: () => {
|
||||
events.push('error');
|
||||
},
|
||||
onClose: () => {
|
||||
events.push('close');
|
||||
},
|
||||
socketFactory: () => new FakeSocket() as unknown as net.Socket,
|
||||
});
|
||||
|
||||
transport.connect();
|
||||
await wait(20);
|
||||
|
||||
assert.deepEqual(events, ['connect']);
|
||||
assert.equal(transport.isConnected, true);
|
||||
});
|
||||
|
||||
test('MpvSocketTransport ignores stale socket events after shutdown and reconnect', async () => {
|
||||
const events: string[] = [];
|
||||
const sockets: ManualCloseSocket[] = [];
|
||||
|
||||
@@ -62,6 +62,8 @@ interface MpvSocketTransportEvents {
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export const MPV_CONNECT_TIMEOUT_MS = 5000;
|
||||
|
||||
export interface MpvSocketTransportOptions {
|
||||
socketPath: string;
|
||||
onConnect: () => void;
|
||||
@@ -69,13 +71,16 @@ export interface MpvSocketTransportOptions {
|
||||
onError: (error: Error) => void;
|
||||
onClose: () => void;
|
||||
socketFactory?: () => net.Socket;
|
||||
connectTimeoutMs?: number;
|
||||
}
|
||||
|
||||
export class MpvSocketTransport {
|
||||
private socketPath: string;
|
||||
private readonly callbacks: MpvSocketTransportEvents;
|
||||
private readonly socketFactory: () => net.Socket;
|
||||
private readonly connectTimeoutMs: number;
|
||||
private socketRef: net.Socket | null = null;
|
||||
private connectTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
public socket: net.Socket | null = null;
|
||||
public connected = false;
|
||||
public connecting = false;
|
||||
@@ -83,6 +88,7 @@ export class MpvSocketTransport {
|
||||
constructor(options: MpvSocketTransportOptions) {
|
||||
this.socketPath = options.socketPath;
|
||||
this.socketFactory = options.socketFactory ?? (() => new net.Socket());
|
||||
this.connectTimeoutMs = options.connectTimeoutMs ?? MPV_CONNECT_TIMEOUT_MS;
|
||||
this.callbacks = {
|
||||
onConnect: options.onConnect,
|
||||
onData: options.onData,
|
||||
@@ -91,6 +97,31 @@ export class MpvSocketTransport {
|
||||
};
|
||||
}
|
||||
|
||||
private clearConnectTimeout(): void {
|
||||
if (this.connectTimer) {
|
||||
clearTimeout(this.connectTimer);
|
||||
this.connectTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
// A named-pipe/socket dial that neither connects nor errors would otherwise
|
||||
// latch `connecting` forever and silently block every future connect().
|
||||
private armConnectTimeout(socket: net.Socket): void {
|
||||
this.clearConnectTimeout();
|
||||
this.connectTimer = setTimeout(() => {
|
||||
this.connectTimer = null;
|
||||
if (this.socketRef !== socket || this.connected) return;
|
||||
this.connecting = false;
|
||||
this.callbacks.onError(
|
||||
new Error(`MPV IPC connect timed out after ${this.connectTimeoutMs}ms: ${this.socketPath}`),
|
||||
);
|
||||
// Destroying the socket emits 'close', which drives the normal
|
||||
// disconnect path (including reconnect scheduling) upstream.
|
||||
socket.destroy();
|
||||
}, this.connectTimeoutMs);
|
||||
this.connectTimer.unref?.();
|
||||
}
|
||||
|
||||
setSocketPath(socketPath: string): void {
|
||||
this.socketPath = socketPath;
|
||||
}
|
||||
@@ -111,6 +142,7 @@ export class MpvSocketTransport {
|
||||
|
||||
socket.on('connect', () => {
|
||||
if (this.socketRef !== socket) return;
|
||||
this.clearConnectTimeout();
|
||||
this.connected = true;
|
||||
this.connecting = false;
|
||||
this.callbacks.onConnect();
|
||||
@@ -123,6 +155,7 @@ export class MpvSocketTransport {
|
||||
|
||||
socket.on('error', (error: Error) => {
|
||||
if (this.socketRef !== socket) return;
|
||||
this.clearConnectTimeout();
|
||||
this.connected = false;
|
||||
this.connecting = false;
|
||||
this.callbacks.onError(error);
|
||||
@@ -130,12 +163,14 @@ export class MpvSocketTransport {
|
||||
|
||||
socket.on('close', () => {
|
||||
if (this.socketRef !== socket) return;
|
||||
this.clearConnectTimeout();
|
||||
this.connected = false;
|
||||
this.connecting = false;
|
||||
this.callbacks.onClose();
|
||||
});
|
||||
|
||||
socket.connect(this.socketPath);
|
||||
this.armConnectTimeout(socket);
|
||||
}
|
||||
|
||||
send(payload: MpvSocketMessagePayload): boolean {
|
||||
@@ -149,6 +184,7 @@ export class MpvSocketTransport {
|
||||
}
|
||||
|
||||
shutdown(): void {
|
||||
this.clearConnectTimeout();
|
||||
const socket = this.socketRef;
|
||||
this.socketRef = null;
|
||||
this.socket = null;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import {
|
||||
MpvIpcClient,
|
||||
MpvIpcClientDeps,
|
||||
@@ -23,6 +24,18 @@ function makeDeps(overrides: Partial<MpvIpcClientProtocolDeps> = {}): MpvIpcClie
|
||||
};
|
||||
}
|
||||
|
||||
const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
async function waitFor(predicate: () => boolean, timeoutMs = 2000): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (!predicate()) {
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error('Timed out waiting for MPV retry connection');
|
||||
}
|
||||
await wait(10);
|
||||
}
|
||||
}
|
||||
|
||||
function captureWarnLogs(run: () => void): string[] {
|
||||
const originalWarn = console.warn;
|
||||
const originalLogLevel = process.env.SUBMINER_LOG_LEVEL;
|
||||
@@ -756,3 +769,117 @@ test('MpvIpcClient playNextSubtitle still auto-pauses at end while already playi
|
||||
assert.equal((client as any).pendingPauseAtSubEnd, true);
|
||||
assert.deepEqual(commands, [{ command: ['sub-seek', 1] }]);
|
||||
});
|
||||
|
||||
class HangingTestSocket extends EventEmitter {
|
||||
public connectedPaths: string[] = [];
|
||||
public destroyed = false;
|
||||
|
||||
connect(path: string): void {
|
||||
this.connectedPaths.push(path);
|
||||
// Never resolves: models a stalled named-pipe dial.
|
||||
}
|
||||
|
||||
write(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
this.destroyed = true;
|
||||
}
|
||||
}
|
||||
|
||||
class RetryTestSocket extends EventEmitter {
|
||||
public connectedPaths: string[] = [];
|
||||
public destroyed = false;
|
||||
|
||||
constructor(private readonly shouldConnect: boolean) {
|
||||
super();
|
||||
}
|
||||
|
||||
connect(path: string): void {
|
||||
this.connectedPaths.push(path);
|
||||
if (this.shouldConnect) {
|
||||
setTimeout(() => this.emit('connect'), 0);
|
||||
}
|
||||
}
|
||||
|
||||
write(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
this.emit('close');
|
||||
}
|
||||
}
|
||||
|
||||
test('MpvIpcClient automatically retries the same socket path after a connect timeout', async () => {
|
||||
const sockets: RetryTestSocket[] = [];
|
||||
let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
const originalLogLevel = process.env.SUBMINER_LOG_LEVEL;
|
||||
const client = new MpvIpcClient(
|
||||
'/tmp/mpv.sock',
|
||||
makeDeps({
|
||||
connectTimeoutMs: 5,
|
||||
getReconnectTimer: () => reconnectTimer,
|
||||
setReconnectTimer: (timer) => {
|
||||
reconnectTimer = timer;
|
||||
},
|
||||
socketFactory: () => {
|
||||
const socket = new RetryTestSocket(sockets.length > 0);
|
||||
sockets.push(socket);
|
||||
return socket as unknown as import('node:net').Socket;
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
process.env.SUBMINER_LOG_LEVEL = 'error';
|
||||
try {
|
||||
client.connect();
|
||||
await waitFor(() => client.connected);
|
||||
|
||||
assert.equal(sockets.length, 2);
|
||||
assert.equal(sockets[0]!.destroyed, true);
|
||||
assert.equal(sockets[0]!.connectedPaths.at(0), '/tmp/mpv.sock');
|
||||
assert.equal(sockets[1]!.connectedPaths.at(0), '/tmp/mpv.sock');
|
||||
assert.equal(client.connected, true);
|
||||
} finally {
|
||||
if (originalLogLevel === undefined) {
|
||||
delete process.env.SUBMINER_LOG_LEVEL;
|
||||
} else {
|
||||
process.env.SUBMINER_LOG_LEVEL = originalLogLevel;
|
||||
}
|
||||
if (reconnectTimer) clearTimeout(reconnectTimer);
|
||||
(client as any).transport.shutdown();
|
||||
}
|
||||
});
|
||||
|
||||
test('MpvIpcClient.setSocketPath aborts an in-flight connect so the next dial targets the new path', () => {
|
||||
const sockets: HangingTestSocket[] = [];
|
||||
const client = new MpvIpcClient(
|
||||
'/tmp/mpv-old.sock',
|
||||
makeDeps({
|
||||
socketFactory: () => {
|
||||
const socket = new HangingTestSocket();
|
||||
sockets.push(socket);
|
||||
return socket as unknown as import('node:net').Socket;
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
client.connect();
|
||||
assert.equal(sockets.length, 1);
|
||||
assert.equal(sockets[0]!.connectedPaths.at(0), '/tmp/mpv-old.sock');
|
||||
assert.equal((client as any).connecting, true);
|
||||
|
||||
client.setSocketPath('/tmp/mpv-new.sock');
|
||||
assert.equal((client as any).connecting, false);
|
||||
assert.equal(sockets[0]!.destroyed, true);
|
||||
|
||||
client.connect();
|
||||
assert.equal(sockets.length, 2);
|
||||
assert.equal(sockets[1]!.connectedPaths.at(0), '/tmp/mpv-new.sock');
|
||||
|
||||
(client as any).transport.shutdown();
|
||||
});
|
||||
|
||||
@@ -9,7 +9,11 @@ import {
|
||||
splitMpvMessagesFromBuffer,
|
||||
} from './mpv-protocol';
|
||||
import { requestMpvInitialState, subscribeToMpvProperties } from './mpv-properties';
|
||||
import { scheduleMpvReconnect, MpvSocketTransport } from './mpv-transport';
|
||||
import {
|
||||
scheduleMpvReconnect,
|
||||
MpvSocketTransport,
|
||||
MpvSocketTransportOptions,
|
||||
} from './mpv-transport';
|
||||
import { createLogger } from '../../logger';
|
||||
|
||||
const logger = createLogger('main:mpv');
|
||||
@@ -110,6 +114,8 @@ export interface MpvIpcClientProtocolDeps {
|
||||
shouldAutoLoadSecondarySubTrack?: (path: string) => boolean;
|
||||
shouldQuitOnMpvShutdown?: () => boolean;
|
||||
requestAppQuit?: () => void;
|
||||
socketFactory?: MpvSocketTransportOptions['socketFactory'];
|
||||
connectTimeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface MpvIpcClientDeps extends MpvIpcClientProtocolDeps {}
|
||||
@@ -125,6 +131,8 @@ export interface MpvIpcClientEventMap {
|
||||
'fullscreen-change': { fullscreen: boolean };
|
||||
'secondary-subtitle-change': { text: string };
|
||||
'subtitle-track-change': { sid: number | null };
|
||||
'secondary-subtitle-track-change': { sid: number | null };
|
||||
'secondary-subtitle-delay-change': { delay: number };
|
||||
'subtitle-track-list-change': { trackList: unknown[] | null };
|
||||
'media-path-change': { path: string };
|
||||
'media-title-change': { title: string | null };
|
||||
@@ -188,6 +196,8 @@ export class MpvIpcClient implements MpvClient {
|
||||
|
||||
this.transport = new MpvSocketTransport({
|
||||
socketPath,
|
||||
socketFactory: deps.socketFactory,
|
||||
connectTimeoutMs: deps.connectTimeoutMs,
|
||||
onConnect: () => {
|
||||
this.connected = true;
|
||||
this.connecting = false;
|
||||
@@ -289,6 +299,14 @@ export class MpvIpcClient implements MpvClient {
|
||||
previousSocketPath: this.socketPath,
|
||||
socketPath,
|
||||
});
|
||||
if (this.connecting && !this.connected) {
|
||||
// Abort the in-flight dial to the old path; otherwise the connecting
|
||||
// latch turns every later connect() into a no-op while we hang on a
|
||||
// stale socket.
|
||||
logger.debug('Aborting in-flight MPV IPC connect for socket path change.');
|
||||
this.transport.shutdown();
|
||||
this.connecting = false;
|
||||
}
|
||||
}
|
||||
this.socketPath = socketPath;
|
||||
this.transport.setSocketPath(socketPath);
|
||||
@@ -422,6 +440,12 @@ export class MpvIpcClient implements MpvClient {
|
||||
emitSubtitleTrackChange: (payload) => {
|
||||
this.emit('subtitle-track-change', payload);
|
||||
},
|
||||
emitSecondarySubtitleTrackChange: (payload) => {
|
||||
this.emit('secondary-subtitle-track-change', payload);
|
||||
},
|
||||
emitSecondarySubtitleDelayChange: (payload) => {
|
||||
this.emit('secondary-subtitle-delay-change', payload);
|
||||
},
|
||||
emitSubtitleTrackListChange: (payload) => {
|
||||
this.emit('subtitle-track-list-change', payload);
|
||||
},
|
||||
|
||||
@@ -39,6 +39,7 @@ test('non-macOS modal overlay remains a regular window', () => {
|
||||
});
|
||||
|
||||
assert.equal(options.type, undefined);
|
||||
assert.equal(options.roundedCorners, false);
|
||||
});
|
||||
|
||||
test('Linux visible overlay window allows compositor resize for mpv-sized placement', () => {
|
||||
|
||||
@@ -37,6 +37,7 @@ export function buildOverlayWindowOptions(
|
||||
paintWhenInitiallyHidden: true,
|
||||
backgroundColor: '#00000000',
|
||||
frame: false,
|
||||
...(platform === 'linux' ? { roundedCorners: false } : {}),
|
||||
alwaysOnTop: shouldStartAlwaysOnTop,
|
||||
skipTaskbar: true,
|
||||
resizable: shouldAllowCompositorResize,
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
parseExcludedWordsBody,
|
||||
parseIntQuery,
|
||||
parsePositiveIdList,
|
||||
loadKnownWordsSet,
|
||||
} from './route-support.js';
|
||||
|
||||
export function registerStatsLibraryRoutes(
|
||||
@@ -31,6 +32,17 @@ export function registerStatsLibraryRoutes(
|
||||
return c.json(statsJson('vocabulary', vocab));
|
||||
});
|
||||
|
||||
app.get('/api/stats/vocabulary/summary', async (c) => {
|
||||
const summary = await tracker.getVocabularySummary(
|
||||
loadKnownWordsSet(options?.knownWordCachePath),
|
||||
);
|
||||
return c.json(statsJson('vocabularySummary', summary));
|
||||
});
|
||||
|
||||
app.get('/api/stats/vocabulary/charts', async (c) => {
|
||||
return c.json(statsJson('vocabularyCharts', await tracker.getVocabularyChartData()));
|
||||
});
|
||||
|
||||
app.get('/api/stats/excluded-words', async (c) => {
|
||||
return c.json(statsJson('excludedWords', await tracker.getStatsExcludedWords()));
|
||||
});
|
||||
|
||||
@@ -89,6 +89,7 @@ const WORD_COPY_COLUMNS = [
|
||||
'last_seen',
|
||||
'frequency',
|
||||
'frequency_rank',
|
||||
'vocabulary_visible',
|
||||
] as const;
|
||||
|
||||
export function mergeAnime(
|
||||
|
||||
@@ -67,6 +67,7 @@ export function buildStatsWindowOptions(options: {
|
||||
width: options.bounds?.width ?? DEFAULT_STATS_WINDOW_WIDTH,
|
||||
height: options.bounds?.height ?? DEFAULT_STATS_WINDOW_HEIGHT,
|
||||
frame: false,
|
||||
...(platform === 'linux' ? { roundedCorners: false } : {}),
|
||||
transparent: false,
|
||||
alwaysOnTop: true,
|
||||
resizable: false,
|
||||
|
||||
@@ -57,6 +57,7 @@ test('buildStatsWindowOptions remains a regular window off macOS', () => {
|
||||
});
|
||||
|
||||
assert.equal(options.type, undefined);
|
||||
assert.equal(options.roundedCorners, false);
|
||||
});
|
||||
|
||||
test('stats panels present after document load on macOS', () => {
|
||||
|
||||
@@ -27,17 +27,188 @@ function cueKey(cue: SubtitleCue): string {
|
||||
|
||||
/**
|
||||
* Identical text over an identical span is redundant however it was authored -- most
|
||||
* often a layered ASS event stacking a shadow copy under the visible one.
|
||||
* often a layered ASS event stacking a shadow copy under the visible one. When one of
|
||||
* the duplicates is a recovered canonical cue, that copy survives: dropping it would
|
||||
* strip the `source` marker and animation envelope the live overlay substitutes on.
|
||||
*/
|
||||
function collapseExactDuplicates(cues: AnnotatedSubtitleCue[]): AnnotatedSubtitleCue[] {
|
||||
const seen = new Set<string>();
|
||||
return cues.filter((cue) => {
|
||||
const survivorByKey = new Map<string, AnnotatedSubtitleCue>();
|
||||
const keysInOrder: string[] = [];
|
||||
for (const cue of cues) {
|
||||
const key = cueKey(cue);
|
||||
if (seen.has(key)) {
|
||||
return false;
|
||||
const existing = survivorByKey.get(key);
|
||||
if (!existing) {
|
||||
survivorByKey.set(key, cue);
|
||||
keysInOrder.push(key);
|
||||
} else if (!existing.source && cue.source) {
|
||||
survivorByKey.set(key, cue);
|
||||
}
|
||||
seen.add(key);
|
||||
}
|
||||
return keysInOrder.map((key) => survivorByKey.get(key)!);
|
||||
}
|
||||
|
||||
const SPATIAL_ASS_OVERRIDE_COMMANDS = new Set([
|
||||
'a',
|
||||
'an',
|
||||
'clip',
|
||||
'iclip',
|
||||
'move',
|
||||
'org',
|
||||
'pbo',
|
||||
'pos',
|
||||
'q',
|
||||
]);
|
||||
|
||||
interface RepeatedPhaseRun {
|
||||
cues: AnnotatedSubtitleCue[];
|
||||
indices: number[];
|
||||
}
|
||||
|
||||
// A changing override signature alone is weak: two ordinary repeats restyled with
|
||||
// different colors look identical to a phase pair. Real phase redraws carry a styling
|
||||
// stack over a full lyric line, and they exist to move a color/highlight boundary
|
||||
// *within* the line -- so every event also has an override block after visible text
|
||||
// began. An ordinary restyled repeat carries only a leading block and stays separate.
|
||||
const MIN_PHASE_EVIDENCE_OVERRIDES = 2;
|
||||
const MIN_PHASE_TEXT_LENGTH = 4;
|
||||
|
||||
function hasMidLineOverrideBlock(rawText: string): boolean {
|
||||
let sawVisibleText = false;
|
||||
for (let i = 0; i < rawText.length; i += 1) {
|
||||
if (rawText[i] === '{') {
|
||||
const close = rawText.indexOf('}', i);
|
||||
if (close === -1) {
|
||||
// Unclosed brace renders as literal text; nothing after it is markup.
|
||||
return false;
|
||||
}
|
||||
if (sawVisibleText) {
|
||||
return true;
|
||||
}
|
||||
i = close;
|
||||
} else if (!/\s/.test(rawText[i]!)) {
|
||||
sawVisibleText = true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function assStyleKey(cue: AnnotatedSubtitleCue): string {
|
||||
return `${cue.style}\0${cue.name}\0${cue.layer}`;
|
||||
}
|
||||
|
||||
function spatialOverrideSignature(cue: AnnotatedSubtitleCue): string {
|
||||
return cue.overrides
|
||||
.filter((command) => SPATIAL_ASS_OVERRIDE_COMMANDS.has(command.name.toLowerCase()))
|
||||
.map((command) => `${command.name.toLowerCase()}(${command.args})`)
|
||||
.join('|');
|
||||
}
|
||||
|
||||
function hasStableSpatialOverrides(run: readonly AnnotatedSubtitleCue[]): boolean {
|
||||
const firstSignature = spatialOverrideSignature(run[0]!);
|
||||
return run.every((cue) => spatialOverrideSignature(cue) === firstSignature);
|
||||
}
|
||||
|
||||
function hasDirectPhaseEvidence(run: readonly AnnotatedSubtitleCue[]): boolean {
|
||||
// Phases redraw one authored line in place. Whatever the animation evidence, a run
|
||||
// whose spatial placement changes is separate authored occurrences -- two flush
|
||||
// same-text `\move` signs at different coordinates must never merge.
|
||||
if (!hasStableSpatialOverrides(run)) {
|
||||
return false;
|
||||
}
|
||||
if (run.every((cue) => hasAssTemporalOverride(cue.overrides))) {
|
||||
return true;
|
||||
}
|
||||
if (run.every((cue) => isAnimatedAssEffectKind(cue.effectKind))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const [first] = run;
|
||||
return (
|
||||
first!.text.replace(/\s+/gu, '').length >= MIN_PHASE_TEXT_LENGTH &&
|
||||
run.every(
|
||||
(cue) =>
|
||||
cue.overrides.length >= MIN_PHASE_EVIDENCE_OVERRIDES &&
|
||||
hasMidLineOverrideBlock(cue.rawText),
|
||||
) &&
|
||||
run.some((cue) => cue.overrideSignature !== first!.overrideSignature)
|
||||
);
|
||||
}
|
||||
|
||||
function collectRepeatedPhaseRuns(cues: AnnotatedSubtitleCue[]): RepeatedPhaseRun[] {
|
||||
const runs: RepeatedPhaseRun[] = [];
|
||||
let start = 0;
|
||||
|
||||
while (start < cues.length) {
|
||||
const first = cues[start]!;
|
||||
const styleKey = assStyleKey(first);
|
||||
let end = start;
|
||||
|
||||
while (end + 1 < cues.length) {
|
||||
const current = cues[end]!;
|
||||
const next = cues[end + 1]!;
|
||||
const isFlush =
|
||||
Math.abs(next.startTime - current.endTime) <= DUPLICATE_CUE_GAP_TOLERANCE_SECONDS;
|
||||
if (
|
||||
first.source === 'canonical-ass' ||
|
||||
next.source === 'canonical-ass' ||
|
||||
next.text !== first.text ||
|
||||
assStyleKey(next) !== styleKey ||
|
||||
!isFlush
|
||||
) {
|
||||
break;
|
||||
}
|
||||
end += 1;
|
||||
}
|
||||
|
||||
if (end > start) {
|
||||
const indices = Array.from({ length: end - start + 1 }, (_, offset) => start + offset);
|
||||
runs.push({
|
||||
cues: indices.map((index) => cues[index]!),
|
||||
indices,
|
||||
});
|
||||
}
|
||||
start = end + 1;
|
||||
}
|
||||
|
||||
return runs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Some karaoke scripts redraw one complete lyric for each color/highlight phase. These
|
||||
* events last far longer than animation frames, but are still one sidebar/history line.
|
||||
* The events must prove themselves through direct animation metadata or changing
|
||||
* non-spatial overrides. Plain repeated dialogue and separately positioned signs stay
|
||||
* intact.
|
||||
*/
|
||||
function collapseAnimatedStylePhases(cues: AnnotatedSubtitleCue[]): AnnotatedSubtitleCue[] {
|
||||
const runs = collectRepeatedPhaseRuns(cues);
|
||||
if (runs.length === 0) {
|
||||
return cues;
|
||||
}
|
||||
|
||||
const dropped = new Set<number>();
|
||||
const extendedEnd = new Map<number, number>();
|
||||
for (const run of runs) {
|
||||
if (!hasDirectPhaseEvidence(run.cues)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const [firstIndex, ...remainingIndices] = run.indices;
|
||||
for (const index of remainingIndices) {
|
||||
dropped.add(index);
|
||||
}
|
||||
extendedEnd.set(firstIndex!, Math.max(...run.cues.map((cue) => cue.endTime)));
|
||||
}
|
||||
|
||||
if (dropped.size === 0) {
|
||||
return cues;
|
||||
}
|
||||
return cues.flatMap((cue, index) => {
|
||||
if (dropped.has(index)) {
|
||||
return [];
|
||||
}
|
||||
const endTime = extendedEnd.get(index);
|
||||
return endTime !== undefined ? [{ ...cue, endTime }] : [cue];
|
||||
});
|
||||
}
|
||||
|
||||
@@ -176,5 +347,8 @@ export function mergeDuplicateCues(
|
||||
cues: AnnotatedSubtitleCue[],
|
||||
format: SubtitleSourceFormat,
|
||||
): AnnotatedSubtitleCue[] {
|
||||
return collapseAnimationBursts(collapseExactDuplicates(cues), format);
|
||||
const exactDeduplicated = collapseExactDuplicates(cues);
|
||||
const phaseDeduplicated =
|
||||
format === 'ass' ? collapseAnimatedStylePhases(exactDeduplicated) : exactDeduplicated;
|
||||
return collapseAnimationBursts(phaseDeduplicated, format);
|
||||
}
|
||||
|
||||
@@ -327,6 +327,122 @@ test('parseSubtitleCues collapses per-frame karaoke duplicates into one cue', ()
|
||||
assert.equal(cues[0]!.text, '過ぎ去ってしまう瞬間を');
|
||||
});
|
||||
|
||||
test('parseSubtitleCues collapses long full-line color phases', () => {
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Dialogue: 1,0:03:49.75,0:03:51.21,OPJP,,0,0,0,,{\\blur0.6\\c&H312D38&\\4c&HFFFFFF&}ちゃんと目を{\\4c&HD590FF&}合わせてよ',
|
||||
'Dialogue: 1,0:03:51.21,0:03:52.25,OPJP,,0,0,0,,{\\blur0.6\\4c&H312D38&\\c&HFFFFFF&}ちゃんと目を{\\4c&HD590FF&}合わせてよ',
|
||||
].join('\n');
|
||||
|
||||
assert.deepEqual(parseSubtitleCues(content, 'test.ass'), [
|
||||
{
|
||||
startTime: 229.75,
|
||||
endTime: 232.25,
|
||||
text: 'ちゃんと目を合わせてよ',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues keeps ordinary repeated dialogue separate', () => {
|
||||
// A single restyle tag on a repeated line is how ordinary dialogue gets decorated;
|
||||
// it is not phase evidence, whatever the line length.
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Dialogue: 1,0:00:01.00,0:00:02.00,OPJP,,0,0,0,,{\\c&H111111&}歌詞',
|
||||
'Dialogue: 1,0:00:02.00,0:00:03.00,OPJP,,0,0,0,,{\\c&H222222&}歌詞',
|
||||
'Dialogue: 1,0:00:04.00,0:00:05.00,OPJP,,0,0,0,,{\\c&H333333&}別の歌詞',
|
||||
'Dialogue: 1,0:00:05.00,0:00:06.00,OPJP,,0,0,0,,{\\c&H444444&}別の歌詞',
|
||||
'Dialogue: 8,0:00:07.00,0:00:08.00,Text - JP,,0,0,0,,えっ?',
|
||||
'Dialogue: 8,0:00:08.00,0:00:09.00,Text - JP,,0,0,0,,えっ?',
|
||||
].join('\n');
|
||||
|
||||
assert.deepEqual(parseSubtitleCues(content, 'test.ass'), [
|
||||
{ startTime: 1, endTime: 2, text: '歌詞' },
|
||||
{ startTime: 2, endTime: 3, text: '歌詞' },
|
||||
{ startTime: 4, endTime: 5, text: '別の歌詞' },
|
||||
{ startTime: 5, endTime: 6, text: '別の歌詞' },
|
||||
{ startTime: 7, endTime: 8, text: 'えっ?' },
|
||||
{ startTime: 8, endTime: 9, text: 'えっ?' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues keeps separately positioned temporal signs separate', () => {
|
||||
// Two flush signs with the same text but different \move paths are separate authored
|
||||
// occurrences, not phases of one redraw: temporal evidence alone must not merge them.
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Dialogue: 0,0:00:01.00,0:00:02.00,Sign,,0,0,0,,{\\move(100,100,200,100)}立入禁止',
|
||||
'Dialogue: 0,0:00:02.00,0:00:03.00,Sign,,0,0,0,,{\\move(500,400,600,400)}立入禁止',
|
||||
].join('\n');
|
||||
|
||||
assert.deepEqual(parseSubtitleCues(content, 'test.ass'), [
|
||||
{ startTime: 1, endTime: 2, text: '立入禁止' },
|
||||
{ startTime: 2, endTime: 3, text: '立入禁止' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues keeps richly styled ordinary repeats separate', () => {
|
||||
// Blur plus a changing color is still an ordinary restyle. Phase redraws are
|
||||
// recognized by the color/highlight boundary moving *within* the line, which these
|
||||
// leading-block-only events do not have.
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Dialogue: 0,0:00:01.00,0:00:02.00,Dial,,0,0,0,,{\\blur0.4\\c&H111111&}待ってよ',
|
||||
'Dialogue: 0,0:00:02.00,0:00:03.00,Dial,,0,0,0,,{\\blur0.4\\c&H222222&}待ってよ',
|
||||
].join('\n');
|
||||
|
||||
assert.deepEqual(parseSubtitleCues(content, 'test.ass'), [
|
||||
{ startTime: 1, endTime: 2, text: '待ってよ' },
|
||||
{ startTime: 2, endTime: 3, text: '待ってよ' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues keeps canonical metadata when an identical plain cue exists', () => {
|
||||
// A plain dialogue line can share exact timing and text with a recovered canonical
|
||||
// cue from another style. The canonical copy must win the exact-duplicate collapse,
|
||||
// or the live overlay loses the marker it substitutes on.
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Dialogue: 0,0:00:05.00,0:00:08.00,Plain,,0,0,0,,ライン',
|
||||
'Comment: 0,0:00:05.00,0:00:08.00,OP,,0,0,0,,ライン',
|
||||
'Dialogue: 0,0:00:05.00,0:00:05.04,OP,,0,0,0,,{\\pos(1,1)\\clip(m 1 1)}ライン',
|
||||
'Dialogue: 0,0:00:05.04,0:00:05.08,OP,,0,0,0,,{\\pos(1,1)\\clip(m 2 2)}ライン',
|
||||
'Dialogue: 0,0:00:05.08,0:00:08.00,OP,,0,0,0,,{\\pos(1,1)\\clip(m 3 3)}ライン',
|
||||
].join('\n');
|
||||
|
||||
assert.deepEqual(parseSubtitleCues(content, 'test.ass'), [
|
||||
{
|
||||
startTime: 5,
|
||||
endTime: 8,
|
||||
text: 'ライン',
|
||||
source: 'canonical-ass',
|
||||
animationStartTime: 5,
|
||||
animationEndTime: 8,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues keeps short styled repeats separate even with richer styling', () => {
|
||||
// Two ordinary えっ lines restyled with different colors are two utterances, not two
|
||||
// phases of one lyric: short text never satisfies the changing-override evidence path.
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Dialogue: 0,0:00:01.00,0:00:02.00,Dial,,0,0,0,,{\\blur0.4\\c&H111111&}えっ',
|
||||
'Dialogue: 0,0:00:02.00,0:00:03.00,Dial,,0,0,0,,{\\blur0.4\\c&H222222&}えっ',
|
||||
].join('\n');
|
||||
|
||||
assert.deepEqual(parseSubtitleCues(content, 'test.ass'), [
|
||||
{ startTime: 1, endTime: 2, text: 'えっ' },
|
||||
{ startTime: 2, endTime: 3, text: 'えっ' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues keeps back-to-back plain dialogue repeats separate', () => {
|
||||
// Several characters greeting in turn: distinct utterances that happen to abut.
|
||||
const content = [
|
||||
@@ -357,6 +473,194 @@ test('parseSubtitleCues collapses exact duplicate cues even without effect tags'
|
||||
assert.equal(cues.length, 1);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues replaces generated glyph animation with its timed canonical comment', () => {
|
||||
// Aegisub automation commonly keeps the authored lyric as a Comment and emits
|
||||
// multiple moving Dialogue layers for every glyph. This mirrors the MyGO ED script:
|
||||
// three entrance copies followed by three exit copies for each character.
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Comment: 0,0:00:01.20,0:00:03.80,ED_JP,,0,0,0,,{\\fad(480,480)}今 手にある',
|
||||
'Dialogue: 0,0:00:00.80,0:00:01.50,ED_JP,,0,0,0,,{\\move(10,20,100,200)\\t(0,600,\\fscx100)}今',
|
||||
'Dialogue: 0,0:00:00.80,0:00:01.50,ED_JP,,0,0,0,,{\\move(30,40,100,200)\\t(0,600,\\fscx100)}今',
|
||||
'Dialogue: 0,0:00:00.80,0:00:01.50,ED_JP,,0,0,0,,{\\move(50,60,100,200)\\t(0,600,\\fscx100)}今',
|
||||
'Dialogue: 1,0:00:01.40,0:00:04.20,ED_JP,,0,0,0,,{\\move(100,200,20,30)\\t(2000,2600,\\blur20)}今',
|
||||
'Dialogue: 1,0:00:01.40,0:00:04.20,ED_JP,,0,0,0,,{\\move(100,200,40,50)\\t(2000,2600,\\blur20)}今',
|
||||
'Dialogue: 1,0:00:01.40,0:00:04.20,ED_JP,,0,0,0,,{\\move(100,200,60,70)\\t(2000,2600,\\blur20)}今',
|
||||
'Dialogue: 0,0:00:00.86,0:00:01.56,ED_JP,,0,0,0,,{\\move(10,20,140,200)\\t(0,600,\\fscx100)}手',
|
||||
'Dialogue: 0,0:00:00.86,0:00:01.56,ED_JP,,0,0,0,,{\\move(30,40,140,200)\\t(0,600,\\fscx100)}手',
|
||||
'Dialogue: 0,0:00:00.86,0:00:01.56,ED_JP,,0,0,0,,{\\move(50,60,140,200)\\t(0,600,\\fscx100)}手',
|
||||
'Dialogue: 1,0:00:01.46,0:00:04.26,ED_JP,,0,0,0,,{\\move(140,200,20,30)\\t(2000,2600,\\blur20)}手',
|
||||
'Dialogue: 1,0:00:01.46,0:00:04.26,ED_JP,,0,0,0,,{\\move(140,200,40,50)\\t(2000,2600,\\blur20)}手',
|
||||
'Dialogue: 1,0:00:01.46,0:00:04.26,ED_JP,,0,0,0,,{\\move(140,200,60,70)\\t(2000,2600,\\blur20)}手',
|
||||
'Dialogue: 0,0:00:00.92,0:00:01.62,ED_JP,,0,0,0,,{\\move(10,20,180,200)\\t(0,600,\\fscx100)}にある',
|
||||
'Dialogue: 0,0:00:00.92,0:00:01.62,ED_JP,,0,0,0,,{\\move(30,40,180,200)\\t(0,600,\\fscx100)}にある',
|
||||
'Dialogue: 0,0:00:00.92,0:00:01.62,ED_JP,,0,0,0,,{\\move(50,60,180,200)\\t(0,600,\\fscx100)}にある',
|
||||
'Dialogue: 1,0:00:01.52,0:00:04.32,ED_JP,,0,0,0,,{\\move(180,200,20,30)\\t(2000,2600,\\blur20)}にある',
|
||||
'Dialogue: 1,0:00:01.52,0:00:04.32,ED_JP,,0,0,0,,{\\move(180,200,40,50)\\t(2000,2600,\\blur20)}にある',
|
||||
'Dialogue: 1,0:00:01.52,0:00:04.32,ED_JP,,0,0,0,,{\\move(180,200,60,70)\\t(2000,2600,\\blur20)}にある',
|
||||
'Dialogue: 0,0:00:06.00,0:00:08.00,Dial_JP,,0,0,0,,普通の会話',
|
||||
].join('\n');
|
||||
|
||||
const cues = parseSubtitleCues(content, 'test.ass');
|
||||
|
||||
assert.deepEqual(cues, [
|
||||
{
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
text: '今 手にある',
|
||||
source: 'canonical-ass',
|
||||
// Entrance frames start before and exit frames end after the authored timing.
|
||||
animationStartTime: 0.8,
|
||||
animationEndTime: 4.32,
|
||||
},
|
||||
{ startTime: 6, endTime: 8, text: '普通の会話' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues recovers a full Dialogue line surrounding generated fragments', () => {
|
||||
// Some scripts do not retain the authored line as a Comment. Instead, brief entrance
|
||||
// and exit events contain the complete line around a long run of generated syllables.
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Dialogue: 1,0:00:01.00,0:00:01.15,ED Romaji,,0,0,0,fx,{\\move(100,40,60,40)}toki yo ugokidase',
|
||||
'Dialogue: 1,0:00:01.15,0:00:03.00,ED Romaji,,0,0,0,fx,{\\t(0,300,\\c&HFFFFFF&)}to',
|
||||
'Dialogue: 1,0:00:01.15,0:00:03.00,ED Romaji,,0,0,0,fx,{\\t(300,500,\\c&HFFFFFF&)}ki',
|
||||
'Dialogue: 1,0:00:01.15,0:00:03.00,ED Romaji,,0,0,0,fx,{\\t(500,700,\\c&HFFFFFF&)}yo',
|
||||
'Dialogue: 1,0:00:01.15,0:00:03.00,ED Romaji,,0,0,0,fx,{\\t(700,900,\\c&HFFFFFF&)}u',
|
||||
'Dialogue: 1,0:00:01.15,0:00:03.00,ED Romaji,,0,0,0,fx,{\\t(900,1100,\\c&HFFFFFF&)}go',
|
||||
'Dialogue: 1,0:00:01.15,0:00:03.00,ED Romaji,,0,0,0,fx,{\\t(1100,1300,\\c&HFFFFFF&)}ki',
|
||||
'Dialogue: 1,0:00:01.15,0:00:03.00,ED Romaji,,0,0,0,fx,{\\t(1300,1500,\\c&HFFFFFF&)}da',
|
||||
'Dialogue: 1,0:00:01.15,0:00:03.00,ED Romaji,,0,0,0,fx,{\\t(1500,1800,\\c&HFFFFFF&)}se',
|
||||
'Dialogue: 1,0:00:03.00,0:00:03.15,ED Romaji,,0,0,0,fx,{\\move(60,40,20,40)}toki yo ugokidase',
|
||||
'Dialogue: 0,0:00:06.00,0:00:08.00,Default,,0,0,0,,Ordinary dialogue',
|
||||
].join('\n');
|
||||
|
||||
assert.deepEqual(parseSubtitleCues(content, 'test.ass'), [
|
||||
{
|
||||
startTime: 1,
|
||||
endTime: 3.15,
|
||||
text: 'toki yo ugokidase',
|
||||
source: 'canonical-ass',
|
||||
animationStartTime: 1,
|
||||
animationEndTime: 3.15,
|
||||
},
|
||||
{ startTime: 6, endTime: 8, text: 'Ordinary dialogue' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues does not promote a short animated fragment as a complete line', () => {
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Dialogue: 1,0:00:01.00,0:00:03.00,OP English,,0,0,0,,{\\pos(100,100)\\t(0,100,\\fscx120)}my',
|
||||
'Dialogue: 2,0:00:01.00,0:00:03.00,OP English,,0,0,0,,{\\pos(100,100)\\t(0,100,\\fscx120)}my',
|
||||
'Dialogue: 1,0:00:01.00,0:00:03.00,OP English,,0,0,0,,{\\pos(100,100)\\t(0,100,\\fscx120)}m',
|
||||
'Dialogue: 2,0:00:01.00,0:00:03.00,OP English,,0,0,0,,{\\pos(100,100)\\t(0,100,\\fscx120)}m',
|
||||
'Dialogue: 1,0:00:01.00,0:00:03.00,OP English,,0,0,0,,{\\pos(120,100)\\t(20,120,\\fscx120)}y',
|
||||
'Dialogue: 2,0:00:01.00,0:00:03.00,OP English,,0,0,0,,{\\pos(120,100)\\t(20,120,\\fscx120)}y',
|
||||
].join('\n');
|
||||
|
||||
const cues = parseSubtitleCues(content, 'test.ass');
|
||||
|
||||
assert.equal(
|
||||
cues.some((cue) => cue.source === 'canonical-ass'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues ignores timed comments without a matching animated dialogue cluster', () => {
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Comment: 0,0:00:01.00,0:00:03.00,Dial_JP,,0,0,0,,編集メモ',
|
||||
'Comment: 0,0:00:04.00,0:00:06.00,Dial_JP,,0,0,0,,別案の字幕',
|
||||
'Dialogue: 0,0:00:01.00,0:00:03.00,Dial_JP,,0,0,0,,通常の字幕',
|
||||
'Dialogue: 0,0:00:04.00,0:00:06.00,Dial_JP,,0,0,0,,別案の字幕',
|
||||
].join('\n');
|
||||
|
||||
const cues = parseSubtitleCues(content, 'test.ass');
|
||||
|
||||
assert.deepEqual(cues, [
|
||||
{ startTime: 1, endTime: 3, text: '通常の字幕' },
|
||||
{ startTime: 4, endTime: 6, text: '別案の字幕' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('parseAssCues returns recovered canonical cues in chronological order', () => {
|
||||
// Recovery appends recovered cues after surviving dialogue; the bare parseAssCues
|
||||
// export must still come back time-ordered.
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Dialogue: 0,0:00:06.00,0:00:08.00,Dial,,0,0,0,,あとのセリフ',
|
||||
'Comment: 0,0:00:01.20,0:00:03.80,OP,,0,0,0,,雨が上がっても',
|
||||
'Dialogue: 0,0:00:01.20,0:00:01.24,OP,,0,0,0,,{\\pos(1,1)\\clip(m 1 1)}雨が上がっても',
|
||||
'Dialogue: 0,0:00:01.24,0:00:01.28,OP,,0,0,0,,{\\pos(1,1)\\clip(m 2 2)}雨が上がっても',
|
||||
'Dialogue: 0,0:00:01.28,0:00:03.80,OP,,0,0,0,,{\\pos(1,1)\\clip(m 3 3)}雨が上がっても',
|
||||
].join('\n');
|
||||
|
||||
assert.deepEqual(
|
||||
parseAssCues(content).map((cue) => cue.startTime),
|
||||
[1.2, 6],
|
||||
);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues withdraws a recovery whose owner is claimed by a later candidate', () => {
|
||||
// The exit boundary event appears first in the file and recovers a canonical cue from
|
||||
// its own small cluster. The entrance candidate then proves that exit event was a
|
||||
// generated frame of the full animation; the earlier recovery is a duplicate of the
|
||||
// same authored line and must not survive alongside it.
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Dialogue: 1,0:00:14.00,0:00:14.20,ED,,0,0,0,,{\\move(100,200,20,30)}ABCDEFGH',
|
||||
'Dialogue: 1,0:00:13.50,0:00:14.50,ED,,0,0,0,,{\\t(0,300,\\c&HFFFFFF&)}ABC',
|
||||
'Dialogue: 1,0:00:13.50,0:00:14.50,ED,,0,0,0,,{\\t(300,600,\\c&HFFFFFF&)}DEF',
|
||||
'Dialogue: 1,0:00:13.50,0:00:14.50,ED,,0,0,0,,{\\t(600,900,\\c&HFFFFFF&)}GH',
|
||||
'Dialogue: 0,0:00:10.00,0:00:10.20,ED,,0,0,0,,{\\move(10,20,100,200)}ABCDEFGH',
|
||||
'Dialogue: 0,0:00:10.00,0:00:12.00,ED,,0,0,0,,{\\t(0,300,\\fscx100)}ABC',
|
||||
'Dialogue: 0,0:00:10.00,0:00:12.00,ED,,0,0,0,,{\\t(300,600,\\fscx100)}DEF',
|
||||
'Dialogue: 0,0:00:10.00,0:00:13.40,ED,,0,0,0,,{\\t(600,900,\\fscx100)}GH',
|
||||
].join('\n');
|
||||
|
||||
assert.deepEqual(parseSubtitleCues(content, 'test.ass'), [
|
||||
{
|
||||
startTime: 10,
|
||||
endTime: 14.2,
|
||||
text: 'ABCDEFGH',
|
||||
source: 'canonical-ass',
|
||||
animationStartTime: 10,
|
||||
animationEndTime: 14.2,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues recovers canonical comments from generated clip frames', () => {
|
||||
const content = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Comment: 0,0:00:01.00,0:00:03.00,OP_JP,,0,0,0,,雨が上がっても',
|
||||
'Dialogue: 0,0:00:01.00,0:00:01.04,OP_JP,,0,0,0,,{\\pos(960,1068)\\clip(m 1 1)}雨が上がっても',
|
||||
'Dialogue: 0,0:00:01.04,0:00:01.08,OP_JP,,0,0,0,,{\\pos(960,1068)\\clip(m 2 2)}雨が上がっても',
|
||||
'Dialogue: 0,0:00:01.08,0:00:03.00,OP_JP,,0,0,0,,{\\pos(960,1068)\\clip(m 3 3)}雨が上がっても',
|
||||
].join('\n');
|
||||
|
||||
const cues = parseSubtitleCues(content, 'test.ass');
|
||||
|
||||
assert.deepEqual(cues, [
|
||||
{
|
||||
startTime: 1,
|
||||
endTime: 3,
|
||||
text: '雨が上がっても',
|
||||
source: 'canonical-ass',
|
||||
animationStartTime: 1,
|
||||
animationEndTime: 3,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('parseSubtitleCues collapses tag-less animation frames in converted SRT', () => {
|
||||
// ASS -> SRT conversion drops override tags, so only the ~0.04s frame timing remains.
|
||||
const lines = ['1', '00:00:07,870 --> 00:00:07,910', 'Kaguya Wants to be Confessed to', ''];
|
||||
|
||||
@@ -6,12 +6,21 @@ import {
|
||||
type AssEffectKind,
|
||||
type AssOverrideCommand,
|
||||
} from './ass-text';
|
||||
import { mergeDuplicateCues } from './subtitle-cue-dedup';
|
||||
import { hasAssAnimationEvidence, mergeDuplicateCues } from './subtitle-cue-dedup';
|
||||
|
||||
export interface SubtitleCue {
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
text: string;
|
||||
/** A complete authored line recovered from matching generated ASS animation events. */
|
||||
source?: 'canonical-ass';
|
||||
/**
|
||||
* Full span of the generated animation events a canonical cue replaced. Entrance and
|
||||
* exit frames routinely run past the authored `startTime`/`endTime`, so live-text
|
||||
* matching must use this envelope while display and history keep the authored timing.
|
||||
*/
|
||||
animationStartTime?: number;
|
||||
animationEndTime?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -19,7 +28,8 @@ export interface SubtitleCue {
|
||||
* Deduplication needs the authoring context -- which style the line belongs to, which
|
||||
* override commands it carries, whether the `Effect` column was set -- to tell a karaoke
|
||||
* burst apart from two characters saying the same word in turn. None of it is meaningful
|
||||
* outside the parser, so the public API stays `{startTime, endTime, text}`.
|
||||
* outside the parser, so the public API exposes only timing, text, and the optional
|
||||
* canonical-source marker used by live subtitle consumers.
|
||||
*/
|
||||
export interface AnnotatedSubtitleCue extends SubtitleCue {
|
||||
/** Text exactly as authored, override blocks and all. */
|
||||
@@ -70,7 +80,11 @@ function sanitizeSubtitleCueText(text: string): string {
|
||||
}
|
||||
|
||||
function toPublicCues(cues: AnnotatedSubtitleCue[]): SubtitleCue[] {
|
||||
return cues.map(({ startTime, endTime, text }) => ({ startTime, endTime, text }));
|
||||
return cues.map(({ startTime, endTime, text, source, animationStartTime, animationEndTime }) =>
|
||||
source
|
||||
? { startTime, endTime, text, source, animationStartTime, animationEndTime }
|
||||
: { startTime, endTime, text },
|
||||
);
|
||||
}
|
||||
|
||||
function parseAnnotatedSrtCues(content: string): AnnotatedSubtitleCue[] {
|
||||
@@ -138,7 +152,13 @@ export function parseSrtCues(content: string): SubtitleCue[] {
|
||||
const ASS_TIMING_PATTERN = /^(\d+):(\d{2}):(\d{2})\.(\d{1,2})$/;
|
||||
const ASS_FORMAT_PREFIX = 'Format:';
|
||||
const ASS_DIALOGUE_PREFIX = 'Dialogue:';
|
||||
const ASS_COMMENT_PREFIX = 'Comment:';
|
||||
const ASS_NAME_FIELD_ALIASES = ['name', 'actor'];
|
||||
const CANONICAL_MATCH_MARGIN_SECONDS = 1;
|
||||
const MIN_CANONICAL_ANIMATION_EVENTS = 3;
|
||||
// A tiny animated fragment can itself be composed from still smaller glyph events. It is
|
||||
// not enough evidence that the fragment represents an authored line boundary.
|
||||
const MIN_CANONICAL_DIALOGUE_TEXT_LENGTH = 4;
|
||||
|
||||
function parseAssTimestamp(raw: string): number | null {
|
||||
const match = ASS_TIMING_PATTERN.exec(raw.trim());
|
||||
@@ -166,10 +186,333 @@ function findFieldIndex(formatFields: string[], aliases: string[]): number {
|
||||
return -1;
|
||||
}
|
||||
|
||||
function parseAnnotatedAssCues(content: string): AnnotatedSubtitleCue[] {
|
||||
interface ParsedAssEvents {
|
||||
dialogue: AnnotatedSubtitleCue[];
|
||||
comments: AnnotatedSubtitleCue[];
|
||||
}
|
||||
|
||||
// Every candidate line re-reads the compacted text of each event in its window, so on
|
||||
// fragment-heavy scripts the same event compacts thousands of times without this cache.
|
||||
const compactMatchTextCache = new WeakMap<AnnotatedSubtitleCue, string>();
|
||||
|
||||
function compactAssMatchText(text: string): string {
|
||||
return text.replace(/\s+/gu, '');
|
||||
}
|
||||
|
||||
function compactCueMatchText(cue: AnnotatedSubtitleCue): string {
|
||||
let compact = compactMatchTextCache.get(cue);
|
||||
if (compact === undefined) {
|
||||
compact = compactAssMatchText(cue.text);
|
||||
compactMatchTextCache.set(cue, compact);
|
||||
}
|
||||
return compact;
|
||||
}
|
||||
|
||||
function assEventGroupKey(cue: AnnotatedSubtitleCue): string {
|
||||
return `${cue.style}\0${cue.name}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Windowed lookup over one style/name group. Every candidate line queries its time
|
||||
* neighborhood, and fragment-heavy scripts put thousands of candidates in one group, so
|
||||
* a linear rescan per candidate is quadratic in practice. Events are sorted by start
|
||||
* once; `prefixMaxEnd` lets the backward walk stop as soon as no earlier event can still
|
||||
* reach the window.
|
||||
*/
|
||||
interface AssEventGroupIndex {
|
||||
byStart: AnnotatedSubtitleCue[];
|
||||
prefixMaxEnd: number[];
|
||||
}
|
||||
|
||||
function buildAssEventGroupIndex(events: readonly AnnotatedSubtitleCue[]): AssEventGroupIndex {
|
||||
const byStart = [...events].sort((a, b) => a.startTime - b.startTime || a.order - b.order);
|
||||
const prefixMaxEnd: number[] = [];
|
||||
let maxEnd = -Infinity;
|
||||
for (const event of byStart) {
|
||||
maxEnd = Math.max(maxEnd, event.endTime);
|
||||
prefixMaxEnd.push(maxEnd);
|
||||
}
|
||||
return { byStart, prefixMaxEnd };
|
||||
}
|
||||
|
||||
/** Group events overlapping `[startTime, endTime]`, returned in source order. */
|
||||
function eventsOverlappingWindow(
|
||||
index: AssEventGroupIndex,
|
||||
startTime: number,
|
||||
endTime: number,
|
||||
): AnnotatedSubtitleCue[] {
|
||||
const { byStart, prefixMaxEnd } = index;
|
||||
let low = 0;
|
||||
let high = byStart.length;
|
||||
while (low < high) {
|
||||
const mid = (low + high) >>> 1;
|
||||
if (byStart[mid]!.startTime <= endTime) {
|
||||
low = mid + 1;
|
||||
} else {
|
||||
high = mid;
|
||||
}
|
||||
}
|
||||
const matches: AnnotatedSubtitleCue[] = [];
|
||||
for (let i = low - 1; i >= 0 && prefixMaxEnd[i]! >= startTime; i -= 1) {
|
||||
if (byStart[i]!.endTime >= startTime) {
|
||||
matches.push(byStart[i]!);
|
||||
}
|
||||
}
|
||||
return matches.sort((a, b) => a.order - b.order);
|
||||
}
|
||||
|
||||
interface FragmentGroup {
|
||||
text: string;
|
||||
events: AnnotatedSubtitleCue[];
|
||||
}
|
||||
|
||||
function fragmentPlacementAnchors(event: AnnotatedSubtitleCue): Set<string> {
|
||||
const anchors = new Set<string>();
|
||||
for (const command of event.overrides) {
|
||||
const name = command.name.toLowerCase();
|
||||
const args = command.args.split(',').map((value) => value.trim());
|
||||
if (name === 'pos' && args.length >= 2) {
|
||||
anchors.add(`pos:${args[0]},${args[1]}`);
|
||||
} else if (name === 'move' && args.length >= 4) {
|
||||
anchors.add(`move:${args[0]},${args[1]}`);
|
||||
anchors.add(`move:${args[2]},${args[3]}`);
|
||||
}
|
||||
}
|
||||
return anchors;
|
||||
}
|
||||
|
||||
function isRepeatedFragmentCopy(
|
||||
previous: AnnotatedSubtitleCue,
|
||||
current: AnnotatedSubtitleCue,
|
||||
): boolean {
|
||||
const previousAnchors = fragmentPlacementAnchors(previous);
|
||||
if ([...fragmentPlacementAnchors(current)].some((anchor) => previousAnchors.has(anchor))) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
previous.startTime === current.startTime &&
|
||||
previous.endTime === current.endTime &&
|
||||
previous.overrideSignature === current.overrideSignature
|
||||
);
|
||||
}
|
||||
|
||||
function groupConsecutiveAssFragments(events: readonly AnnotatedSubtitleCue[]): FragmentGroup[] {
|
||||
const groups: FragmentGroup[] = [];
|
||||
for (const event of events) {
|
||||
const text = compactCueMatchText(event);
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
const previous = groups.at(-1);
|
||||
if (
|
||||
previous?.text === text &&
|
||||
previous.events.some((previousEvent) => isRepeatedFragmentCopy(previousEvent, event))
|
||||
) {
|
||||
previous.events.push(event);
|
||||
} else {
|
||||
groups.push({ text, events: [event] });
|
||||
}
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
function findCanonicalFragmentEvents(
|
||||
events: readonly AnnotatedSubtitleCue[],
|
||||
canonicalText: string,
|
||||
): AnnotatedSubtitleCue[] {
|
||||
const groups = groupConsecutiveAssFragments(events);
|
||||
const matches = new Set<AnnotatedSubtitleCue>();
|
||||
|
||||
for (let start = 0; start < groups.length; start += 1) {
|
||||
let combined = '';
|
||||
for (let end = start; end < groups.length; end += 1) {
|
||||
const group = groups[end]!;
|
||||
// A complete rendered copy cannot prove that the neighboring events are its
|
||||
// fragments. Exact full-line animation is handled separately for comments.
|
||||
if (group.text.length >= canonicalText.length) {
|
||||
break;
|
||||
}
|
||||
const next = combined + group.text;
|
||||
if (!canonicalText.startsWith(next)) {
|
||||
break;
|
||||
}
|
||||
combined = next;
|
||||
if (combined !== canonicalText) {
|
||||
continue;
|
||||
}
|
||||
for (let index = start; index <= end; index += 1) {
|
||||
for (const event of groups[index]!.events) {
|
||||
matches.add(event);
|
||||
}
|
||||
}
|
||||
start = end;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return [...matches];
|
||||
}
|
||||
|
||||
function matchingAssAnimationEvents(options: {
|
||||
candidate: AnnotatedSubtitleCue;
|
||||
group: AssEventGroupIndex;
|
||||
allowFullLineFrames: boolean;
|
||||
}): AnnotatedSubtitleCue[] {
|
||||
const canonicalText = compactCueMatchText(options.candidate);
|
||||
// The group index already restricts to the candidate's style and name.
|
||||
const nearby = eventsOverlappingWindow(
|
||||
options.group,
|
||||
options.candidate.startTime - CANONICAL_MATCH_MARGIN_SECONDS,
|
||||
options.candidate.endTime + CANONICAL_MATCH_MARGIN_SECONDS,
|
||||
);
|
||||
const fragments = findCanonicalFragmentEvents(nearby, canonicalText);
|
||||
if (fragments.length >= MIN_CANONICAL_ANIMATION_EVENTS && hasAssAnimationEvidence(fragments)) {
|
||||
return fragments;
|
||||
}
|
||||
|
||||
if (!options.allowFullLineFrames) {
|
||||
return [];
|
||||
}
|
||||
const fullLineFrames = nearby.filter((cue) => compactCueMatchText(cue) === canonicalText);
|
||||
return fullLineFrames.length >= MIN_CANONICAL_ANIMATION_EVENTS &&
|
||||
hasAssAnimationEvidence(fullLineFrames)
|
||||
? fullLineFrames
|
||||
: [];
|
||||
}
|
||||
|
||||
// Reductions rather than `Math.min(...events)`: one generated line can carry an
|
||||
// unbounded number of events, and spreading them all as arguments risks the engine's
|
||||
// argument-count limit.
|
||||
function earliestStartTime(events: readonly AnnotatedSubtitleCue[], seed = Infinity): number {
|
||||
return events.reduce((earliest, event) => Math.min(earliest, event.startTime), seed);
|
||||
}
|
||||
|
||||
function latestEndTime(events: readonly AnnotatedSubtitleCue[], seed = -Infinity): number {
|
||||
return events.reduce((latest, event) => Math.max(latest, event.endTime), seed);
|
||||
}
|
||||
|
||||
function includeCanonicalBoundaryEvents(options: {
|
||||
candidate: AnnotatedSubtitleCue;
|
||||
group: AssEventGroupIndex;
|
||||
animationEvents: readonly AnnotatedSubtitleCue[];
|
||||
}): AnnotatedSubtitleCue[] {
|
||||
const canonicalText = compactCueMatchText(options.candidate);
|
||||
const startTime = earliestStartTime(options.animationEvents);
|
||||
const endTime = latestEndTime(options.animationEvents);
|
||||
return eventsOverlappingWindow(
|
||||
options.group,
|
||||
startTime - CANONICAL_MATCH_MARGIN_SECONDS,
|
||||
endTime + CANONICAL_MATCH_MARGIN_SECONDS,
|
||||
).filter((cue) => compactCueMatchText(cue) === canonicalText);
|
||||
}
|
||||
|
||||
function recoverCanonicalAssEvents({
|
||||
dialogue,
|
||||
comments,
|
||||
}: ParsedAssEvents): AnnotatedSubtitleCue[] {
|
||||
const recovered: AnnotatedSubtitleCue[] = [];
|
||||
const suppressed = new Set<AnnotatedSubtitleCue>();
|
||||
// A recovery is only as good as its owning event. When a later candidate proves that
|
||||
// an earlier candidate was itself a generated frame of its animation, the earlier
|
||||
// recovery is a duplicate of the same authored line and must be withdrawn.
|
||||
const recoveredByOwner = new Map<AnnotatedSubtitleCue, AnnotatedSubtitleCue>();
|
||||
const withdrawn = new Set<AnnotatedSubtitleCue>();
|
||||
const eventsByGroup = new Map<string, AnnotatedSubtitleCue[]>();
|
||||
for (const cue of dialogue) {
|
||||
const key = assEventGroupKey(cue);
|
||||
const group = eventsByGroup.get(key);
|
||||
if (group) {
|
||||
group.push(cue);
|
||||
} else {
|
||||
eventsByGroup.set(key, [cue]);
|
||||
}
|
||||
}
|
||||
const indexByGroup = new Map<string, AssEventGroupIndex>();
|
||||
for (const [key, events] of eventsByGroup) {
|
||||
indexByGroup.set(key, buildAssEventGroupIndex(events));
|
||||
}
|
||||
const emptyGroupIndex: AssEventGroupIndex = { byStart: [], prefixMaxEnd: [] };
|
||||
const candidates = [
|
||||
...comments.map((cue) => ({ cue, kind: 'comment' as const })),
|
||||
...dialogue
|
||||
.filter(
|
||||
(cue) =>
|
||||
compactCueMatchText(cue).length >= MIN_CANONICAL_DIALOGUE_TEXT_LENGTH &&
|
||||
hasAssAnimationEvidence([cue]),
|
||||
)
|
||||
.sort((left, right) => right.text.length - left.text.length || left.order - right.order)
|
||||
.map((cue) => ({ cue, kind: 'dialogue' as const })),
|
||||
];
|
||||
|
||||
for (const { cue: candidate, kind } of candidates) {
|
||||
if (candidate.endTime <= candidate.startTime || suppressed.has(candidate)) {
|
||||
continue;
|
||||
}
|
||||
const canonicalText = compactCueMatchText(candidate);
|
||||
if (!canonicalText) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const group = indexByGroup.get(assEventGroupKey(candidate)) ?? emptyGroupIndex;
|
||||
const animationEvents = matchingAssAnimationEvents({
|
||||
candidate,
|
||||
group,
|
||||
allowFullLineFrames: kind === 'comment',
|
||||
});
|
||||
if (animationEvents.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const boundaryEvents = includeCanonicalBoundaryEvents({
|
||||
candidate,
|
||||
group,
|
||||
animationEvents,
|
||||
});
|
||||
const generatedEvents = [...new Set([...animationEvents, ...boundaryEvents])];
|
||||
const animationStartTime = earliestStartTime(generatedEvents, candidate.startTime);
|
||||
const animationEndTime = latestEndTime(generatedEvents, candidate.endTime);
|
||||
const startTime = kind === 'comment' ? candidate.startTime : animationStartTime;
|
||||
const endTime = kind === 'comment' ? candidate.endTime : animationEndTime;
|
||||
const recoveredCue: AnnotatedSubtitleCue = {
|
||||
...candidate,
|
||||
startTime,
|
||||
endTime,
|
||||
animationStartTime,
|
||||
animationEndTime,
|
||||
source: 'canonical-ass',
|
||||
};
|
||||
recovered.push(recoveredCue);
|
||||
recoveredByOwner.set(candidate, recoveredCue);
|
||||
for (const event of generatedEvents) {
|
||||
suppressed.add(event);
|
||||
if (event === candidate) {
|
||||
continue;
|
||||
}
|
||||
const priorRecovery = recoveredByOwner.get(event);
|
||||
if (priorRecovery) {
|
||||
// No text is lost by withdrawing: a fragment claim means the withdrawn line is
|
||||
// a contiguous piece of this candidate's text, and a boundary claim means the
|
||||
// texts are equal, so the surviving canonical cue always contains it.
|
||||
withdrawn.add(priorRecovery);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const survivingRecovered = recovered.filter((cue) => !withdrawn.has(cue));
|
||||
if (survivingRecovered.length === 0) {
|
||||
return dialogue;
|
||||
}
|
||||
return [...dialogue.filter((cue) => !suppressed.has(cue)), ...survivingRecovered].sort(
|
||||
(a, b) => a.startTime - b.startTime || a.endTime - b.endTime || a.order - b.order,
|
||||
);
|
||||
}
|
||||
|
||||
function parseAnnotatedAssEvents(content: string): ParsedAssEvents {
|
||||
const cues: AnnotatedSubtitleCue[] = [];
|
||||
const comments: AnnotatedSubtitleCue[] = [];
|
||||
const lines = content.split(/\r?\n/);
|
||||
let inEventsSection = false;
|
||||
let eventOrder = 0;
|
||||
const fieldIndex = {
|
||||
start: -1,
|
||||
end: -1,
|
||||
@@ -222,7 +565,12 @@ function parseAnnotatedAssCues(content: string): AnnotatedSubtitleCue[] {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!trimmed.startsWith(ASS_DIALOGUE_PREFIX)) {
|
||||
const eventPrefix = trimmed.startsWith(ASS_DIALOGUE_PREFIX)
|
||||
? ASS_DIALOGUE_PREFIX
|
||||
: trimmed.startsWith(ASS_COMMENT_PREFIX)
|
||||
? ASS_COMMENT_PREFIX
|
||||
: null;
|
||||
if (!eventPrefix) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -230,7 +578,7 @@ function parseAnnotatedAssCues(content: string): AnnotatedSubtitleCue[] {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fields = trimmed.slice(ASS_DIALOGUE_PREFIX.length).split(',');
|
||||
const fields = trimmed.slice(eventPrefix.length).split(',');
|
||||
if (
|
||||
fieldIndex.start >= fields.length ||
|
||||
fieldIndex.end >= fields.length ||
|
||||
@@ -254,7 +602,7 @@ function parseAnnotatedAssCues(content: string): AnnotatedSubtitleCue[] {
|
||||
const effect = readField(fields, fieldIndex.effect);
|
||||
const layer = Number(readField(fields, fieldIndex.layer));
|
||||
const overrides = collectAssOverrideCommands(rawText);
|
||||
cues.push({
|
||||
const cue: AnnotatedSubtitleCue = {
|
||||
startTime,
|
||||
endTime,
|
||||
text,
|
||||
@@ -266,11 +614,21 @@ function parseAnnotatedAssCues(content: string): AnnotatedSubtitleCue[] {
|
||||
effectKind: parseAssEffectField(effect),
|
||||
overrides,
|
||||
overrideSignature: assOverrideSignature(overrides),
|
||||
order: cues.length,
|
||||
});
|
||||
order: eventOrder,
|
||||
};
|
||||
eventOrder += 1;
|
||||
if (eventPrefix === ASS_COMMENT_PREFIX) {
|
||||
comments.push(cue);
|
||||
} else {
|
||||
cues.push(cue);
|
||||
}
|
||||
}
|
||||
|
||||
return cues;
|
||||
return { dialogue: cues, comments };
|
||||
}
|
||||
|
||||
function parseAnnotatedAssCues(content: string): AnnotatedSubtitleCue[] {
|
||||
return recoverCanonicalAssEvents(parseAnnotatedAssEvents(content));
|
||||
}
|
||||
|
||||
export function parseAssCues(content: string): SubtitleCue[] {
|
||||
|
||||
@@ -1,6 +1,22 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { createNotifySendReplacer, resolveDefaultNotificationIconPath } from './notification';
|
||||
import {
|
||||
buildNotifySendEnv,
|
||||
createNotifySendReplacer,
|
||||
resolveDefaultNotificationIconPath,
|
||||
} from './notification';
|
||||
|
||||
test('notify-send child environment drops the AppImage library-path override', () => {
|
||||
const env = buildNotifySendEnv({
|
||||
LD_LIBRARY_PATH: '/tmp/.mount_SubMinXXXXXX/usr/lib',
|
||||
DBUS_SESSION_BUS_ADDRESS: 'unix:path=/run/user/1000/bus',
|
||||
HOME: '/home/user',
|
||||
});
|
||||
|
||||
assert.equal(env.LD_LIBRARY_PATH, undefined);
|
||||
assert.equal(env.DBUS_SESSION_BUS_ADDRESS, 'unix:path=/run/user/1000/bus');
|
||||
assert.equal(env.HOME, '/home/user');
|
||||
});
|
||||
|
||||
test('default notification icon resolves packaged SubMiner asset when no per-notification icon is provided', () => {
|
||||
const path = resolveDefaultNotificationIconPath({
|
||||
|
||||
@@ -203,8 +203,19 @@ export function createNotifySendReplacer(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Electron AppImages export `LD_LIBRARY_PATH=<mount>/usr/lib`, whose bundled libnotify predates the
|
||||
* symbols the system notify-send links against, so an inherited environment kills the child with a
|
||||
* symbol lookup error before it can send anything. A system binary resolves its own libraries fine,
|
||||
* so the override is dropped entirely rather than filtered.
|
||||
*/
|
||||
export function buildNotifySendEnv(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
|
||||
const { LD_LIBRARY_PATH: _dropped, ...rest } = env;
|
||||
return rest;
|
||||
}
|
||||
|
||||
const showLinuxReplaceableNotification = createNotifySendReplacer((args, callback) =>
|
||||
execFile('notify-send', args, { timeout: 5_000 }, (error, stdout) =>
|
||||
execFile('notify-send', args, { timeout: 5_000, env: buildNotifySendEnv() }, (error, stdout) =>
|
||||
callback(error, stdout ?? ''),
|
||||
),
|
||||
);
|
||||
|
||||
@@ -606,3 +606,93 @@ test('configureEarlyAppPaths pins userData to canonical SubMiner config dir', ()
|
||||
assert.equal(userDataPath, '/tmp/xdg/SubMiner');
|
||||
assert.deepEqual(calls, ['name:SubMiner', 'path:userData:/tmp/xdg/SubMiner']);
|
||||
});
|
||||
|
||||
test('configureEarlyAppPaths isolates development runs from the production profile', () => {
|
||||
const calls: string[] = [];
|
||||
|
||||
const userDataPath = configureEarlyAppPaths(
|
||||
{
|
||||
setName: (name) => calls.push(`name:${name}`),
|
||||
setPath: (key, value) => calls.push(`path:${key}:${value}`),
|
||||
},
|
||||
{
|
||||
platform: 'linux',
|
||||
homeDir: '/home/tester',
|
||||
xdgConfigHome: '/tmp/xdg',
|
||||
existsSync: () => false,
|
||||
argv: ['electron', '.', '--start', '--dev'],
|
||||
env: {},
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(userDataPath, '/tmp/xdg/SubMiner-dev');
|
||||
assert.deepEqual(calls, ['name:SubMiner', 'path:userData:/tmp/xdg/SubMiner-dev']);
|
||||
});
|
||||
|
||||
test('configureEarlyAppPaths ignores development flags forwarded to mpv', () => {
|
||||
for (const forwardedFlag of ['--dev', '--debug']) {
|
||||
let selectedPath = '';
|
||||
const userDataPath = configureEarlyAppPaths(
|
||||
{
|
||||
setName: () => {},
|
||||
setPath: (_key, value) => {
|
||||
selectedPath = value;
|
||||
},
|
||||
},
|
||||
{
|
||||
platform: 'linux',
|
||||
homeDir: '/home/tester',
|
||||
xdgConfigHome: '/tmp/xdg',
|
||||
existsSync: () => false,
|
||||
argv: ['electron', '.', '--launch-mpv', forwardedFlag],
|
||||
env: {},
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(userDataPath, '/tmp/xdg/SubMiner');
|
||||
assert.equal(selectedPath, '/tmp/xdg/SubMiner');
|
||||
}
|
||||
});
|
||||
|
||||
test('configureEarlyAppPaths uses the supplied environment for config discovery', () => {
|
||||
const paths: string[] = [];
|
||||
|
||||
const userDataPath = configureEarlyAppPaths(
|
||||
{
|
||||
setName: () => {},
|
||||
setPath: (_key, value) => paths.push(value),
|
||||
},
|
||||
{
|
||||
platform: 'linux',
|
||||
homeDir: '/home/tester',
|
||||
existsSync: () => false,
|
||||
argv: ['electron', '.', '--start'],
|
||||
env: { XDG_CONFIG_HOME: '/tmp/injected-xdg' },
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(userDataPath, '/tmp/injected-xdg/SubMiner');
|
||||
assert.deepEqual(paths, ['/tmp/injected-xdg/SubMiner']);
|
||||
});
|
||||
|
||||
test('configureEarlyAppPaths allows an explicit production-profile development run', () => {
|
||||
const paths: string[] = [];
|
||||
|
||||
const userDataPath = configureEarlyAppPaths(
|
||||
{
|
||||
setName: () => {},
|
||||
setPath: (_key, value) => paths.push(value),
|
||||
},
|
||||
{
|
||||
platform: 'win32',
|
||||
appDataDir: 'C:\\Users\\tester\\AppData\\Roaming',
|
||||
homeDir: 'C:\\Users\\tester',
|
||||
existsSync: () => false,
|
||||
argv: ['electron.exe', '.', '--debug'],
|
||||
env: { SUBMINER_USE_PRODUCTION_PROFILE: '1' },
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(userDataPath, 'C:\\Users\\tester\\AppData\\Roaming\\SubMiner');
|
||||
assert.deepEqual(paths, ['C:\\Users\\tester\\AppData\\Roaming\\SubMiner']);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { CliArgs, hasExplicitCommand, parseArgs, shouldStartApp } from './cli/args';
|
||||
import { resolveConfigDir } from './config/path-resolution';
|
||||
@@ -14,6 +15,8 @@ const TRANSPORTED_APP_ARGC_ENV = 'SUBMINER_APP_ARGC';
|
||||
const TRANSPORTED_APP_ARG_PREFIX = 'SUBMINER_APP_ARG_';
|
||||
const MAX_TRANSPORTED_APP_ARGS = 256;
|
||||
const APP_NAME = 'SubMiner';
|
||||
const DEVELOPMENT_APP_NAME = 'SubMiner-dev';
|
||||
export const USE_PRODUCTION_PROFILE_ENV = 'SUBMINER_USE_PRODUCTION_PROFILE';
|
||||
const DEFAULT_APP_CONTROL_HANDOFF_TIMEOUT_MS = 500;
|
||||
const MACOS_APP_CONTROL_HANDOFF_TIMEOUT_MS = 3000;
|
||||
const MPV_LONG_OPTIONS_WITH_SEPARATE_VALUES = new Set([
|
||||
@@ -53,6 +56,8 @@ type EarlyAppPathOptions = {
|
||||
xdgConfigHome?: string;
|
||||
homeDir?: string;
|
||||
existsSync?: (candidate: string) => boolean;
|
||||
argv?: string[];
|
||||
env?: NodeJS.ProcessEnv;
|
||||
};
|
||||
|
||||
function removeLsfgLayer(env: NodeJS.ProcessEnv): void {
|
||||
@@ -252,13 +257,25 @@ export function normalizeStartupArgv(argv: string[], env: NodeJS.ProcessEnv): st
|
||||
}
|
||||
|
||||
export function configureEarlyAppPaths(app: EarlyAppLike, options?: EarlyAppPathOptions): string {
|
||||
const userDataPath = resolveConfigDir({
|
||||
platform: options?.platform ?? process.platform,
|
||||
appDataDir: options?.appDataDir ?? process.env.APPDATA,
|
||||
xdgConfigHome: options?.xdgConfigHome ?? process.env.XDG_CONFIG_HOME,
|
||||
const platform = options?.platform ?? process.platform;
|
||||
const env = options?.env ?? process.env;
|
||||
const configDir = resolveConfigDir({
|
||||
platform,
|
||||
appDataDir: options?.appDataDir ?? env.APPDATA,
|
||||
xdgConfigHome: options?.xdgConfigHome ?? env.XDG_CONFIG_HOME,
|
||||
homeDir: options?.homeDir ?? os.homedir(),
|
||||
existsSync: options?.existsSync ?? fs.existsSync,
|
||||
});
|
||||
const argv = options?.argv ?? process.argv;
|
||||
const launchMpvIndex = argv.indexOf('--launch-mpv');
|
||||
const appArgv = launchMpvIndex === -1 ? argv : argv.slice(0, launchMpvIndex);
|
||||
const useDevelopmentProfile =
|
||||
(appArgv.includes('--dev') || appArgv.includes('--debug')) &&
|
||||
env[USE_PRODUCTION_PROFILE_ENV] !== '1';
|
||||
const platformPath = platform === 'win32' ? path.win32 : path.posix;
|
||||
const userDataPath = useDevelopmentProfile
|
||||
? platformPath.join(platformPath.dirname(configDir), DEVELOPMENT_APP_NAME)
|
||||
: configDir;
|
||||
|
||||
app.setName(APP_NAME);
|
||||
app.setPath('userData', userDataPath);
|
||||
|
||||
+20
-2
@@ -22,7 +22,10 @@ import {
|
||||
shouldHandleStatsDaemonCommandAtEntry,
|
||||
spawnDetachedApp,
|
||||
} from './main-entry-runtime';
|
||||
import { requestSingleInstanceLockEarly } from './main/early-single-instance';
|
||||
import {
|
||||
requestSingleInstanceLockEarly,
|
||||
shouldBypassSingleInstanceLockForArgv,
|
||||
} from './main/early-single-instance';
|
||||
import { readConfiguredWindowsMpvLaunch } from './main-entry-launch-config';
|
||||
import { isAppControlServerAvailable, sendAppControlCommand } from './shared/app-control-client';
|
||||
import {
|
||||
@@ -35,6 +38,7 @@ import { createWindowsMpvLaunchDeps, launchWindowsMpv } from './main/runtime/win
|
||||
import { runStatsDaemonControlFromProcess } from './stats-daemon-entry';
|
||||
import { handleSyncCliAtEntry } from './main/sync-cli';
|
||||
import { createFatalErrorReporter, registerFatalErrorHandlers } from './main/fatal-error';
|
||||
import { enforceElectronRuntimeGuard } from './main/electron-runtime-guard';
|
||||
import { buildMpvLoggingArgs } from './shared/mpv-logging-args';
|
||||
import {
|
||||
applyLogFileTogglesToEnv,
|
||||
@@ -192,11 +196,25 @@ registerFatalErrorHandlers({
|
||||
});
|
||||
|
||||
function startMainProcess(): void {
|
||||
const gotSingleInstanceLock = requestSingleInstanceLockEarly(app);
|
||||
// Normal launches serialize the runtime guard with the profile-scoped lock. Stats daemon
|
||||
// commands keep their existing lock bypass when Electron runs in Node mode.
|
||||
const gotSingleInstanceLock =
|
||||
shouldBypassSingleInstanceLockForArgv(process.argv) || requestSingleInstanceLockEarly(app);
|
||||
if (!gotSingleInstanceLock) {
|
||||
app.exit(0);
|
||||
return;
|
||||
}
|
||||
|
||||
const runtimeGuard = enforceElectronRuntimeGuard({
|
||||
electronVersion: process.versions.electron ?? '',
|
||||
userDataPath,
|
||||
});
|
||||
if (!runtimeGuard.ok) {
|
||||
console.error(runtimeGuard.details);
|
||||
dialog.showErrorBox(runtimeGuard.title, runtimeGuard.details);
|
||||
app.exit(1);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
require('./main.js');
|
||||
} catch (error) {
|
||||
|
||||
+84
-7
@@ -36,6 +36,10 @@ import { openPlaylistBrowser as openPlaylistBrowserRuntime } from './main/runtim
|
||||
import { createAniSkipRuntime } from './main/runtime/aniskip-runtime';
|
||||
import { resolveAniSkipMetadataForFile } from './main/runtime/aniskip-metadata';
|
||||
import { createDiscordRpcClient } from './main/runtime/discord-rpc-client.js';
|
||||
import {
|
||||
assertYomitanDictionaryMutationSafe,
|
||||
observeYomitanDictionaryCount,
|
||||
} from './main/runtime/yomitan-dictionary-integrity';
|
||||
import { startAppControlServer } from './main/runtime/app-control-server';
|
||||
import { createEnsureBackgroundStatsServerHandler } from './main/runtime/background-stats-startup';
|
||||
import {
|
||||
@@ -235,6 +239,7 @@ import {
|
||||
createCycleSecondarySubModeRuntimeHandler,
|
||||
} from './main/runtime/domains/mpv';
|
||||
import { buildSubtitleTrackDiagnostics } from './main/runtime/mpv-track-diagnostics';
|
||||
import { resolveCanonicalPrimarySubtitle } from './main/runtime/primary-subtitle-text';
|
||||
import {
|
||||
createBuildCopyCurrentSubtitleMainDepsHandler,
|
||||
createBuildHandleMineSentenceDigitMainDepsHandler,
|
||||
@@ -527,6 +532,7 @@ import {
|
||||
createRefreshSubtitlePrefetchFromActiveTrackHandler,
|
||||
createResolveActiveSubtitleSidebarSourceHandler,
|
||||
} from './main/runtime/subtitle-prefetch-runtime';
|
||||
import { createSecondarySubtitleTrackController } from './main/runtime/secondary-subtitle-track';
|
||||
import {
|
||||
createCreateAnilistSetupWindowHandler,
|
||||
createCreateConfigSettingsWindowHandler,
|
||||
@@ -762,6 +768,7 @@ type BootServices = MainBootServicesResult<
|
||||
const bootServices = createMainBootServices({
|
||||
platform: process.platform,
|
||||
argv: process.argv,
|
||||
configDir: app.getPath('userData'),
|
||||
appDataDir: process.env.APPDATA,
|
||||
xdgConfigHome: process.env.XDG_CONFIG_HOME,
|
||||
homeDir: os.homedir(),
|
||||
@@ -1430,6 +1437,10 @@ const firstRunSetupService = createFirstRunSetupService({
|
||||
error: (message, ...args) => logger.error(message, ...args),
|
||||
info: (message, ...args) => logger.info(message, ...args),
|
||||
});
|
||||
const integrity = observeYomitanDictionaryCount(USER_DATA_PATH, dictionaries.length);
|
||||
if (!integrity.safe) {
|
||||
logger.error(`[dictionary:integrity] ${integrity.message}`);
|
||||
}
|
||||
return dictionaries.length;
|
||||
},
|
||||
isExternalYomitanConfigured: () =>
|
||||
@@ -1806,10 +1817,42 @@ async function openYoutubeTrackPickerFromPlayback(): Promise<void> {
|
||||
let appTray: Tray | null = null;
|
||||
let tokenizeSubtitleDeferred: ((text: string) => Promise<SubtitleData>) | null = null;
|
||||
function withCurrentSubtitleTiming(payload: SubtitleData): SubtitleData {
|
||||
const canonical = resolveCanonicalPrimarySubtitle({
|
||||
liveText: payload.text,
|
||||
currentTimeSec: Number(appState.mpvClient?.currentTimePos),
|
||||
cues: appState.activeParsedSubtitleCues,
|
||||
});
|
||||
return {
|
||||
...payload,
|
||||
startTime: appState.mpvClient?.currentSubStart ?? null,
|
||||
endTime: appState.mpvClient?.currentSubEnd ?? null,
|
||||
startTime: canonical?.startTime ?? appState.mpvClient?.currentSubStart ?? null,
|
||||
endTime: canonical?.endTime ?? appState.mpvClient?.currentSubEnd ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function captureCurrentPrimarySubtitleMiningContext(): SubtitleMiningContext | null {
|
||||
const canonical = resolveCanonicalPrimarySubtitle({
|
||||
liveText: appState.mpvClient?.currentSubText ?? '',
|
||||
currentTimeSec: Number(appState.mpvClient?.currentTimePos),
|
||||
cues: appState.activeParsedSubtitleCues,
|
||||
});
|
||||
// Same validity bar as the live capture path: an unusable canonical span must fall
|
||||
// back rather than hand mining an empty line or an inverted range.
|
||||
const canonicalText = canonical?.text.trim();
|
||||
if (
|
||||
!canonical ||
|
||||
!canonicalText ||
|
||||
!Number.isFinite(canonical.startTime) ||
|
||||
!Number.isFinite(canonical.endTime) ||
|
||||
canonical.endTime <= canonical.startTime
|
||||
) {
|
||||
return captureLiveSubtitleMiningContext(appState.mpvClient);
|
||||
}
|
||||
return {
|
||||
source: 'overlay',
|
||||
text: canonicalText,
|
||||
startTime: canonical.startTime,
|
||||
endTime: canonical.endTime,
|
||||
capturedAtMs: Date.now(),
|
||||
};
|
||||
}
|
||||
function emitSubtitlePayload(payload: SubtitleData, options?: { resumePrefetch?: boolean }): void {
|
||||
@@ -1943,7 +1986,7 @@ const autoplaySubtitlePrimingRuntime = createAutoplaySubtitlePrimingRuntime({
|
||||
getLastObservedTimePos: () => lastObservedTimePos,
|
||||
getVisibleOverlayVisible: () => overlayManager.getVisibleOverlayVisible(),
|
||||
emitSecondarySubtitle: (text) => {
|
||||
overlayManager.broadcastToOverlayWindows('secondary-subtitle:set', text);
|
||||
secondarySubtitleTrackController.handleLiveText(text);
|
||||
},
|
||||
initSubtitlePrefetch: (sourcePath, currentTimePos, sourceKey) =>
|
||||
subtitlePrefetchInitController.initSubtitlePrefetch(sourcePath, currentTimePos, sourceKey),
|
||||
@@ -2001,6 +2044,24 @@ const resolveActiveSubtitleSidebarSourceHandler = createResolveActiveSubtitleSid
|
||||
logDebug: (message) => logger.debug(message),
|
||||
});
|
||||
|
||||
const secondarySubtitleTrackController = createSecondarySubtitleTrackController({
|
||||
getMpvClient: () => appState.mpvClient,
|
||||
getCurrentTimePos: () => appState.mpvClient?.currentTimePos ?? lastObservedTimePos,
|
||||
resolveSubtitleSource: (input) => resolveActiveSubtitleSidebarSourceHandler(input),
|
||||
loadSubtitleSourceText,
|
||||
parseSubtitleCues: (content, filename) => parseSubtitleCues(content, filename),
|
||||
setCurrentSecondaryText: (text) => {
|
||||
if (appState.mpvClient) {
|
||||
appState.mpvClient.currentSecondarySubText = text;
|
||||
}
|
||||
},
|
||||
broadcastSecondaryText: (text) => {
|
||||
overlayManager.broadcastToOverlayWindows('secondary-subtitle:set', text);
|
||||
},
|
||||
logDebug: (message) => logger.debug(message),
|
||||
logWarn: (message, error) => logger.warn(message, error),
|
||||
});
|
||||
|
||||
const refreshSubtitlePrefetchFromActiveTrackHandler =
|
||||
createRefreshSubtitlePrefetchFromActiveTrackHandler({
|
||||
getMpvClient: () => appState.mpvClient,
|
||||
@@ -2471,10 +2532,12 @@ const characterDictionaryAutoSyncRuntime = createCharacterDictionaryAutoSyncRunt
|
||||
),
|
||||
getYomitanDictionaryInfo: async () => {
|
||||
await ensureYomitanExtensionLoaded();
|
||||
return await getYomitanDictionaryInfo(getYomitanParserRuntimeDeps(), {
|
||||
const dictionaries = await getYomitanDictionaryInfo(getYomitanParserRuntimeDeps(), {
|
||||
error: (message, ...args) => logger.error(message, ...args),
|
||||
info: (message, ...args) => logger.info(message, ...args),
|
||||
});
|
||||
assertYomitanDictionaryMutationSafe(USER_DATA_PATH, dictionaries.length);
|
||||
return dictionaries;
|
||||
},
|
||||
importYomitanDictionary: async (zipPath) => {
|
||||
if (yomitanProfilePolicy.isExternalReadOnlyMode()) {
|
||||
@@ -4383,6 +4446,7 @@ const {
|
||||
onMpvConnected: () => {
|
||||
maybeStartOverlayLoadingOsd();
|
||||
flushQueuedMpvOsdNotifications();
|
||||
secondarySubtitleTrackController.scheduleRefresh(0);
|
||||
if (appState.sessionBindingsInitialized) {
|
||||
sendMpvCommandRuntime(appState.mpvClient, [
|
||||
'script-message',
|
||||
@@ -4401,6 +4465,9 @@ const {
|
||||
broadcastToOverlayWindows: (channel, payload) => {
|
||||
overlayManager.broadcastToOverlayWindows(channel, payload);
|
||||
},
|
||||
onSecondarySubtitleChange: (text) => {
|
||||
secondarySubtitleTrackController.handleLiveText(text);
|
||||
},
|
||||
getImmediateSubtitlePayload: (text) => subtitleProcessingController.consumeCachedSubtitle(text),
|
||||
emitImmediateSubtitle: (payload) => {
|
||||
emitSubtitlePayload(payload);
|
||||
@@ -4434,6 +4501,7 @@ const {
|
||||
appState.activeParsedSubtitleMediaPath,
|
||||
);
|
||||
if ((normalizedPath || null) !== previousPath) {
|
||||
secondarySubtitleTrackController.reset();
|
||||
const resetSubtitlePayload = { text: '', tokens: null };
|
||||
const frequencyDictionary = configService.getConfig().subtitleStyle.frequencyDictionary;
|
||||
const frequencyOptions = {
|
||||
@@ -4468,6 +4536,7 @@ const {
|
||||
void youtubeMediaCachePlaybackRuntime.handleMediaPathChange(path);
|
||||
if (path) {
|
||||
ensureImmersionTrackerStarted();
|
||||
secondarySubtitleTrackController.scheduleRefresh();
|
||||
void subtitlePrefetchRuntime.refreshSubtitlePrefetchFromActiveTrack();
|
||||
// Retry after a short delay because MPV can populate track-list after path.
|
||||
subtitlePrefetchRuntime.scheduleSubtitlePrefetchRefresh(500);
|
||||
@@ -4522,6 +4591,7 @@ const {
|
||||
subtitlePrefetchService.onSeek(time);
|
||||
}
|
||||
lastObservedTimePos = time;
|
||||
secondarySubtitleTrackController.handleTimePos(time);
|
||||
},
|
||||
onFullscreenChange: (fullscreen) => {
|
||||
cancelLinuxMpvFullscreenOverlayRefreshBurst = updateLinuxMpvFullscreenOverlayRefreshBurst(
|
||||
@@ -4549,6 +4619,13 @@ const {
|
||||
autoplaySubtitlePrimingRuntime.scheduleSubtitlePrefetchRefresh();
|
||||
youtubePrimarySubtitleNotificationRuntime.handleSubtitleTrackChange(sid);
|
||||
},
|
||||
onSecondarySubtitleTrackChange: () => {
|
||||
secondarySubtitleTrackController.handleTrackChange();
|
||||
secondarySubtitleTrackController.scheduleRefresh(0);
|
||||
},
|
||||
onSecondarySubtitleDelayChange: (delay) => {
|
||||
secondarySubtitleTrackController.handleDelayChange(delay);
|
||||
},
|
||||
onSubtitleTrackListChange: (trackList) => {
|
||||
const diagnostics = buildSubtitleTrackDiagnostics(
|
||||
lastObservedPrimarySubtitleTrackId,
|
||||
@@ -4562,6 +4639,7 @@ const {
|
||||
logger.info('[mpv-subtitles] subtitle track list updated', diagnostics);
|
||||
}
|
||||
managedLocalSubtitleSelectionRuntime.handleSubtitleTrackListChange(trackList);
|
||||
secondarySubtitleTrackController.scheduleRefresh(0);
|
||||
autoplaySubtitlePrimingRuntime.scheduleSubtitlePrefetchRefresh();
|
||||
youtubePrimarySubtitleNotificationRuntime.handleSubtitleTrackListChange(trackList);
|
||||
},
|
||||
@@ -5231,6 +5309,7 @@ const markLastCardAsAudioCardHandler = createMarkLastCardAsAudioCardHandler(
|
||||
const buildMineSentenceCardMainDepsHandler = createBuildMineSentenceCardMainDepsHandler({
|
||||
getAnkiIntegration: () => appState.ankiIntegration,
|
||||
getMpvClient: () => appState.mpvClient,
|
||||
getPrimarySubtitle: () => captureCurrentPrimarySubtitleMiningContext(),
|
||||
showMpvOsd: (text) => overlayNotificationsRuntime.showConfiguredStatusNotification(text),
|
||||
mineSentenceCardCore,
|
||||
recordCardsMined: (count, noteIds) => {
|
||||
@@ -5544,9 +5623,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
|
||||
// live mpv sub timings at lookup time so media generation clips the mined line even
|
||||
// when extraction finishes long after playback has moved on.
|
||||
recordSubtitleMiningContext: (context) =>
|
||||
recordSubtitleMiningContext(
|
||||
context ?? captureLiveSubtitleMiningContext(appState.mpvClient),
|
||||
),
|
||||
recordSubtitleMiningContext(context ?? captureCurrentPrimarySubtitleMiningContext()),
|
||||
quitApp: () => requestAppQuit(),
|
||||
toggleVisibleOverlay: () => toggleVisibleOverlay(),
|
||||
tokenizeCurrentSubtitle: async () => {
|
||||
|
||||
@@ -138,3 +138,56 @@ test('createMainBootServices builds boot-phase service bundle', () => {
|
||||
assert.deepEqual(calls, ['mkdir:/tmp/subminer-config', 'exit:7']);
|
||||
assert.equal(setPathValue, '/tmp/subminer-config');
|
||||
});
|
||||
|
||||
test('createMainBootServices honors the profile selected by the early entrypoint', () => {
|
||||
const services = createMainBootServices({
|
||||
platform: 'linux',
|
||||
argv: ['electron', '.', '--dev'],
|
||||
configDir: '/tmp/SubMiner-dev',
|
||||
appDataDir: undefined,
|
||||
xdgConfigHome: undefined,
|
||||
homeDir: '/home/tester',
|
||||
defaultMpvLogFile: '/tmp/default.log',
|
||||
envMpvLog: undefined,
|
||||
defaultTexthookerPort: 5174,
|
||||
getDefaultSocketPath: () => '/tmp/subminer.sock',
|
||||
resolveConfigDir: () => {
|
||||
throw new Error('early profile should be authoritative');
|
||||
},
|
||||
existsSync: () => false,
|
||||
mkdirSync: () => {},
|
||||
joinPath: (...parts) => parts.join('/'),
|
||||
app: {
|
||||
setPath: () => {},
|
||||
quit: () => {},
|
||||
exit: () => {},
|
||||
on: () => ({}),
|
||||
whenReady: async () => {},
|
||||
},
|
||||
shouldBypassSingleInstanceLock: () => false,
|
||||
requestSingleInstanceLockEarly: () => true,
|
||||
registerSecondInstanceHandlerEarly: () => {},
|
||||
onConfigStartupParseError: () => {},
|
||||
createConfigService: (configDir) => ({ configDir }),
|
||||
createAnilistTokenStore: (targetPath) => ({ targetPath }),
|
||||
createJellyfinTokenStore: (targetPath) => ({ targetPath }),
|
||||
createAnilistUpdateQueue: (targetPath) => ({ targetPath }),
|
||||
createSubtitleWebSocket: (payloadMode) => ({ payloadMode }),
|
||||
createLogger: () => ({ warn: () => {}, info: () => {}, error: () => {} }),
|
||||
createMainRuntimeRegistry: () => ({}),
|
||||
createOverlayManager: () => ({ getMainWindow: () => null, getModalWindow: () => null }),
|
||||
createOverlayModalInputState: () => ({
|
||||
getModalInputExclusive: () => false,
|
||||
handleModalInputStateChange: () => {},
|
||||
}),
|
||||
createOverlayContentMeasurementStore: () => ({}),
|
||||
getSyncOverlayShortcutsForModal: () => () => {},
|
||||
getSyncOverlayVisibilityForModal: () => () => {},
|
||||
createOverlayModalRuntime: () => ({}),
|
||||
createAppState: (input) => input,
|
||||
});
|
||||
|
||||
assert.equal(services.configDir, '/tmp/SubMiner-dev');
|
||||
assert.equal(services.userDataPath, '/tmp/SubMiner-dev');
|
||||
assert.deepEqual(services.configService, { configDir: '/tmp/SubMiner-dev' });
|
||||
});
|
||||
|
||||
@@ -31,6 +31,7 @@ export interface MainBootServicesParams<
|
||||
> {
|
||||
platform: NodeJS.Platform;
|
||||
argv: string[];
|
||||
configDir?: string;
|
||||
appDataDir: string | undefined;
|
||||
xdgConfigHome: string | undefined;
|
||||
homeDir: string;
|
||||
@@ -174,13 +175,15 @@ export function createMainBootServices<
|
||||
TAppState,
|
||||
TAppLifecycleApp
|
||||
> {
|
||||
const configDir = params.resolveConfigDir({
|
||||
platform: params.platform,
|
||||
appDataDir: params.appDataDir,
|
||||
xdgConfigHome: params.xdgConfigHome,
|
||||
homeDir: params.homeDir,
|
||||
existsSync: params.existsSync,
|
||||
});
|
||||
const configDir =
|
||||
params.configDir ??
|
||||
params.resolveConfigDir({
|
||||
platform: params.platform,
|
||||
appDataDir: params.appDataDir,
|
||||
xdgConfigHome: params.xdgConfigHome,
|
||||
homeDir: params.homeDir,
|
||||
existsSync: params.existsSync,
|
||||
});
|
||||
const userDataPath = configDir;
|
||||
const defaultMpvLogPath = params.envMpvLog?.trim() || params.defaultMpvLogFile;
|
||||
const defaultImmersionDbPath = params.joinPath(userDataPath, 'immersion.sqlite');
|
||||
|
||||
@@ -244,13 +244,13 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
};
|
||||
};
|
||||
|
||||
const findCachedSnapshotForSeriesKey = (
|
||||
const findCachedSnapshotForSeriesKey = async (
|
||||
seriesKey: string,
|
||||
fallbackSeriesKey?: string,
|
||||
): CharacterDictionarySnapshot | null => {
|
||||
): Promise<CharacterDictionarySnapshot | null> => {
|
||||
const acceptedKeys = new Set([seriesKey, fallbackSeriesKey].filter(Boolean));
|
||||
return (
|
||||
readCachedSnapshots(outputDir).find((snapshot) => {
|
||||
(await readCachedSnapshots(outputDir)).find((snapshot) => {
|
||||
const snapshotSeriesKey = buildCharacterDictionarySeriesKey({
|
||||
mediaPath: null,
|
||||
mediaTitle: snapshot.mediaTitle,
|
||||
@@ -293,7 +293,9 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
|
||||
const cachedResolution = readCachedMediaResolution(outputDir, seriesKey);
|
||||
if (cachedResolution) {
|
||||
const cachedSnapshot = readSnapshot(getSnapshotPath(outputDir, cachedResolution.mediaId));
|
||||
const cachedSnapshot = await readSnapshot(
|
||||
getSnapshotPath(outputDir, cachedResolution.mediaId),
|
||||
);
|
||||
if (cachedSnapshot) {
|
||||
deps.logInfo?.(
|
||||
`[dictionary] cached AniList match: ${cachedSnapshot.mediaTitle} -> AniList ${cachedSnapshot.mediaId}`,
|
||||
@@ -305,7 +307,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
}
|
||||
}
|
||||
|
||||
const cachedSnapshot = findCachedSnapshotForSeriesKey(seriesKey, unscopedSeriesKey);
|
||||
const cachedSnapshot = await findCachedSnapshotForSeriesKey(seriesKey, unscopedSeriesKey);
|
||||
if (cachedSnapshot) {
|
||||
writeCachedMediaResolution(outputDir, {
|
||||
seriesKey,
|
||||
@@ -348,7 +350,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
progress?: CharacterDictionarySnapshotProgressCallbacks,
|
||||
): Promise<CharacterDictionarySnapshotResult> => {
|
||||
const snapshotPath = getSnapshotPath(outputDir, mediaId);
|
||||
const cachedSnapshot = readSnapshot(snapshotPath);
|
||||
const cachedSnapshot = await readSnapshot(snapshotPath);
|
||||
const refreshReason = cachedSnapshot ? getCachedSnapshotRefreshReason(cachedSnapshot) : null;
|
||||
if (cachedSnapshot && refreshReason === null) {
|
||||
deps.logInfo?.(`[dictionary] snapshot hit for AniList ${mediaId}`);
|
||||
@@ -485,7 +487,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
resolvedNameSplits,
|
||||
nameSplitSource,
|
||||
);
|
||||
writeSnapshot(snapshotPath, snapshot);
|
||||
await writeSnapshot(snapshotPath, snapshot);
|
||||
deps.logInfo?.(
|
||||
`[dictionary] stored snapshot for AniList ${mediaId}: ${snapshot.entryCount} terms`,
|
||||
);
|
||||
@@ -526,19 +528,22 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
const snapshotResults = await Promise.all(
|
||||
normalizedMediaIds.map((mediaId) => getOrCreateSnapshot(mediaId)),
|
||||
);
|
||||
const snapshots = snapshotResults.map(({ mediaId }) => {
|
||||
const snapshot = readSnapshot(getSnapshotPath(outputDir, mediaId));
|
||||
// Sequential on purpose: each snapshot parse is a chunk of main-thread work, so reading them
|
||||
// one at a time keeps the event loop breathing between files.
|
||||
const snapshots: CharacterDictionarySnapshot[] = [];
|
||||
for (const { mediaId } of snapshotResults) {
|
||||
const snapshot = await readSnapshot(getSnapshotPath(outputDir, mediaId));
|
||||
if (!snapshot) {
|
||||
throw new Error(`Missing character dictionary snapshot for AniList ${mediaId}.`);
|
||||
}
|
||||
return snapshot;
|
||||
});
|
||||
snapshots.push(snapshot);
|
||||
}
|
||||
const revision = buildMergedRevision(normalizedMediaIds, snapshots);
|
||||
const description =
|
||||
snapshots.length === 1
|
||||
? `Character names from ${snapshots[0]!.mediaTitle}`
|
||||
: `Character names from ${snapshots.length} recent anime`;
|
||||
const { zipPath, entryCount } = buildDictionaryZip(
|
||||
const { zipPath, entryCount } = await buildDictionaryZip(
|
||||
getMergedZipPath(outputDir),
|
||||
CHARACTER_DICTIONARY_MERGED_TITLE,
|
||||
description,
|
||||
@@ -633,7 +638,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
resolvedMedia.title,
|
||||
waitForAniListRequestSlot,
|
||||
);
|
||||
const storedSnapshot = readSnapshot(getSnapshotPath(outputDir, resolvedMedia.id));
|
||||
const storedSnapshot = await readSnapshot(getSnapshotPath(outputDir, resolvedMedia.id));
|
||||
if (!storedSnapshot) {
|
||||
throw new Error(`Snapshot missing after generation for AniList ${resolvedMedia.id}.`);
|
||||
}
|
||||
@@ -642,7 +647,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
const description = `Character names from ${storedSnapshot.mediaTitle} [AniList media ID ${resolvedMedia.id}]`;
|
||||
const zipPath = path.join(outputDir, `anilist-${resolvedMedia.id}.zip`);
|
||||
deps.logInfo?.(`[dictionary] building ZIP for AniList ${resolvedMedia.id}`);
|
||||
buildDictionaryZip(
|
||||
await buildDictionaryZip(
|
||||
zipPath,
|
||||
dictionaryTitle,
|
||||
description,
|
||||
|
||||
@@ -3,6 +3,7 @@ import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import test from 'node:test';
|
||||
import { isDeepStrictEqual } from 'node:util';
|
||||
import { getSnapshotPath, readSnapshot, writeSnapshot } from './cache';
|
||||
import { CHARACTER_DICTIONARY_FORMAT_VERSION } from './constants';
|
||||
import type { CharacterDictionarySnapshot } from './types';
|
||||
@@ -29,17 +30,72 @@ function createSnapshot(): CharacterDictionarySnapshot {
|
||||
};
|
||||
}
|
||||
|
||||
test('writeSnapshot persists and readSnapshot restores current-format snapshots', () => {
|
||||
test('writeSnapshot persists and readSnapshot restores current-format snapshots', async () => {
|
||||
const outputDir = makeTempDir();
|
||||
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
||||
const snapshot = createSnapshot();
|
||||
|
||||
writeSnapshot(snapshotPath, snapshot);
|
||||
await writeSnapshot(snapshotPath, snapshot);
|
||||
|
||||
assert.deepEqual(readSnapshot(snapshotPath), { ...snapshot, nameSplitSource: 'heuristic' });
|
||||
assert.deepEqual(await readSnapshot(snapshotPath), { ...snapshot, nameSplitSource: 'heuristic' });
|
||||
});
|
||||
|
||||
test('readSnapshot preserves the mecab name-split source and defaults missing values to heuristic', () => {
|
||||
// A manual generate and an auto-sync can both land on the same media, so two writes for one
|
||||
// snapshot can overlap. They must not stream into a shared temp file and interleave into a
|
||||
// half-and-half snapshot.
|
||||
test('concurrent writeSnapshot calls for the same media leave one complete snapshot', async () => {
|
||||
const outputDir = makeTempDir();
|
||||
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
||||
const base = createSnapshot();
|
||||
// Distinct titles, lengths, and term text so the surviving file can be pinned to exactly one
|
||||
// writer rather than merely "a snapshot that parses". A shared temp file is caught by the
|
||||
// losing writers failing to rename; interleaved content is only caught when the timing happens
|
||||
// to leave a mix, which is why the assertion checks identity rather than shape.
|
||||
const variants: CharacterDictionarySnapshot[] = ['alpha', 'beta', 'gamma'].map((label, index) => {
|
||||
const entryCount = 400 + index * 100;
|
||||
return {
|
||||
...base,
|
||||
mediaTitle: `${base.mediaTitle} ${label}`,
|
||||
entryCount,
|
||||
termEntries: Array.from({ length: entryCount }, (_entry, entryIndex) => [
|
||||
`${label}${entryIndex}`,
|
||||
'なまえ',
|
||||
'name primary',
|
||||
'',
|
||||
75,
|
||||
[`${label} character ${entryIndex} `.repeat(600)],
|
||||
0,
|
||||
'',
|
||||
]) as CharacterDictionarySnapshot['termEntries'],
|
||||
};
|
||||
});
|
||||
|
||||
await Promise.all(variants.map((variant) => writeSnapshot(snapshotPath, variant)));
|
||||
|
||||
const restored = await readSnapshot(snapshotPath);
|
||||
const expected = variants.map((variant) => ({
|
||||
...variant,
|
||||
nameSplitSource: 'heuristic' as const,
|
||||
}));
|
||||
const matches = expected.filter((candidate) => isDeepStrictEqual(restored, candidate));
|
||||
assert.equal(
|
||||
matches.length,
|
||||
1,
|
||||
`expected exactly one writer's complete snapshot to survive, got ${
|
||||
restored === null
|
||||
? 'an unreadable file'
|
||||
: `entryCount=${restored.entryCount}, terms=${restored.termEntries.length}, title=${restored.mediaTitle}`
|
||||
}`,
|
||||
);
|
||||
|
||||
// Every writer cleaned up after itself, so no temp files are left behind.
|
||||
const leftovers = fs
|
||||
.readdirSync(path.dirname(snapshotPath))
|
||||
.filter((name) => name.includes('.tmp-'));
|
||||
assert.deepEqual(leftovers, []);
|
||||
});
|
||||
|
||||
test('readSnapshot preserves the mecab name-split source and defaults missing values to heuristic', async () => {
|
||||
const outputDir = makeTempDir();
|
||||
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
||||
const snapshot: CharacterDictionarySnapshot = {
|
||||
@@ -47,12 +103,12 @@ test('readSnapshot preserves the mecab name-split source and defaults missing va
|
||||
nameSplitSource: 'mecab',
|
||||
};
|
||||
|
||||
writeSnapshot(snapshotPath, snapshot);
|
||||
await writeSnapshot(snapshotPath, snapshot);
|
||||
|
||||
assert.equal(readSnapshot(snapshotPath)?.nameSplitSource, 'mecab');
|
||||
assert.equal((await readSnapshot(snapshotPath))?.nameSplitSource, 'mecab');
|
||||
});
|
||||
|
||||
test('readSnapshot ignores snapshots written with an older format version', () => {
|
||||
test('readSnapshot ignores snapshots written with an older format version', async () => {
|
||||
const outputDir = makeTempDir();
|
||||
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
||||
const staleSnapshot = {
|
||||
@@ -63,10 +119,10 @@ test('readSnapshot ignores snapshots written with an older format version', () =
|
||||
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
|
||||
fs.writeFileSync(snapshotPath, JSON.stringify(staleSnapshot), 'utf8');
|
||||
|
||||
assert.equal(readSnapshot(snapshotPath), null);
|
||||
assert.equal(await readSnapshot(snapshotPath), null);
|
||||
});
|
||||
|
||||
test('readSnapshot ignores v15 snapshots with stale romanized character-name entries', () => {
|
||||
test('readSnapshot ignores v15 snapshots with stale romanized character-name entries', async () => {
|
||||
const outputDir = makeTempDir();
|
||||
const snapshotPath = getSnapshotPath(outputDir, 130298);
|
||||
const staleSnapshot = {
|
||||
@@ -78,5 +134,5 @@ test('readSnapshot ignores v15 snapshots with stale romanized character-name ent
|
||||
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
|
||||
fs.writeFileSync(snapshotPath, JSON.stringify(staleSnapshot), 'utf8');
|
||||
|
||||
assert.equal(readSnapshot(snapshotPath), null);
|
||||
assert.equal(await readSnapshot(snapshotPath), null);
|
||||
});
|
||||
|
||||
@@ -102,24 +102,42 @@ export function writeCachedMediaResolution(
|
||||
writeMediaResolutionEntries(outputDir, [...remaining, normalized]);
|
||||
}
|
||||
|
||||
export function readCachedSnapshots(outputDir: string): CharacterDictionarySnapshot[] {
|
||||
/**
|
||||
* Snapshots for long series run to hundreds of MB each, so everything here reads them off the main
|
||||
* thread's critical path: file IO is async and only the unavoidable JSON.parse runs on the loop,
|
||||
* one file at a time. Reading the whole directory synchronously used to block the process for
|
||||
* multiple seconds, long enough for the compositor to declare the app unresponsive mid-playback.
|
||||
*/
|
||||
export async function readCachedSnapshots(
|
||||
outputDir: string,
|
||||
): Promise<CharacterDictionarySnapshot[]> {
|
||||
let entries: fs.Dirent[] = [];
|
||||
try {
|
||||
entries = fs.readdirSync(getSnapshotsDir(outputDir), { withFileTypes: true });
|
||||
entries = await fs.promises.readdir(getSnapshotsDir(outputDir), { withFileTypes: true });
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
return entries
|
||||
const names = entries
|
||||
.filter((entry) => entry.isFile() && /^anilist-\d+\.json$/.test(entry.name))
|
||||
.sort((left, right) => left.name.localeCompare(right.name))
|
||||
.map((entry) => readSnapshot(path.join(getSnapshotsDir(outputDir), entry.name)))
|
||||
.filter((snapshot): snapshot is CharacterDictionarySnapshot => snapshot !== null);
|
||||
.map((entry) => entry.name)
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
|
||||
const snapshots: CharacterDictionarySnapshot[] = [];
|
||||
for (const name of names) {
|
||||
const snapshot = await readSnapshot(path.join(getSnapshotsDir(outputDir), name));
|
||||
if (snapshot) {
|
||||
snapshots.push(snapshot);
|
||||
}
|
||||
}
|
||||
return snapshots;
|
||||
}
|
||||
|
||||
export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot | null {
|
||||
export async function readSnapshot(
|
||||
snapshotPath: string,
|
||||
): Promise<CharacterDictionarySnapshot | null> {
|
||||
try {
|
||||
const raw = fs.readFileSync(snapshotPath, 'utf8');
|
||||
const raw = await fs.promises.readFile(snapshotPath, 'utf8');
|
||||
const parsed = JSON.parse(raw) as Partial<CharacterDictionarySnapshot>;
|
||||
if (!parsed || typeof parsed !== 'object') {
|
||||
return null;
|
||||
@@ -150,9 +168,64 @@ export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot
|
||||
}
|
||||
}
|
||||
|
||||
export function writeSnapshot(snapshotPath: string, snapshot: CharacterDictionarySnapshot): void {
|
||||
// Flushing in a few-MB batches keeps each stringify-and-write slice short; a single
|
||||
// JSON.stringify of a large snapshot blocks the event loop for seconds.
|
||||
const SNAPSHOT_WRITE_FLUSH_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
// Distinguishes concurrent writes of the same snapshot within one process; the pid alone only
|
||||
// separates processes, so two overlapping writers would otherwise stream into the same temp file.
|
||||
let snapshotWriteSequence = 0;
|
||||
|
||||
/**
|
||||
* Streams the snapshot to disk piece by piece instead of stringifying it in one shot, then renames
|
||||
* the finished file into place so a crash mid-write (or two concurrent writers for the same media)
|
||||
* can never leave a torn file where a snapshot used to be.
|
||||
*/
|
||||
export async function writeSnapshot(
|
||||
snapshotPath: string,
|
||||
snapshot: CharacterDictionarySnapshot,
|
||||
): Promise<void> {
|
||||
ensureDir(path.dirname(snapshotPath));
|
||||
fs.writeFileSync(snapshotPath, JSON.stringify(snapshot, null, 2), 'utf8');
|
||||
snapshotWriteSequence += 1;
|
||||
const tempPath = `${snapshotPath}.tmp-${process.pid}-${snapshotWriteSequence}`;
|
||||
const handle = await fs.promises.open(tempPath, 'w');
|
||||
try {
|
||||
let buffered: string[] = [];
|
||||
let bufferedBytes = 0;
|
||||
const push = async (chunk: string): Promise<void> => {
|
||||
buffered.push(chunk);
|
||||
bufferedBytes += chunk.length;
|
||||
if (bufferedBytes >= SNAPSHOT_WRITE_FLUSH_BYTES) {
|
||||
const joined = buffered.join('');
|
||||
buffered = [];
|
||||
bufferedBytes = 0;
|
||||
await handle.write(joined, null, 'utf8');
|
||||
}
|
||||
};
|
||||
const writeArray = async (key: string, items: readonly unknown[]): Promise<void> => {
|
||||
await push(`,${JSON.stringify(key)}:[`);
|
||||
for (let i = 0; i < items.length; i += 1) {
|
||||
await push(`${i > 0 ? ',' : ''}${JSON.stringify(items[i])}`);
|
||||
}
|
||||
await push(']');
|
||||
};
|
||||
|
||||
const { termEntries, images, ...scalars } = snapshot;
|
||||
const head = JSON.stringify(scalars);
|
||||
await push(head.slice(0, -1));
|
||||
await writeArray('termEntries', termEntries);
|
||||
await writeArray('images', images);
|
||||
await push('}');
|
||||
if (buffered.length > 0) {
|
||||
await handle.write(buffered.join(''), null, 'utf8');
|
||||
}
|
||||
} catch (error) {
|
||||
await handle.close();
|
||||
await fs.promises.rm(tempPath, { force: true });
|
||||
throw error;
|
||||
}
|
||||
await handle.close();
|
||||
await fs.promises.rename(tempPath, snapshotPath);
|
||||
}
|
||||
|
||||
export function buildMergedRevision(
|
||||
|
||||
@@ -11,6 +11,22 @@ import {
|
||||
} from './image-lookup';
|
||||
import type { CharacterDictionarySnapshot } from './types';
|
||||
|
||||
// Lookup indexes rebuild in the background while gets serve stale data, so tests poll until the
|
||||
// refresh they triggered has landed.
|
||||
async function waitForRefresh<T>(probe: () => T | null | undefined): Promise<T> {
|
||||
const deadline = Date.now() + 5000;
|
||||
for (;;) {
|
||||
const value = probe();
|
||||
if (value !== null && value !== undefined) {
|
||||
return value;
|
||||
}
|
||||
if (Date.now() > deadline) {
|
||||
throw new Error('timed out waiting for background snapshot refresh');
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
}
|
||||
|
||||
const PNG_1X1_BASE64 =
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+nmX8AAAAASUVORK5CYII=';
|
||||
|
||||
@@ -18,7 +34,7 @@ function makeTempDir(): string {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-character-image-lookup-'));
|
||||
}
|
||||
|
||||
test('buildCharacterNameImageIndexFromSnapshots maps name terms to character portrait data URLs', () => {
|
||||
test('buildCharacterNameImageIndexFromSnapshots maps name terms to character portrait data URLs', async () => {
|
||||
const outputDir = makeTempDir();
|
||||
const snapshot: CharacterDictionarySnapshot = {
|
||||
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
|
||||
@@ -75,9 +91,9 @@ test('buildCharacterNameImageIndexFromSnapshots maps name terms to character por
|
||||
{ path: 'img/m130298-va456.png', dataBase64: 'BBBB' },
|
||||
],
|
||||
};
|
||||
writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
|
||||
await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
|
||||
|
||||
const index = buildCharacterNameImageIndexFromSnapshots(outputDir);
|
||||
const index = await buildCharacterNameImageIndexFromSnapshots(outputDir);
|
||||
|
||||
assert.deepEqual(index.get('アレクシア'), {
|
||||
src: 'data:image/png;base64,AAAA',
|
||||
@@ -85,7 +101,7 @@ test('buildCharacterNameImageIndexFromSnapshots maps name terms to character por
|
||||
});
|
||||
});
|
||||
|
||||
test('buildCharacterNameImageIndexFromSnapshots sniffs image MIME from bytes before path extension', () => {
|
||||
test('buildCharacterNameImageIndexFromSnapshots sniffs image MIME from bytes before path extension', async () => {
|
||||
const outputDir = makeTempDir();
|
||||
const snapshot: CharacterDictionarySnapshot = {
|
||||
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
|
||||
@@ -116,14 +132,14 @@ test('buildCharacterNameImageIndexFromSnapshots sniffs image MIME from bytes bef
|
||||
],
|
||||
images: [{ path: 'img/m130298-c123.jpg', dataBase64: PNG_1X1_BASE64 }],
|
||||
};
|
||||
writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
|
||||
await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
|
||||
|
||||
const index = buildCharacterNameImageIndexFromSnapshots(outputDir);
|
||||
const index = await buildCharacterNameImageIndexFromSnapshots(outputDir);
|
||||
|
||||
assert.equal(index.get('アレクシア')?.src, `data:image/png;base64,${PNG_1X1_BASE64}`);
|
||||
});
|
||||
|
||||
test('createCharacterDictionaryImageLookup can scope duplicate names to the current media', () => {
|
||||
test('createCharacterDictionaryImageLookup can scope duplicate names to the current media', async () => {
|
||||
const outputDir = makeTempDir();
|
||||
const towerSnapshot: CharacterDictionarySnapshot = {
|
||||
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
|
||||
@@ -173,15 +189,16 @@ test('createCharacterDictionaryImageLookup can scope duplicate names to the curr
|
||||
],
|
||||
images: [{ path: 'img/m21202-c2.png', dataBase64: 'KONOSUBA' }],
|
||||
};
|
||||
writeSnapshot(getSnapshotPath(outputDir, towerSnapshot.mediaId), towerSnapshot);
|
||||
writeSnapshot(getSnapshotPath(outputDir, konosubaSnapshot.mediaId), konosubaSnapshot);
|
||||
await writeSnapshot(getSnapshotPath(outputDir, towerSnapshot.mediaId), towerSnapshot);
|
||||
await writeSnapshot(getSnapshotPath(outputDir, konosubaSnapshot.mediaId), konosubaSnapshot);
|
||||
|
||||
const lookup = createCharacterDictionaryImageLookup({ outputDir });
|
||||
|
||||
assert.equal(lookup.get('カズ', 21202)?.alt, 'Kazuma');
|
||||
const scoped = await waitForRefresh(() => lookup.get('カズ', 21202));
|
||||
assert.equal(scoped.alt, 'Kazuma');
|
||||
});
|
||||
|
||||
test('createCharacterDictionaryImageLookup does not fall back globally on scoped miss', () => {
|
||||
test('createCharacterDictionaryImageLookup does not fall back globally on scoped miss', async () => {
|
||||
const outputDir = makeTempDir();
|
||||
const snapshot: CharacterDictionarySnapshot = {
|
||||
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
|
||||
@@ -208,10 +225,11 @@ test('createCharacterDictionaryImageLookup does not fall back globally on scoped
|
||||
],
|
||||
images: [{ path: 'img/m115230-c1.png', dataBase64: 'TOWER' }],
|
||||
};
|
||||
writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
|
||||
await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
|
||||
|
||||
const lookup = createCharacterDictionaryImageLookup({ outputDir });
|
||||
|
||||
const unscoped = await waitForRefresh(() => lookup.get('カズ'));
|
||||
assert.equal(unscoped.alt, 'Kaz');
|
||||
assert.equal(lookup.get('カズ', 21202), null);
|
||||
assert.equal(lookup.get('カズ')?.alt, 'Kaz');
|
||||
});
|
||||
|
||||
@@ -204,11 +204,11 @@ function getSnapshotDirectorySignature(outputDir: string): string {
|
||||
return parts.sort().join('|');
|
||||
}
|
||||
|
||||
export function buildCharacterNameImageIndexFromSnapshots(
|
||||
export async function buildCharacterNameImageIndexFromSnapshots(
|
||||
outputDir: string,
|
||||
): Map<string, CharacterNameImage> {
|
||||
): Promise<Map<string, CharacterNameImage>> {
|
||||
const index = new Map<string, CharacterNameImage>();
|
||||
for (const snapshot of readCachedSnapshots(outputDir)) {
|
||||
for (const snapshot of await readCachedSnapshots(outputDir)) {
|
||||
appendSnapshotImages(index, snapshot);
|
||||
}
|
||||
return index;
|
||||
@@ -228,7 +228,12 @@ export function createCharacterDictionaryImageLookup(deps: {
|
||||
let signature: string | null = null;
|
||||
let index = new Map<string, CharacterNameImage>();
|
||||
let indexByMediaId = new Map<number, Map<string, CharacterNameImage>>();
|
||||
let refreshInFlight = false;
|
||||
|
||||
// Rebuilding means re-reading every cached snapshot (potentially GBs of JSON), which used to run
|
||||
// synchronously inside a lookup and froze the whole app right after a snapshot changed. Lookups
|
||||
// now serve the previous index while a single background rebuild catches up; the swap is atomic
|
||||
// and the signature only advances once the rebuild it belongs to has landed.
|
||||
function refreshIfNeeded(): void {
|
||||
if (!outputDir) {
|
||||
index = new Map<string, CharacterNameImage>();
|
||||
@@ -237,20 +242,30 @@ export function createCharacterDictionaryImageLookup(deps: {
|
||||
return;
|
||||
}
|
||||
const nextSignature = getSnapshotDirectorySignature(outputDir);
|
||||
if (nextSignature === signature) {
|
||||
if (nextSignature === signature || refreshInFlight) {
|
||||
return;
|
||||
}
|
||||
signature = nextSignature;
|
||||
index = new Map<string, CharacterNameImage>();
|
||||
indexByMediaId = new Map<number, Map<string, CharacterNameImage>>();
|
||||
for (const snapshot of readCachedSnapshots(outputDir)) {
|
||||
appendSnapshotImages(index, snapshot);
|
||||
const mediaIndex = new Map<string, CharacterNameImage>();
|
||||
appendSnapshotImages(mediaIndex, snapshot);
|
||||
if (mediaIndex.size > 0) {
|
||||
indexByMediaId.set(snapshot.mediaId, mediaIndex);
|
||||
refreshInFlight = true;
|
||||
void (async () => {
|
||||
try {
|
||||
const snapshots = await readCachedSnapshots(outputDir);
|
||||
const nextIndex = new Map<string, CharacterNameImage>();
|
||||
const nextIndexByMediaId = new Map<number, Map<string, CharacterNameImage>>();
|
||||
for (const snapshot of snapshots) {
|
||||
appendSnapshotImages(nextIndex, snapshot);
|
||||
const mediaIndex = new Map<string, CharacterNameImage>();
|
||||
appendSnapshotImages(mediaIndex, snapshot);
|
||||
if (mediaIndex.size > 0) {
|
||||
nextIndexByMediaId.set(snapshot.mediaId, mediaIndex);
|
||||
}
|
||||
}
|
||||
index = nextIndex;
|
||||
indexByMediaId = nextIndexByMediaId;
|
||||
signature = nextSignature;
|
||||
} finally {
|
||||
refreshInFlight = false;
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -32,17 +32,33 @@ function writeSnapshot(outputDir: string, mediaId: number, entries: Array<[strin
|
||||
);
|
||||
}
|
||||
|
||||
function withTempDir<T>(run: (dir: string) => T): T {
|
||||
async function withTempDir<T>(run: (dir: string) => Promise<T> | T): Promise<T> {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-name-candidates-'));
|
||||
try {
|
||||
return run(dir);
|
||||
return await run(dir);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test('collects terms and readings for the current media', () => {
|
||||
withTempDir((dir) => {
|
||||
// The snapshot index rebuilds in the background while lookups serve stale data, so tests poll the
|
||||
// probe until the refresh they triggered has landed.
|
||||
async function waitForRefresh<T>(probe: () => T | null | undefined): Promise<T> {
|
||||
const deadline = Date.now() + 5000;
|
||||
for (;;) {
|
||||
const value = probe();
|
||||
if (value !== null && value !== undefined) {
|
||||
return value;
|
||||
}
|
||||
if (Date.now() > deadline) {
|
||||
throw new Error('timed out waiting for background snapshot refresh');
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
}
|
||||
|
||||
test('collects terms and readings for the current media', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
writeSnapshot(dir, 1, [
|
||||
['ミナト', 'みなと'],
|
||||
['湊', 'みなと'],
|
||||
@@ -53,17 +69,16 @@ test('collects terms and readings for the current media', () => {
|
||||
outputDir: dir,
|
||||
getCurrentMediaId: () => 1,
|
||||
});
|
||||
const candidates = lookup.get();
|
||||
const candidates = await waitForRefresh(() => lookup.get());
|
||||
|
||||
assert.ok(candidates);
|
||||
assert.deepEqual([...candidates.forms].sort(), ['みなと', 'ミナト', '湊'].sort());
|
||||
// Deduplicated: both entries share the みなと reading.
|
||||
assert.equal(candidates.forms.length, 3);
|
||||
});
|
||||
});
|
||||
|
||||
test('returns null without a media scope so the scanner stays exhaustive', () => {
|
||||
withTempDir((dir) => {
|
||||
test('returns null without a media scope so the scanner stays exhaustive', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
||||
|
||||
const lookup = createCharacterNameCandidateLookup({
|
||||
@@ -71,12 +86,14 @@ test('returns null without a media scope so the scanner stays exhaustive', () =>
|
||||
getCurrentMediaId: () => null,
|
||||
});
|
||||
|
||||
// The explicitly-scoped probe proves the index has loaded before the unscoped case is judged.
|
||||
await waitForRefresh(() => lookup.get(1));
|
||||
assert.equal(lookup.get(), null);
|
||||
});
|
||||
});
|
||||
|
||||
test('returns null for a media with no cached snapshot', () => {
|
||||
withTempDir((dir) => {
|
||||
test('returns null for a media with no cached snapshot', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
||||
|
||||
const lookup = createCharacterNameCandidateLookup({
|
||||
@@ -84,29 +101,31 @@ test('returns null for a media with no cached snapshot', () => {
|
||||
getCurrentMediaId: () => 999,
|
||||
});
|
||||
|
||||
await waitForRefresh(() => lookup.get(1));
|
||||
assert.equal(lookup.get(), null);
|
||||
});
|
||||
});
|
||||
|
||||
test('key changes when the snapshot content changes', () => {
|
||||
withTempDir((dir) => {
|
||||
test('key changes when the snapshot content changes', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
||||
const lookup = createCharacterNameCandidateLookup({
|
||||
outputDir: dir,
|
||||
getCurrentMediaId: () => 1,
|
||||
});
|
||||
const first = lookup.get();
|
||||
const first = await waitForRefresh(() => lookup.get());
|
||||
|
||||
writeSnapshot(dir, 1, [
|
||||
['ミナト', 'みなと'],
|
||||
['アクア', 'あくあ'],
|
||||
]);
|
||||
lookup.invalidate();
|
||||
const second = lookup.get();
|
||||
const second = await waitForRefresh(() => {
|
||||
const candidates = lookup.get();
|
||||
return candidates && candidates.forms.length === 4 ? candidates : null;
|
||||
});
|
||||
|
||||
assert.ok(first && second);
|
||||
assert.notEqual(first.key, second.key);
|
||||
assert.equal(second.forms.length, 4);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -114,8 +133,8 @@ test('key changes when the snapshot content changes', () => {
|
||||
// directory every call. Asserted behaviorally: an unannounced on-disk change is
|
||||
// invisible until the recheck interval elapses, which can only be true if the
|
||||
// filesystem is not consulted per lookup.
|
||||
test('does not re-read the snapshot directory on every lookup', () => {
|
||||
withTempDir((dir) => {
|
||||
test('does not re-read the snapshot directory on every lookup', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
||||
let nowMs = 1_000_000;
|
||||
const lookup = createCharacterNameCandidateLookup({
|
||||
@@ -124,6 +143,7 @@ test('does not re-read the snapshot directory on every lookup', () => {
|
||||
now: () => nowMs,
|
||||
});
|
||||
|
||||
await waitForRefresh(() => lookup.get());
|
||||
assert.equal(lookup.get()?.forms.length, 2);
|
||||
|
||||
writeSnapshot(dir, 1, [
|
||||
@@ -135,12 +155,16 @@ test('does not re-read the snapshot directory on every lookup', () => {
|
||||
assert.equal(lookup.get()?.forms.length, 2, 'expected the cached list within the interval');
|
||||
|
||||
nowMs += 10_000;
|
||||
assert.equal(lookup.get()?.forms.length, 4, 'expected a refresh past the interval');
|
||||
const refreshed = await waitForRefresh(() => {
|
||||
const candidates = lookup.get();
|
||||
return candidates && candidates.forms.length === 4 ? candidates : null;
|
||||
});
|
||||
assert.equal(refreshed.forms.length, 4, 'expected a refresh past the interval');
|
||||
});
|
||||
});
|
||||
|
||||
test('invalidate picks up a snapshot change immediately', () => {
|
||||
withTempDir((dir) => {
|
||||
test('invalidate picks up a snapshot change on the next refresh', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
|
||||
let nowMs = 1_000_000;
|
||||
const lookup = createCharacterNameCandidateLookup({
|
||||
@@ -149,6 +173,7 @@ test('invalidate picks up a snapshot change immediately', () => {
|
||||
now: () => nowMs,
|
||||
});
|
||||
|
||||
await waitForRefresh(() => lookup.get());
|
||||
assert.equal(lookup.get()?.forms.length, 2);
|
||||
|
||||
writeSnapshot(dir, 1, [
|
||||
@@ -158,6 +183,10 @@ test('invalidate picks up a snapshot change immediately', () => {
|
||||
nowMs += 1;
|
||||
lookup.invalidate();
|
||||
|
||||
assert.equal(lookup.get()?.forms.length, 4);
|
||||
const refreshed = await waitForRefresh(() => {
|
||||
const candidates = lookup.get();
|
||||
return candidates && candidates.forms.length === 4 ? candidates : null;
|
||||
});
|
||||
assert.equal(refreshed.forms.length, 4);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -98,7 +98,12 @@ export function createCharacterNameCandidateLookup(deps: {
|
||||
let signature: string | null = null;
|
||||
let lastSignatureCheckAtMs = 0;
|
||||
let formsByMediaId = new Map<number, string[]>();
|
||||
let refreshInFlight = false;
|
||||
|
||||
// Same stale-while-revalidate shape as the image lookup: the rebuild re-reads every cached
|
||||
// snapshot, so it runs in the background while lookups keep serving the previous forms. The
|
||||
// signature only advances once its rebuild has landed, so a failed or superseded rebuild is
|
||||
// retried on the next signature check.
|
||||
function refreshIfNeeded(): void {
|
||||
if (!outputDir) {
|
||||
formsByMediaId = new Map<number, string[]>();
|
||||
@@ -114,17 +119,26 @@ export function createCharacterNameCandidateLookup(deps: {
|
||||
}
|
||||
lastSignatureCheckAtMs = nowMs;
|
||||
const nextSignature = getSnapshotDirectorySignature(outputDir);
|
||||
if (nextSignature === signature) {
|
||||
if (nextSignature === signature || refreshInFlight) {
|
||||
return;
|
||||
}
|
||||
signature = nextSignature;
|
||||
formsByMediaId = new Map<number, string[]>();
|
||||
for (const snapshot of readCachedSnapshots(outputDir)) {
|
||||
const forms = collectSnapshotNameForms(snapshot);
|
||||
if (forms.length > 0) {
|
||||
formsByMediaId.set(snapshot.mediaId, forms);
|
||||
refreshInFlight = true;
|
||||
void (async () => {
|
||||
try {
|
||||
const snapshots = await readCachedSnapshots(outputDir);
|
||||
const nextFormsByMediaId = new Map<number, string[]>();
|
||||
for (const snapshot of snapshots) {
|
||||
const forms = collectSnapshotNameForms(snapshot);
|
||||
if (forms.length > 0) {
|
||||
nextFormsByMediaId.set(snapshot.mediaId, forms);
|
||||
}
|
||||
}
|
||||
formsByMediaId = nextFormsByMediaId;
|
||||
signature = nextSignature;
|
||||
} finally {
|
||||
refreshInFlight = false;
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -34,7 +34,7 @@ function createSnapshotWithoutImages(): CharacterDictionarySnapshot {
|
||||
test('generateForCurrentMedia refreshes same-version snapshots missing images when inline images are enabled', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||
writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
|
||||
await writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
|
||||
const originalFetch = globalThis.fetch;
|
||||
const fetchUrls: string[] = [];
|
||||
|
||||
@@ -124,7 +124,7 @@ test('generateForCurrentMedia refreshes same-version snapshots missing images wh
|
||||
test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||
writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
||||
await writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
||||
...createSnapshotWithoutImages(),
|
||||
nameSplitSource: 'heuristic',
|
||||
});
|
||||
@@ -213,7 +213,7 @@ test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable'
|
||||
test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is available', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||
writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
||||
await writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
||||
...createSnapshotWithoutImages(),
|
||||
nameSplitSource: 'mecab',
|
||||
});
|
||||
@@ -253,7 +253,7 @@ test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is availabl
|
||||
test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is unavailable', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||
writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
||||
await writeSnapshot(getSnapshotPath(outputDir, 130298), {
|
||||
...createSnapshotWithoutImages(),
|
||||
nameSplitSource: 'heuristic',
|
||||
});
|
||||
@@ -293,7 +293,7 @@ test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is una
|
||||
test('generateForCurrentMedia keeps same-version snapshots without images when inline images are disabled', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const outputDir = path.join(userDataPath, 'character-dictionaries');
|
||||
writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
|
||||
await writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = (async (input: string | URL | Request) => {
|
||||
|
||||
@@ -42,7 +42,7 @@ function readStoredZipEntries(zipPath: string): Map<string, Buffer> {
|
||||
return entries;
|
||||
}
|
||||
|
||||
test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', () => {
|
||||
test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', async () => {
|
||||
const tempDir = makeTempDir();
|
||||
const outputPath = path.join(tempDir, 'dictionary.zip');
|
||||
const termEntries: CharacterDictionaryTermEntry[] = [
|
||||
@@ -62,7 +62,7 @@ test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', ()
|
||||
);
|
||||
}) as typeof Buffer.concat;
|
||||
|
||||
const result = buildDictionaryZip(
|
||||
const result = await buildDictionaryZip(
|
||||
outputPath,
|
||||
'Dictionary Title',
|
||||
'Dictionary Description',
|
||||
@@ -106,11 +106,11 @@ test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', ()
|
||||
}
|
||||
});
|
||||
|
||||
test('readDictionaryZipRevision reads the built revision and rejects foreign archives', () => {
|
||||
test('readDictionaryZipRevision reads the built revision and rejects foreign archives', async () => {
|
||||
const dir = makeTempDir();
|
||||
try {
|
||||
const zipPath = path.join(dir, 'merged.zip');
|
||||
buildDictionaryZip(
|
||||
await buildDictionaryZip(
|
||||
zipPath,
|
||||
'SubMiner Character Dictionary',
|
||||
'Character names',
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import * as path from 'path';
|
||||
import { readStoredZipFirstFile, writeStoredZip } from '../../shared/stored-zip';
|
||||
import { readStoredZipFirstFile, writeStoredZipAsync } from '../../shared/stored-zip';
|
||||
import { ensureDir } from './fs-utils';
|
||||
import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } from './types';
|
||||
|
||||
@@ -48,14 +48,14 @@ export function readDictionaryZipRevision(zipPath: string): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
export function buildDictionaryZip(
|
||||
export async function buildDictionaryZip(
|
||||
outputPath: string,
|
||||
dictionaryTitle: string,
|
||||
description: string,
|
||||
revision: string,
|
||||
termEntries: CharacterDictionaryTermEntry[],
|
||||
images: CharacterDictionarySnapshotImage[],
|
||||
): { zipPath: string; entryCount: number } {
|
||||
): Promise<{ zipPath: string; entryCount: number }> {
|
||||
ensureDir(path.dirname(outputPath));
|
||||
|
||||
function* zipFiles(): Iterable<{ name: string; data: Buffer }> {
|
||||
@@ -78,7 +78,11 @@ export function buildDictionaryZip(
|
||||
};
|
||||
}
|
||||
|
||||
const entriesPerBank = 10_000;
|
||||
// Each bank is stringified in one shot, so the bank size sets the longest single block in the
|
||||
// build. 10k entries measured ~38MB and ~135ms per bank on a real merged dictionary; 2k keeps
|
||||
// every bank under the archive writer's yield budget at ~27ms. Yomitan reads any number of
|
||||
// term_bank_N.json files, so this only changes how the terms are split across them.
|
||||
const entriesPerBank = 2_000;
|
||||
for (let i = 0; i < termEntries.length; i += entriesPerBank) {
|
||||
yield {
|
||||
name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`,
|
||||
@@ -87,6 +91,6 @@ export function buildDictionaryZip(
|
||||
}
|
||||
}
|
||||
|
||||
writeStoredZip(outputPath, zipFiles());
|
||||
await writeStoredZipAsync(outputPath, zipFiles());
|
||||
return { zipPath: outputPath, entryCount: termEntries.length };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { enforceElectronRuntimeGuard, SUPPORTED_ELECTRON_MAJOR } from './electron-runtime-guard';
|
||||
|
||||
function withTempDir(run: (directory: string) => void): void {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-electron-guard-'));
|
||||
try {
|
||||
run(directory);
|
||||
} finally {
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test('runtime guard major matches the pinned Electron dependency', () => {
|
||||
const packageJson = JSON.parse(
|
||||
fs.readFileSync(path.join(process.cwd(), 'package.json'), 'utf8'),
|
||||
) as { devDependencies: { electron: string } };
|
||||
|
||||
assert.equal(Number.parseInt(packageJson.devDependencies.electron.split('.', 1)[0]!, 10), 43);
|
||||
assert.equal(SUPPORTED_ELECTRON_MAJOR, 43);
|
||||
});
|
||||
|
||||
test('runtime guard records the supported Electron major', () => {
|
||||
withTempDir((userDataPath) => {
|
||||
const result = enforceElectronRuntimeGuard({
|
||||
electronVersion: '43.4.1',
|
||||
userDataPath,
|
||||
supportedElectronMajor: 43,
|
||||
});
|
||||
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(JSON.parse(fs.readFileSync(result.statePath, 'utf8')), {
|
||||
highestElectronMajor: 43,
|
||||
lastElectronVersion: '43.4.1',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test('runtime guard rejects a runtime outside the build major without writing state', () => {
|
||||
withTempDir((userDataPath) => {
|
||||
const result = enforceElectronRuntimeGuard({
|
||||
electronVersion: '44.0.0',
|
||||
userDataPath,
|
||||
supportedElectronMajor: 43,
|
||||
});
|
||||
|
||||
assert.equal(result.ok, false);
|
||||
if (result.ok) return;
|
||||
assert.equal(result.title, 'Unsupported Electron runtime');
|
||||
assert.match(result.details, /requires Electron 43/);
|
||||
assert.equal(fs.existsSync(result.statePath), false);
|
||||
});
|
||||
});
|
||||
|
||||
test('runtime guard rejects prerelease Electron versions without writing state', () => {
|
||||
withTempDir((userDataPath) => {
|
||||
const result = enforceElectronRuntimeGuard({
|
||||
electronVersion: '43.4.1-beta.1',
|
||||
userDataPath,
|
||||
supportedElectronMajor: 43,
|
||||
});
|
||||
|
||||
assert.equal(result.ok, false);
|
||||
if (result.ok) return;
|
||||
assert.equal(result.title, 'SubMiner could not verify Electron');
|
||||
assert.equal(fs.existsSync(result.statePath), false);
|
||||
});
|
||||
});
|
||||
|
||||
test('runtime guard blocks a profile downgrade before rewriting its safety record', () => {
|
||||
withTempDir((userDataPath) => {
|
||||
const statePath = path.join(userDataPath, 'electron-runtime.json');
|
||||
fs.writeFileSync(
|
||||
statePath,
|
||||
JSON.stringify({ highestElectronMajor: 44, lastElectronVersion: '44.1.0' }),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
const result = enforceElectronRuntimeGuard({
|
||||
electronVersion: '43.4.1',
|
||||
userDataPath,
|
||||
supportedElectronMajor: 43,
|
||||
});
|
||||
|
||||
assert.equal(result.ok, false);
|
||||
if (result.ok) return;
|
||||
assert.equal(result.title, 'Electron downgrade blocked');
|
||||
assert.match(result.details, /destroy Yomitan dictionaries/);
|
||||
assert.equal(result.details.includes(statePath), true);
|
||||
assert.deepEqual(JSON.parse(fs.readFileSync(statePath, 'utf8')), {
|
||||
highestElectronMajor: 44,
|
||||
lastElectronVersion: '44.1.0',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test('runtime guard blocks a downgrade within the supported Electron major', () => {
|
||||
withTempDir((userDataPath) => {
|
||||
const statePath = path.join(userDataPath, 'electron-runtime.json');
|
||||
fs.writeFileSync(
|
||||
statePath,
|
||||
JSON.stringify({ highestElectronMajor: 43, lastElectronVersion: '43.4.1' }),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
const result = enforceElectronRuntimeGuard({
|
||||
electronVersion: '43.3.0',
|
||||
userDataPath,
|
||||
supportedElectronMajor: 43,
|
||||
});
|
||||
|
||||
assert.equal(result.ok, false);
|
||||
if (result.ok) return;
|
||||
assert.equal(result.title, 'Electron downgrade blocked');
|
||||
assert.equal(result.details.includes(statePath), true);
|
||||
assert.deepEqual(JSON.parse(fs.readFileSync(statePath, 'utf8')), {
|
||||
highestElectronMajor: 43,
|
||||
lastElectronVersion: '43.4.1',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test('runtime guard fails closed when its safety record is malformed', () => {
|
||||
withTempDir((userDataPath) => {
|
||||
const statePath = path.join(userDataPath, 'electron-runtime.json');
|
||||
fs.writeFileSync(statePath, '{}', 'utf8');
|
||||
|
||||
const result = enforceElectronRuntimeGuard({
|
||||
electronVersion: '43.4.1',
|
||||
userDataPath,
|
||||
supportedElectronMajor: 43,
|
||||
});
|
||||
|
||||
assert.equal(result.ok, false);
|
||||
if (result.ok) return;
|
||||
assert.equal(result.title, 'SubMiner profile safety check failed');
|
||||
assert.match(result.details, /invalid format/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { writeTextFileAtomicallyDurable } from '../shared/fs-utils';
|
||||
|
||||
export const SUPPORTED_ELECTRON_MAJOR = 43;
|
||||
const RUNTIME_STATE_FILE_NAME = 'electron-runtime.json';
|
||||
|
||||
type ElectronRuntimeState = {
|
||||
highestElectronMajor: number;
|
||||
lastElectronVersion: string;
|
||||
};
|
||||
|
||||
type ParsedElectronVersion = {
|
||||
major: number;
|
||||
minor: number;
|
||||
patch: number;
|
||||
};
|
||||
|
||||
type ValidatedElectronRuntimeState = {
|
||||
state: ElectronRuntimeState;
|
||||
version: ParsedElectronVersion;
|
||||
};
|
||||
|
||||
export type ElectronRuntimeGuardResult =
|
||||
| { ok: true; statePath: string }
|
||||
| { ok: false; title: string; details: string; statePath: string };
|
||||
|
||||
function parseElectronVersion(version: string): ParsedElectronVersion | null {
|
||||
const match = /^(\d+)\.(\d+)\.(\d+)(?:\+[0-9A-Za-z.-]+)?$/.exec(version.trim());
|
||||
if (!match) return null;
|
||||
|
||||
const major = Number.parseInt(match[1]!, 10);
|
||||
const minor = Number.parseInt(match[2]!, 10);
|
||||
const patch = Number.parseInt(match[3]!, 10);
|
||||
if (![major, minor, patch].every((part) => Number.isSafeInteger(part) && part >= 0)) {
|
||||
return null;
|
||||
}
|
||||
if (major === 0) return null;
|
||||
return { major, minor, patch };
|
||||
}
|
||||
|
||||
function compareElectronVersions(
|
||||
left: ParsedElectronVersion,
|
||||
right: ParsedElectronVersion,
|
||||
): number {
|
||||
return left.major - right.major || left.minor - right.minor || left.patch - right.patch;
|
||||
}
|
||||
|
||||
function readRuntimeState(statePath: string): ValidatedElectronRuntimeState | null {
|
||||
if (!fs.existsSync(statePath)) return null;
|
||||
|
||||
const parsed = JSON.parse(fs.readFileSync(statePath, 'utf8')) as Partial<ElectronRuntimeState>;
|
||||
const highestElectronMajor = parsed.highestElectronMajor;
|
||||
const lastElectronVersion =
|
||||
typeof parsed.lastElectronVersion === 'string'
|
||||
? parseElectronVersion(parsed.lastElectronVersion)
|
||||
: null;
|
||||
if (
|
||||
typeof highestElectronMajor !== 'number' ||
|
||||
!Number.isSafeInteger(highestElectronMajor) ||
|
||||
highestElectronMajor <= 0 ||
|
||||
lastElectronVersion === null
|
||||
) {
|
||||
throw new Error('The runtime safety record has an invalid format.');
|
||||
}
|
||||
|
||||
return {
|
||||
state: {
|
||||
highestElectronMajor,
|
||||
lastElectronVersion: parsed.lastElectronVersion!,
|
||||
},
|
||||
version: lastElectronVersion,
|
||||
};
|
||||
}
|
||||
|
||||
function writeRuntimeState(statePath: string, state: ElectronRuntimeState): void {
|
||||
writeTextFileAtomicallyDurable(statePath, `${JSON.stringify(state, null, 2)}\n`);
|
||||
}
|
||||
|
||||
export function enforceElectronRuntimeGuard(options: {
|
||||
electronVersion: string;
|
||||
userDataPath: string;
|
||||
supportedElectronMajor?: number;
|
||||
}): ElectronRuntimeGuardResult {
|
||||
const supportedElectronMajor = options.supportedElectronMajor ?? SUPPORTED_ELECTRON_MAJOR;
|
||||
const statePath = path.join(options.userDataPath, RUNTIME_STATE_FILE_NAME);
|
||||
const currentVersion = parseElectronVersion(options.electronVersion);
|
||||
|
||||
if (currentVersion === null) {
|
||||
return {
|
||||
ok: false,
|
||||
title: 'SubMiner could not verify Electron',
|
||||
details: `Electron reported an invalid version: ${JSON.stringify(options.electronVersion)}. SubMiner did not load Yomitan storage.`,
|
||||
statePath,
|
||||
};
|
||||
}
|
||||
|
||||
if (currentVersion.major !== supportedElectronMajor) {
|
||||
return {
|
||||
ok: false,
|
||||
title: 'Unsupported Electron runtime',
|
||||
details: [
|
||||
`This SubMiner build requires Electron ${supportedElectronMajor}.`,
|
||||
`The current runtime is Electron ${options.electronVersion}.`,
|
||||
'',
|
||||
'Launch SubMiner through its packaged application or the repository package scripts. Yomitan storage was not loaded.',
|
||||
].join('\n'),
|
||||
statePath,
|
||||
};
|
||||
}
|
||||
|
||||
let previousState: ValidatedElectronRuntimeState | null;
|
||||
try {
|
||||
previousState = readRuntimeState(statePath);
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
title: 'SubMiner profile safety check failed',
|
||||
details: [
|
||||
`SubMiner could not read the runtime safety record at ${statePath}.`,
|
||||
(error as Error).message,
|
||||
'',
|
||||
'Yomitan storage was not loaded. Repair or remove only this safety record after verifying the profile backup.',
|
||||
].join('\n'),
|
||||
statePath,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
previousState &&
|
||||
(currentVersion.major < previousState.state.highestElectronMajor ||
|
||||
compareElectronVersions(currentVersion, previousState.version) < 0)
|
||||
) {
|
||||
return {
|
||||
ok: false,
|
||||
title: 'Electron downgrade blocked',
|
||||
details: [
|
||||
`This profile was previously opened with Electron ${previousState.state.lastElectronVersion}.`,
|
||||
`The current runtime is Electron ${options.electronVersion}.`,
|
||||
`Runtime safety record: ${statePath}.`,
|
||||
'',
|
||||
'Opening Chromium storage with an older Electron version can destroy Yomitan dictionaries. Upgrade SubMiner before using this profile.',
|
||||
].join('\n'),
|
||||
statePath,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
writeRuntimeState(statePath, {
|
||||
highestElectronMajor: Math.max(
|
||||
currentVersion.major,
|
||||
previousState?.state.highestElectronMajor ?? 0,
|
||||
),
|
||||
lastElectronVersion: options.electronVersion,
|
||||
});
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
title: 'SubMiner profile safety check failed',
|
||||
details: [
|
||||
`SubMiner could not update the runtime safety record at ${statePath}.`,
|
||||
(error as Error).message,
|
||||
'',
|
||||
'Yomitan storage was not loaded.',
|
||||
].join('\n'),
|
||||
statePath,
|
||||
};
|
||||
}
|
||||
|
||||
return { ok: true, statePath };
|
||||
}
|
||||
@@ -64,11 +64,17 @@ test('anki action main deps builders map callbacks', async () => {
|
||||
const mine = createBuildMineSentenceCardMainDepsHandler({
|
||||
getAnkiIntegration: () => ({ enabled: true }),
|
||||
getMpvClient: () => ({ connected: true }),
|
||||
getPrimarySubtitle: () => ({ text: '正式な字幕', startTime: 1, endTime: 3 }),
|
||||
showMpvOsd: (text) => calls.push(`mine:${text}`),
|
||||
mineSentenceCardCore: async () => true,
|
||||
recordCardsMined: (count) => calls.push(`cards:${count}`),
|
||||
})();
|
||||
assert.deepEqual(mine.getMpvClient(), { connected: true });
|
||||
assert.deepEqual(mine.getPrimarySubtitle?.(), {
|
||||
text: '正式な字幕',
|
||||
startTime: 1,
|
||||
endTime: 3,
|
||||
});
|
||||
mine.showMpvOsd('m');
|
||||
await mine.mineSentenceCardCore({
|
||||
ankiIntegration: { enabled: true },
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { createRefreshKnownWordCacheHandler } from './anki-actions';
|
||||
import type { createRefreshKnownWordCacheHandler, PrimarySubtitle } from './anki-actions';
|
||||
|
||||
type RefreshKnownWordCacheMainDeps = Parameters<typeof createRefreshKnownWordCacheHandler>[0];
|
||||
|
||||
@@ -72,10 +72,12 @@ export function createBuildMarkLastCardAsAudioCardMainDepsHandler<TAnki>(deps: {
|
||||
export function createBuildMineSentenceCardMainDepsHandler<TAnki, TMpv>(deps: {
|
||||
getAnkiIntegration: () => TAnki;
|
||||
getMpvClient: () => TMpv;
|
||||
getPrimarySubtitle?: () => PrimarySubtitle | null;
|
||||
showMpvOsd: (text: string) => void;
|
||||
mineSentenceCardCore: (options: {
|
||||
ankiIntegration: TAnki;
|
||||
mpvClient: TMpv;
|
||||
primarySubtitle?: PrimarySubtitle;
|
||||
showMpvOsd: (text: string) => void;
|
||||
}) => Promise<boolean>;
|
||||
recordCardsMined: (count: number, noteIds?: number[]) => void;
|
||||
@@ -83,10 +85,14 @@ export function createBuildMineSentenceCardMainDepsHandler<TAnki, TMpv>(deps: {
|
||||
return () => ({
|
||||
getAnkiIntegration: () => deps.getAnkiIntegration(),
|
||||
getMpvClient: () => deps.getMpvClient(),
|
||||
...(deps.getPrimarySubtitle
|
||||
? { getPrimarySubtitle: () => deps.getPrimarySubtitle?.() ?? null }
|
||||
: {}),
|
||||
showMpvOsd: (text: string) => deps.showMpvOsd(text),
|
||||
mineSentenceCardCore: (options: {
|
||||
ankiIntegration: TAnki;
|
||||
mpvClient: TMpv;
|
||||
primarySubtitle?: PrimarySubtitle;
|
||||
showMpvOsd: (text: string) => void;
|
||||
}) => deps.mineSentenceCardCore(options),
|
||||
recordCardsMined: (count: number, noteIds?: number[]) => deps.recordCardsMined(count, noteIds),
|
||||
|
||||
@@ -87,3 +87,20 @@ test('mine sentence handler records mined cards only when core returns true', as
|
||||
await mineSentenceCard();
|
||||
assert.deepEqual(calls, ['osd:mine', 'osd:mine', 'cards:1']);
|
||||
});
|
||||
|
||||
test('mine sentence handler forwards the canonical primary subtitle snapshot', async () => {
|
||||
const primarySubtitle = { text: '正式な字幕', startTime: 1, endTime: 3 };
|
||||
const mineSentenceCard = createMineSentenceCardHandler({
|
||||
getAnkiIntegration: () => ({}),
|
||||
getMpvClient: () => ({}),
|
||||
getPrimarySubtitle: () => primarySubtitle,
|
||||
showMpvOsd: () => {},
|
||||
mineSentenceCardCore: async (options) => {
|
||||
assert.equal(options.primarySubtitle, primarySubtitle);
|
||||
return true;
|
||||
},
|
||||
recordCardsMined: () => {},
|
||||
});
|
||||
|
||||
await mineSentenceCard();
|
||||
});
|
||||
|
||||
@@ -2,6 +2,12 @@ type AnkiIntegrationLike = {
|
||||
refreshKnownWordCache: () => Promise<void>;
|
||||
};
|
||||
|
||||
export type PrimarySubtitle = {
|
||||
text: string;
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
};
|
||||
|
||||
export function createUpdateLastCardFromClipboardHandler<TAnki>(deps: {
|
||||
getAnkiIntegration: () => TAnki;
|
||||
readClipboardText: () => string;
|
||||
@@ -69,18 +75,22 @@ export function createMarkLastCardAsAudioCardHandler<TAnki>(deps: {
|
||||
export function createMineSentenceCardHandler<TAnki, TMpv>(deps: {
|
||||
getAnkiIntegration: () => TAnki;
|
||||
getMpvClient: () => TMpv;
|
||||
getPrimarySubtitle?: () => PrimarySubtitle | null;
|
||||
showMpvOsd: (text: string) => void;
|
||||
mineSentenceCardCore: (options: {
|
||||
ankiIntegration: TAnki;
|
||||
mpvClient: TMpv;
|
||||
primarySubtitle?: PrimarySubtitle;
|
||||
showMpvOsd: (text: string) => void;
|
||||
}) => Promise<boolean>;
|
||||
recordCardsMined: (count: number, noteIds?: number[]) => void;
|
||||
}) {
|
||||
return async (): Promise<void> => {
|
||||
const primarySubtitle = deps.getPrimarySubtitle?.();
|
||||
const created = await deps.mineSentenceCardCore({
|
||||
ankiIntegration: deps.getAnkiIntegration(),
|
||||
mpvClient: deps.getMpvClient(),
|
||||
...(primarySubtitle ? { primarySubtitle } : {}),
|
||||
showMpvOsd: deps.showMpvOsd,
|
||||
});
|
||||
if (created) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { SubtitleCue, SubtitleData } from '../../types';
|
||||
import { selectAutoplayStartupCue } from './autoplay-subtitle-primer';
|
||||
import { primeVisibleOverlaySubtitleFromMpv } from './current-subtitle-snapshot';
|
||||
import { resolvePrimarySubtitleText } from './primary-subtitle-text';
|
||||
import { resolveSubtitleSourcePath } from './subtitle-prefetch-source';
|
||||
|
||||
const AUTOPLAY_SUBTITLE_PRIME_LOOKAHEAD_SECONDS = 2;
|
||||
@@ -141,6 +142,16 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
|
||||
return true;
|
||||
}
|
||||
|
||||
function resolveLivePrimarySubtitleText(text: string): string {
|
||||
const client = deps.getMpvClient();
|
||||
const currentTimeSec = Number(client?.currentTimePos ?? deps.getLastObservedTimePos());
|
||||
return resolvePrimarySubtitleText({
|
||||
liveText: text,
|
||||
currentTimeSec,
|
||||
cues: deps.getActiveParsedSubtitleCues(),
|
||||
});
|
||||
}
|
||||
|
||||
async function primeCurrentSubtitleForAutoplay(mediaPath: string): Promise<void> {
|
||||
const client = deps.getMpvClient();
|
||||
if (!client?.connected || !isCurrentAutoplayMediaPath(mediaPath)) {
|
||||
@@ -155,7 +166,8 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
|
||||
);
|
||||
return null;
|
||||
});
|
||||
const text = typeof subTextRaw === 'string' ? subTextRaw : '';
|
||||
const liveText = typeof subTextRaw === 'string' ? subTextRaw : '';
|
||||
const text = resolveLivePrimarySubtitleText(liveText);
|
||||
if (emitAutoplayPrimedSubtitle(mediaPath, text)) {
|
||||
return;
|
||||
}
|
||||
@@ -175,6 +187,7 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
|
||||
async function primeCurrentSubtitleForVisibleOverlay(): Promise<void> {
|
||||
await primeVisibleOverlaySubtitleFromMpv({
|
||||
getMpvClient: () => deps.getMpvClient(),
|
||||
resolvePrimarySubtitleText: (text) => resolveLivePrimarySubtitleText(text),
|
||||
setCurrentSubText: (text) => {
|
||||
deps.setCurrentSubText(text);
|
||||
},
|
||||
|
||||
@@ -46,6 +46,7 @@ export async function resolveCurrentSubtitleForRenderer(deps: {
|
||||
export async function primeVisibleOverlaySubtitleFromMpv(deps: {
|
||||
getMpvClient: () => CurrentSubtitleMpvClient | null;
|
||||
setCurrentSubText: (text: string) => void;
|
||||
resolvePrimarySubtitleText?: (text: string) => string;
|
||||
getCurrentSubtitleData: () => SubtitleData | null;
|
||||
consumeCachedSubtitle: (text: string) => SubtitleData | null;
|
||||
onSubtitleChange: (text: string) => void;
|
||||
@@ -73,7 +74,8 @@ export async function primeVisibleOverlaySubtitleFromMpv(deps: {
|
||||
return;
|
||||
}
|
||||
|
||||
const text = typeof subTextRaw === 'string' ? subTextRaw : '';
|
||||
const liveText = typeof subTextRaw === 'string' ? subTextRaw : '';
|
||||
const text = deps.resolvePrimarySubtitleText?.(liveText) ?? liveText;
|
||||
deps.setCurrentSubText(text);
|
||||
|
||||
const primeSecondarySubtitle = async (): Promise<void> => {
|
||||
|
||||
@@ -191,6 +191,8 @@ test('mpv event bindings register all expected events', () => {
|
||||
onSubtitleAssChange: () => {},
|
||||
onSecondarySubtitleChange: () => {},
|
||||
onSubtitleTrackChange: () => {},
|
||||
onSecondarySubtitleTrackChange: () => {},
|
||||
onSecondarySubtitleDelayChange: () => {},
|
||||
onSubtitleTrackListChange: () => {},
|
||||
onSubtitleTiming: () => {},
|
||||
onMediaPathChange: () => {},
|
||||
@@ -215,6 +217,8 @@ test('mpv event bindings register all expected events', () => {
|
||||
'subtitle-ass-change',
|
||||
'secondary-subtitle-change',
|
||||
'subtitle-track-change',
|
||||
'secondary-subtitle-track-change',
|
||||
'secondary-subtitle-delay-change',
|
||||
'subtitle-track-list-change',
|
||||
'subtitle-timing',
|
||||
'media-path-change',
|
||||
|
||||
@@ -4,6 +4,8 @@ type MpvBindingEventName =
|
||||
| 'subtitle-ass-change'
|
||||
| 'secondary-subtitle-change'
|
||||
| 'subtitle-track-change'
|
||||
| 'secondary-subtitle-track-change'
|
||||
| 'secondary-subtitle-delay-change'
|
||||
| 'subtitle-track-list-change'
|
||||
| 'subtitle-timing'
|
||||
| 'media-path-change'
|
||||
@@ -90,6 +92,8 @@ export function createBindMpvClientEventHandlers(deps: {
|
||||
onSubtitleAssChange: (payload: { text: string }) => void;
|
||||
onSecondarySubtitleChange: (payload: { text: string }) => void;
|
||||
onSubtitleTrackChange: (payload: { sid: number | null }) => void;
|
||||
onSecondarySubtitleTrackChange: (payload: { sid: number | null }) => void;
|
||||
onSecondarySubtitleDelayChange: (payload: { delay: number }) => void;
|
||||
onSubtitleTrackListChange: (payload: { trackList: unknown[] | null }) => void;
|
||||
onSubtitleTiming: (payload: { text: string; start: number; end: number }) => void;
|
||||
onMediaPathChange: (payload: { path: string | null }) => void;
|
||||
@@ -107,6 +111,8 @@ export function createBindMpvClientEventHandlers(deps: {
|
||||
mpvClient.on('subtitle-ass-change', deps.onSubtitleAssChange);
|
||||
mpvClient.on('secondary-subtitle-change', deps.onSecondarySubtitleChange);
|
||||
mpvClient.on('subtitle-track-change', deps.onSubtitleTrackChange);
|
||||
mpvClient.on('secondary-subtitle-track-change', deps.onSecondarySubtitleTrackChange);
|
||||
mpvClient.on('secondary-subtitle-delay-change', deps.onSecondarySubtitleDelayChange);
|
||||
mpvClient.on('subtitle-track-list-change', deps.onSubtitleTrackListChange);
|
||||
mpvClient.on('subtitle-timing', deps.onSubtitleTiming);
|
||||
mpvClient.on('media-path-change', deps.onMediaPathChange);
|
||||
|
||||
@@ -26,6 +26,29 @@ test('subtitle change handler updates state and forwards uncached text without r
|
||||
assert.deepEqual(calls, ['set:line', 'process:line', 'presence']);
|
||||
});
|
||||
|
||||
test('subtitle change handler consistently forwards resolved canonical text', () => {
|
||||
const calls: string[] = [];
|
||||
const handler = createHandleMpvSubtitleChangeHandler({
|
||||
resolveSubtitleText: () => '今 手にある物差しでは',
|
||||
setCurrentSubText: (text) => calls.push(`set:${text}`),
|
||||
getImmediateSubtitlePayload: (text) => {
|
||||
calls.push(`lookup:${text}`);
|
||||
return null;
|
||||
},
|
||||
broadcastSubtitle: () => {},
|
||||
onSubtitleChange: (text) => calls.push(`process:${text}`),
|
||||
refreshDiscordPresence: () => {},
|
||||
});
|
||||
|
||||
handler({ text: '今今今手手手ににに' });
|
||||
|
||||
assert.deepEqual(calls, [
|
||||
'set:今 手にある物差しでは',
|
||||
'lookup:今 手にある物差しでは',
|
||||
'process:今 手にある物差しでは',
|
||||
]);
|
||||
});
|
||||
|
||||
test('subtitle change handler clears immediately for empty subtitle text', () => {
|
||||
const calls: string[] = [];
|
||||
const handler = createHandleMpvSubtitleChangeHandler({
|
||||
|
||||
@@ -4,7 +4,8 @@ type AnilistPostWatchRunOptions = {
|
||||
watchedSeconds?: number;
|
||||
};
|
||||
|
||||
const SEEK_LIKE_TIME_DELTA_SECONDS = 2.5;
|
||||
/** Jump size that marks a time-pos change as a seek rather than normal playback. */
|
||||
export const SEEK_LIKE_TIME_DELTA_SECONDS = 2.5;
|
||||
|
||||
function isSeekLikeTimeChange(previousTime: number | null, nextTime: number): boolean {
|
||||
if (previousTime === null || !Number.isFinite(previousTime) || !Number.isFinite(nextTime)) {
|
||||
@@ -14,6 +15,7 @@ function isSeekLikeTimeChange(previousTime: number | null, nextTime: number): bo
|
||||
}
|
||||
|
||||
export function createHandleMpvSubtitleChangeHandler(deps: {
|
||||
resolveSubtitleText?: (text: string) => string;
|
||||
setCurrentSubText: (text: string) => void;
|
||||
getImmediateSubtitlePayload?: (text: string) => SubtitleData | null;
|
||||
emitImmediateSubtitle?: (payload: SubtitleData) => void;
|
||||
@@ -22,7 +24,8 @@ export function createHandleMpvSubtitleChangeHandler(deps: {
|
||||
refreshDiscordPresence: () => void;
|
||||
logDebug?: (message: string) => void;
|
||||
}) {
|
||||
return ({ text }: { text: string }): void => {
|
||||
return ({ text: liveText }: { text: string }): void => {
|
||||
const text = deps.resolveSubtitleText?.(liveText) ?? liveText;
|
||||
deps.setCurrentSubText(text);
|
||||
const immediatePayload = deps.getImmediateSubtitlePayload?.(text) ?? null;
|
||||
if (immediatePayload) {
|
||||
|
||||
@@ -37,6 +37,9 @@ test('main mpv event binder wires callbacks through to runtime deps', () => {
|
||||
broadcastSubtitleAss: (text) => calls.push(`broadcast-ass:${text}`),
|
||||
broadcastSecondarySubtitle: (text) => calls.push(`broadcast-secondary:${text}`),
|
||||
onSubtitleTrackChange: () => calls.push('subtitle-track-change'),
|
||||
onSecondarySubtitleTrackChange: () => calls.push('secondary-subtitle-track-change'),
|
||||
onSecondarySubtitleDelayChange: (delay) =>
|
||||
calls.push(`secondary-subtitle-delay-change:${delay}`),
|
||||
onSubtitleTrackListChange: () => calls.push('subtitle-track-list-change'),
|
||||
|
||||
updateCurrentMediaPath: (path) => calls.push(`media-path:${path}`),
|
||||
@@ -73,6 +76,8 @@ test('main mpv event binder wires callbacks through to runtime deps', () => {
|
||||
handlers.get('connection-change')?.({ connected: true });
|
||||
handlers.get('subtitle-change')?.({ text: 'line' });
|
||||
handlers.get('subtitle-track-change')?.({ sid: 3 });
|
||||
handlers.get('secondary-subtitle-track-change')?.({ sid: 4 });
|
||||
handlers.get('secondary-subtitle-delay-change')?.({ delay: 0.5 });
|
||||
handlers.get('subtitle-track-list-change')?.({ trackList: [] });
|
||||
handlers.get('media-path-change')?.({ path: '/tmp/video.mkv' });
|
||||
handlers.get('media-path-change')?.({ path: '' });
|
||||
@@ -86,6 +91,8 @@ test('main mpv event binder wires callbacks through to runtime deps', () => {
|
||||
assert.equal(calls.includes('broadcast-sub:line'), true);
|
||||
assert.ok(calls.includes('subtitle-change:line'));
|
||||
assert.ok(calls.includes('subtitle-track-change'));
|
||||
assert.ok(calls.includes('secondary-subtitle-track-change'));
|
||||
assert.ok(calls.includes('secondary-subtitle-delay-change:0.5'));
|
||||
assert.ok(calls.includes('subtitle-track-list-change'));
|
||||
assert.ok(calls.includes('media-title:Episode 1'));
|
||||
assert.ok(calls.includes('media-path:/tmp/video.mkv'));
|
||||
|
||||
@@ -43,6 +43,7 @@ export function createBindMpvMainEventHandlersHandler(deps: {
|
||||
logSubtitleTimingError: (message: string, error: unknown) => void;
|
||||
|
||||
setCurrentSubText: (text: string) => void;
|
||||
resolveSubtitleText?: (text: string) => string;
|
||||
getImmediateSubtitlePayload?: (text: string) => SubtitleData | null;
|
||||
emitImmediateSubtitle?: (payload: SubtitleData) => void;
|
||||
broadcastSubtitle: (payload: SubtitleData) => void;
|
||||
@@ -54,6 +55,8 @@ export function createBindMpvMainEventHandlersHandler(deps: {
|
||||
broadcastSubtitleAss: (text: string) => void;
|
||||
broadcastSecondarySubtitle: (text: string) => void;
|
||||
onSubtitleTrackChange?: (sid: number | null) => void;
|
||||
onSecondarySubtitleTrackChange?: (sid: number | null) => void;
|
||||
onSecondarySubtitleDelayChange?: (delay: number) => void;
|
||||
onSubtitleTrackListChange?: (trackList: unknown[] | null) => void;
|
||||
|
||||
updateCurrentMediaPath: (path: string) => void;
|
||||
@@ -117,6 +120,7 @@ export function createBindMpvMainEventHandlersHandler(deps: {
|
||||
logError: (message, error) => deps.logSubtitleTimingError(message, error),
|
||||
});
|
||||
const handleMpvSubtitleChange = createHandleMpvSubtitleChangeHandler({
|
||||
resolveSubtitleText: deps.resolveSubtitleText,
|
||||
setCurrentSubText: (text) => deps.setCurrentSubText(text),
|
||||
getImmediateSubtitlePayload: (text) => deps.getImmediateSubtitlePayload?.(text) ?? null,
|
||||
emitImmediateSubtitle: deps.emitImmediateSubtitle
|
||||
@@ -189,6 +193,8 @@ export function createBindMpvMainEventHandlersHandler(deps: {
|
||||
onSubtitleAssChange: handleMpvSubtitleAssChange,
|
||||
onSecondarySubtitleChange: handleMpvSecondarySubtitleChange,
|
||||
onSubtitleTrackChange: ({ sid }) => deps.onSubtitleTrackChange?.(sid),
|
||||
onSecondarySubtitleTrackChange: ({ sid }) => deps.onSecondarySubtitleTrackChange?.(sid),
|
||||
onSecondarySubtitleDelayChange: ({ delay }) => deps.onSecondarySubtitleDelayChange?.(delay),
|
||||
onSubtitleTrackListChange: ({ trackList }) => deps.onSubtitleTrackListChange?.(trackList),
|
||||
onSubtitleTiming: handleMpvSubtitleTiming,
|
||||
onMediaPathChange: handleMpvMediaPathChange,
|
||||
|
||||
@@ -47,6 +47,9 @@ test('mpv main event main deps map app state updates and delegate callbacks', as
|
||||
logSubtitleTimingError: (message) => calls.push(`subtitle-error:${message}`),
|
||||
broadcastToOverlayWindows: (channel, payload) =>
|
||||
calls.push(`broadcast:${channel}:${String(payload)}`),
|
||||
onSecondarySubtitleChange: (text) => calls.push(`secondary:${text}`),
|
||||
onSecondarySubtitleTrackChange: (sid) => calls.push(`secondary-track:${String(sid)}`),
|
||||
onSecondarySubtitleDelayChange: (delay) => calls.push(`secondary-delay:${delay}`),
|
||||
onSubtitleChange: (text) => calls.push(`subtitle-change:${text}`),
|
||||
ensureImmersionTrackerInitialized: () => calls.push('ensure-immersion'),
|
||||
updateCurrentMediaPath: (path) => calls.push(`path:${path}`),
|
||||
@@ -86,6 +89,8 @@ test('mpv main event main deps map app state updates and delegate callbacks', as
|
||||
deps.setCurrentSubAssText('ass');
|
||||
deps.broadcastSubtitleAss('ass');
|
||||
deps.broadcastSecondarySubtitle('sec');
|
||||
deps.onSecondarySubtitleTrackChange?.(4);
|
||||
deps.onSecondarySubtitleDelayChange?.(0.5);
|
||||
deps.updateCurrentMediaPath('/tmp/video');
|
||||
deps.restoreMpvSubVisibility();
|
||||
deps.resetSubtitleSidebarEmbeddedLayout();
|
||||
@@ -116,6 +121,10 @@ test('mpv main event main deps map app state updates and delegate callbacks', as
|
||||
assert.ok(calls.includes('sync-overlay-mpv-sub'));
|
||||
assert.ok(calls.includes('anilist-post-watch'));
|
||||
assert.ok(calls.includes('timing:y:secondary'));
|
||||
assert.ok(calls.includes('secondary:sec'));
|
||||
assert.ok(calls.includes('secondary-track:4'));
|
||||
assert.ok(calls.includes('secondary-delay:0.5'));
|
||||
assert.ok(!calls.includes('broadcast:secondary-subtitle:set:sec'));
|
||||
assert.ok(calls.includes('ensure-immersion'));
|
||||
assert.ok(calls.includes('sync-immersion'));
|
||||
assert.ok(calls.includes('autoplay:/tmp/video'));
|
||||
@@ -387,3 +396,170 @@ test('subtitle-track transitions ignore stale parsed cues until replacement cues
|
||||
handlers.recordImmersionSubtitleLine('飛び上がる', 20.04, 20.08);
|
||||
assert.deepEqual(recordedStarts.slice(-1), [20]);
|
||||
});
|
||||
|
||||
test('canonical ASS cues replace live glyph spam for display, history, and immersion', () => {
|
||||
const immersion: Array<{ text: string; start: number; end: number }> = [];
|
||||
const timing: Array<{ text: string; start: number; end: number }> = [];
|
||||
const handlers = createBuildBindMpvMainEventHandlersMainDepsHandler({
|
||||
appState: {
|
||||
initialArgs: null,
|
||||
overlayRuntimeInitialized: true,
|
||||
mpvClient: { currentTimePos: 2 },
|
||||
immersionTracker: {
|
||||
recordSubtitleLine: (text: string, start: number, end: number) =>
|
||||
immersion.push({ text, start, end }),
|
||||
},
|
||||
subtitleTimingTracker: {
|
||||
recordSubtitle: (text: string, start: number, end: number) =>
|
||||
timing.push({ text, start, end }),
|
||||
},
|
||||
activeParsedSubtitleCues: [
|
||||
{
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
text: '今 手にある物差しでは',
|
||||
source: 'canonical-ass',
|
||||
},
|
||||
{
|
||||
startTime: 3,
|
||||
endTime: 6,
|
||||
text: '飛び越えてみたくて',
|
||||
source: 'canonical-ass',
|
||||
},
|
||||
],
|
||||
currentMediaPath: '/video.mkv',
|
||||
currentSubText: '',
|
||||
currentSubAssText: '',
|
||||
playbackPaused: null,
|
||||
previousSecondarySubVisibility: false,
|
||||
},
|
||||
getQuitOnDisconnectArmed: () => false,
|
||||
scheduleQuitCheck: () => {},
|
||||
quitApp: () => {},
|
||||
reportJellyfinRemoteStopped: () => {},
|
||||
syncOverlayMpvSubtitleSuppression: () => {},
|
||||
maybeRunAnilistPostWatchUpdate: async () => {},
|
||||
logSubtitleTimingError: () => {},
|
||||
broadcastToOverlayWindows: () => {},
|
||||
onSubtitleChange: () => {},
|
||||
ensureImmersionTrackerInitialized: () => {},
|
||||
updateCurrentMediaPath: () => {},
|
||||
restoreMpvSubVisibility: () => {},
|
||||
getCurrentAnilistMediaKey: () => null,
|
||||
resetAnilistMediaTracking: () => {},
|
||||
maybeProbeAnilistDuration: () => {},
|
||||
ensureAnilistMediaGuess: () => {},
|
||||
syncImmersionMediaState: () => {},
|
||||
updateCurrentMediaTitle: () => {},
|
||||
resetAnilistMediaGuessState: () => {},
|
||||
reportJellyfinRemoteProgress: () => {},
|
||||
updateSubtitleRenderMetrics: () => {},
|
||||
refreshDiscordPresence: () => {},
|
||||
})();
|
||||
|
||||
assert.equal(handlers.resolveSubtitleText?.('今\n今\n今\n手\n手\n手'), '今 手にある物差しでは');
|
||||
handlers.recordImmersionSubtitleLine('今', 0.8, 1.5);
|
||||
handlers.recordImmersionSubtitleLine('手', 0.86, 1.56);
|
||||
handlers.recordSubtitleTiming('今', 0.8, 1.5);
|
||||
|
||||
assert.deepEqual(immersion, [{ text: '今 手にある物差しでは', start: 1.2, end: 3.8 }]);
|
||||
assert.deepEqual(timing, [{ text: '今 手にある物差しでは', start: 1.2, end: 3.8 }]);
|
||||
|
||||
// Concurrent dialogue during the song is not part of the animation: it must be
|
||||
// recorded as itself -- without the fragment lines beside it -- and must not cause
|
||||
// the song line to be recorded again when the animation frames resume.
|
||||
assert.equal(handlers.resolveSubtitleText?.('普通のセリフ\n今\n手'), '普通のセリフ\n今\n手');
|
||||
handlers.recordImmersionSubtitleLine('普通のセリフ\n今\n手', 1.9, 3.2);
|
||||
handlers.recordImmersionSubtitleLine('にある', 2.1, 2.9);
|
||||
handlers.recordSubtitleTiming('次のセリフ', 3.9, 5.0);
|
||||
|
||||
assert.deepEqual(immersion.slice(1), [{ text: '普通のセリフ', start: 1.9, end: 3.2 }]);
|
||||
assert.deepEqual(timing.slice(1), [{ text: '次のセリフ', start: 3.9, end: 5 }]);
|
||||
|
||||
// Overlapping canonical lines resolve as shifting subsets (A, then A+B, then A).
|
||||
// Every recorded cue is remembered, so each authored line still records exactly once.
|
||||
handlers.recordImmersionSubtitleLine('飛び越えて', 3.2, 3.4);
|
||||
handlers.recordImmersionSubtitleLine('手にある', 3.5, 3.7);
|
||||
handlers.recordSubtitleTiming('飛び越えて', 3.2, 3.4);
|
||||
handlers.recordSubtitleTiming('手にある', 3.5, 3.7);
|
||||
|
||||
assert.deepEqual(immersion.slice(2), [{ text: '飛び越えてみたくて', start: 3, end: 6 }]);
|
||||
assert.deepEqual(timing.slice(2), [{ text: '飛び越えてみたくて', start: 3, end: 6 }]);
|
||||
|
||||
// A backward seek means the user is rewatching: the timing history (a viewing log)
|
||||
// records the revisited line again, while immersion stays once-per-media.
|
||||
handlers.onTimePosUpdate?.(30);
|
||||
handlers.onTimePosUpdate?.(2);
|
||||
handlers.recordSubtitleTiming('今', 0.8, 1.5);
|
||||
handlers.recordImmersionSubtitleLine('今', 0.8, 1.5);
|
||||
|
||||
assert.deepEqual(timing.slice(3), [{ text: '今 手にある物差しでは', start: 1.2, end: 3.8 }]);
|
||||
assert.equal(immersion.length, 3);
|
||||
|
||||
// A jump of exactly the seek threshold counts as a seek, matching the time-pos
|
||||
// handler's own `>=` boundary.
|
||||
handlers.onTimePosUpdate?.(4.5);
|
||||
handlers.onTimePosUpdate?.(2);
|
||||
handlers.recordSubtitleTiming('今', 0.8, 1.5);
|
||||
|
||||
assert.deepEqual(timing.slice(4), [{ text: '今 手にある物差しでは', start: 1.2, end: 3.8 }]);
|
||||
});
|
||||
|
||||
test('subtitle-track changes stop stale canonical cues from substituting immediately', () => {
|
||||
const appState = {
|
||||
initialArgs: null,
|
||||
overlayRuntimeInitialized: true,
|
||||
mpvClient: { currentTimePos: 2 },
|
||||
immersionTracker: { recordSubtitleLine: () => {} },
|
||||
subtitleTimingTracker: { recordSubtitle: () => {} },
|
||||
activeParsedSubtitleCues: [
|
||||
{
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
text: '今 手にある物差しでは',
|
||||
source: 'canonical-ass' as const,
|
||||
},
|
||||
] as Array<{ startTime: number; endTime: number; text: string; source?: 'canonical-ass' }>,
|
||||
activeParsedSubtitleSource: 'track-a.ass' as string | null,
|
||||
currentMediaPath: '/video.mkv',
|
||||
currentSubText: '',
|
||||
currentSubAssText: '',
|
||||
playbackPaused: null,
|
||||
previousSecondarySubVisibility: false,
|
||||
};
|
||||
const handlers = createBuildBindMpvMainEventHandlersMainDepsHandler({
|
||||
appState,
|
||||
getQuitOnDisconnectArmed: () => false,
|
||||
scheduleQuitCheck: () => {},
|
||||
quitApp: () => {},
|
||||
reportJellyfinRemoteStopped: () => {},
|
||||
syncOverlayMpvSubtitleSuppression: () => {},
|
||||
maybeRunAnilistPostWatchUpdate: async () => {},
|
||||
logSubtitleTimingError: () => {},
|
||||
broadcastToOverlayWindows: () => {},
|
||||
onSubtitleChange: () => {},
|
||||
ensureImmersionTrackerInitialized: () => {},
|
||||
updateCurrentMediaPath: () => {},
|
||||
restoreMpvSubVisibility: () => {},
|
||||
getCurrentAnilistMediaKey: () => null,
|
||||
resetAnilistMediaTracking: () => {},
|
||||
maybeProbeAnilistDuration: () => {},
|
||||
ensureAnilistMediaGuess: () => {},
|
||||
syncImmersionMediaState: () => {},
|
||||
updateCurrentMediaTitle: () => {},
|
||||
resetAnilistMediaGuessState: () => {},
|
||||
reportJellyfinRemoteProgress: () => {},
|
||||
updateSubtitleRenderMetrics: () => {},
|
||||
refreshDiscordPresence: () => {},
|
||||
})();
|
||||
|
||||
assert.equal(handlers.resolveSubtitleText?.('今\n手にある'), '今 手にある物差しでは');
|
||||
|
||||
// The new track's cues arrive only after an async re-parse; until then, the old
|
||||
// track's canonical lyric must not replace the new track's live text.
|
||||
handlers.onSubtitleTrackChange?.(2);
|
||||
|
||||
assert.deepEqual(appState.activeParsedSubtitleCues, []);
|
||||
assert.equal(appState.activeParsedSubtitleSource, null);
|
||||
assert.equal(handlers.resolveSubtitleText?.('今\n手にある'), '今\n手にある');
|
||||
});
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
import { createSubtitleLineDedupGate } from '../../core/services/subtitle-line-dedup-gate';
|
||||
import type { MergedToken, SubtitleCue, SubtitleData } from '../../types';
|
||||
import { SEEK_LIKE_TIME_DELTA_SECONDS } from './mpv-main-event-actions';
|
||||
import {
|
||||
resolveCanonicalPrimarySubtitle,
|
||||
resolvePrimarySubtitleText,
|
||||
stripCanonicalFragmentLines,
|
||||
} from './primary-subtitle-text';
|
||||
|
||||
type AnilistPostWatchRunOptions = {
|
||||
watchedSeconds?: number;
|
||||
@@ -36,6 +42,8 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
|
||||
recordSubtitle?: (text: string, start: number, end: number, secondaryText?: string) => void;
|
||||
} | null;
|
||||
activeParsedSubtitleCues?: SubtitleCue[] | null;
|
||||
/** Cache key of the source the cues were parsed from; cleared with the cues. */
|
||||
activeParsedSubtitleSource?: string | null;
|
||||
currentMediaPath?: string | null;
|
||||
currentSubText: string;
|
||||
currentSubAssText: string;
|
||||
@@ -53,11 +61,14 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
|
||||
recordAnilistMediaDuration?: (durationSec: number) => void;
|
||||
logSubtitleTimingError: (message: string, error: unknown) => void;
|
||||
broadcastToOverlayWindows: (channel: string, payload: unknown) => void;
|
||||
onSecondarySubtitleChange?: (text: string) => void;
|
||||
getImmediateSubtitlePayload?: (text: string) => SubtitleData | null;
|
||||
emitImmediateSubtitle?: (payload: SubtitleData) => void;
|
||||
onSubtitleChange: (text: string) => void;
|
||||
logSubtitleProcessingDebug?: (message: string) => void;
|
||||
onSubtitleTrackChange?: (sid: number | null) => void;
|
||||
onSecondarySubtitleTrackChange?: (sid: number | null) => void;
|
||||
onSecondarySubtitleDelayChange?: (delay: number) => void;
|
||||
onSubtitleTrackListChange?: (trackList: unknown[] | null) => void;
|
||||
updateCurrentMediaPath: (path: string) => void;
|
||||
restoreMpvSubVisibility: () => void;
|
||||
@@ -93,6 +104,38 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
|
||||
const immersionLineDedupGate = createSubtitleLineDedupGate({
|
||||
getParsedCues: () => deps.appState.activeParsedSubtitleCues,
|
||||
});
|
||||
// One seen-set per consumer: canonical cues overlap, so live samples resolve to
|
||||
// shifting subsets (A, then A+B, then B). Remembering every recorded cue -- not just
|
||||
// the previous sample -- keeps each authored line recorded exactly once per source.
|
||||
const recordedImmersionCanonicalKeys = new Set<string>();
|
||||
const recordedTimingCanonicalKeys = new Set<string>();
|
||||
// Bumped on track/media changes so an immersion record whose tokenization resolves
|
||||
// after the change is dropped instead of landing in the next session.
|
||||
let subtitleSessionEpoch = 0;
|
||||
let lastTimePosForTimingReset: number | null = null;
|
||||
const canonicalCueKey = (cue: SubtitleCue): string =>
|
||||
`${cue.startTime}|${cue.endTime}|${cue.text}`;
|
||||
const resetSubtitleDeduplication = (): void => {
|
||||
immersionLineDedupGate.reset();
|
||||
recordedImmersionCanonicalKeys.clear();
|
||||
recordedTimingCanonicalKeys.clear();
|
||||
subtitleSessionEpoch += 1;
|
||||
lastTimePosForTimingReset = null;
|
||||
};
|
||||
const resolveCanonicalSample = (liveText: string, startSec: number) =>
|
||||
resolveCanonicalPrimarySubtitle({
|
||||
liveText,
|
||||
currentTimeSec: startSec,
|
||||
cues: deps.appState.activeParsedSubtitleCues,
|
||||
});
|
||||
// When substitution declined because dialogue shares the screen with a song, record
|
||||
// the dialogue alone rather than the combined dialogue-plus-fragments stack.
|
||||
const stripFragmentsForRecording = (liveText: string, startSec: number) =>
|
||||
stripCanonicalFragmentLines({
|
||||
liveText,
|
||||
currentTimeSec: startSec,
|
||||
cues: deps.appState.activeParsedSubtitleCues,
|
||||
});
|
||||
const hasInitialPlaybackQuitOnDisconnectArg = (): boolean =>
|
||||
Boolean(
|
||||
deps.appState.initialArgs?.managedPlayback ||
|
||||
@@ -111,45 +154,99 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
|
||||
scheduleQuitCheck: (callback: () => void) => deps.scheduleQuitCheck(callback),
|
||||
isMpvConnected: () => Boolean(deps.appState.mpvClient?.connected),
|
||||
quitApp: () => deps.quitApp(),
|
||||
resolveSubtitleText: (liveText: string) =>
|
||||
resolvePrimarySubtitleText({
|
||||
liveText,
|
||||
currentTimeSec: Number(deps.appState.mpvClient?.currentTimePos),
|
||||
cues: deps.appState.activeParsedSubtitleCues,
|
||||
}),
|
||||
recordImmersionSubtitleLine: (text: string, start: number, end: number) => {
|
||||
deps.ensureImmersionTrackerInitialized();
|
||||
const tracker = deps.appState.immersionTracker;
|
||||
if (!tracker?.recordSubtitleLine) {
|
||||
return;
|
||||
}
|
||||
const recordLine = (lineText: string, startSec: number, endSec: number): void => {
|
||||
const secondaryText = deps.appState.mpvClient?.currentSecondarySubText || null;
|
||||
const cachedTokens =
|
||||
deps.appState.currentSubtitleData?.text === lineText
|
||||
? deps.appState.currentSubtitleData.tokens
|
||||
: null;
|
||||
if (cachedTokens) {
|
||||
tracker.recordSubtitleLine?.(lineText, startSec, endSec, cachedTokens, secondaryText);
|
||||
return;
|
||||
}
|
||||
if (!deps.tokenizeSubtitleForImmersion) {
|
||||
tracker.recordSubtitleLine?.(lineText, startSec, endSec, null, secondaryText);
|
||||
return;
|
||||
}
|
||||
const epochAtRecord = subtitleSessionEpoch;
|
||||
void deps
|
||||
.tokenizeSubtitleForImmersion(lineText)
|
||||
.then((payload) => {
|
||||
if (subtitleSessionEpoch !== epochAtRecord) {
|
||||
return;
|
||||
}
|
||||
tracker.recordSubtitleLine?.(
|
||||
lineText,
|
||||
startSec,
|
||||
endSec,
|
||||
payload?.tokens ?? null,
|
||||
secondaryText,
|
||||
);
|
||||
})
|
||||
.catch(() => {
|
||||
if (subtitleSessionEpoch !== epochAtRecord) {
|
||||
return;
|
||||
}
|
||||
tracker.recordSubtitleLine?.(lineText, startSec, endSec, null, secondaryText);
|
||||
});
|
||||
};
|
||||
const canonical = resolveCanonicalSample(text, start);
|
||||
if (canonical) {
|
||||
for (const cue of canonical.cues) {
|
||||
const key = canonicalCueKey(cue);
|
||||
if (recordedImmersionCanonicalKeys.has(key)) {
|
||||
continue;
|
||||
}
|
||||
recordedImmersionCanonicalKeys.add(key);
|
||||
recordLine(cue.text, cue.startTime, cue.endTime);
|
||||
}
|
||||
return;
|
||||
}
|
||||
text = stripFragmentsForRecording(text, start);
|
||||
if (!immersionLineDedupGate.shouldRecord({ text, startSec: start, endSec: end })) {
|
||||
return;
|
||||
}
|
||||
const secondaryText = deps.appState.mpvClient?.currentSecondarySubText || null;
|
||||
const cachedTokens =
|
||||
deps.appState.currentSubtitleData?.text === text
|
||||
? deps.appState.currentSubtitleData.tokens
|
||||
: null;
|
||||
if (cachedTokens) {
|
||||
tracker.recordSubtitleLine(text, start, end, cachedTokens, secondaryText);
|
||||
return;
|
||||
}
|
||||
if (!deps.tokenizeSubtitleForImmersion) {
|
||||
tracker.recordSubtitleLine(text, start, end, null, secondaryText);
|
||||
return;
|
||||
}
|
||||
void deps
|
||||
.tokenizeSubtitleForImmersion(text)
|
||||
.then((payload) => {
|
||||
tracker.recordSubtitleLine?.(text, start, end, payload?.tokens ?? null, secondaryText);
|
||||
})
|
||||
.catch(() => {
|
||||
tracker.recordSubtitleLine?.(text, start, end, null, secondaryText);
|
||||
});
|
||||
recordLine(text, start, end);
|
||||
},
|
||||
hasSubtitleTimingTracker: () => Boolean(deps.appState.subtitleTimingTracker),
|
||||
recordSubtitleTiming: (text: string, start: number, end: number) =>
|
||||
deps.appState.subtitleTimingTracker?.recordSubtitle?.(
|
||||
text,
|
||||
start,
|
||||
end,
|
||||
deps.appState.mpvClient?.currentSecondarySubText || undefined,
|
||||
),
|
||||
recordSubtitleTiming: (text: string, start: number, end: number) => {
|
||||
const secondaryText = deps.appState.mpvClient?.currentSecondarySubText || undefined;
|
||||
const canonical = resolveCanonicalSample(text, start);
|
||||
if (!canonical) {
|
||||
deps.appState.subtitleTimingTracker?.recordSubtitle?.(
|
||||
stripFragmentsForRecording(text, start),
|
||||
start,
|
||||
end,
|
||||
secondaryText,
|
||||
);
|
||||
return;
|
||||
}
|
||||
for (const cue of canonical.cues) {
|
||||
const key = canonicalCueKey(cue);
|
||||
if (recordedTimingCanonicalKeys.has(key)) {
|
||||
continue;
|
||||
}
|
||||
recordedTimingCanonicalKeys.add(key);
|
||||
deps.appState.subtitleTimingTracker?.recordSubtitle?.(
|
||||
cue.text,
|
||||
cue.startTime,
|
||||
cue.endTime,
|
||||
secondaryText,
|
||||
);
|
||||
}
|
||||
},
|
||||
maybeRunAnilistPostWatchUpdate: (options?: AnilistPostWatchRunOptions) =>
|
||||
deps.maybeRunAnilistPostWatchUpdate(options),
|
||||
logSubtitleTimingError: (message: string, error: unknown) =>
|
||||
@@ -170,9 +267,22 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
|
||||
? (message: string) => deps.logSubtitleProcessingDebug!(message)
|
||||
: undefined,
|
||||
onSubtitleTrackChange: (sid: number | null) => {
|
||||
immersionLineDedupGate.reset();
|
||||
resetSubtitleDeduplication();
|
||||
// The replacement track's cues arrive only after an async re-read and re-parse.
|
||||
// Clearing synchronously keeps the previous track's canonical cues from
|
||||
// substituting into, or recording against, the new track's live text. The source
|
||||
// key is cleared with the cues so cue-list consumers (the sidebar snapshot)
|
||||
// re-parse on demand instead of trusting the stale pairing.
|
||||
deps.appState.activeParsedSubtitleCues = [];
|
||||
deps.appState.activeParsedSubtitleSource = null;
|
||||
deps.onSubtitleTrackChange?.(sid);
|
||||
},
|
||||
onSecondarySubtitleTrackChange: deps.onSecondarySubtitleTrackChange
|
||||
? (sid: number | null) => deps.onSecondarySubtitleTrackChange!(sid)
|
||||
: undefined,
|
||||
onSecondarySubtitleDelayChange: deps.onSecondarySubtitleDelayChange
|
||||
? (delay: number) => deps.onSecondarySubtitleDelayChange!(delay)
|
||||
: undefined,
|
||||
onSubtitleTrackListChange: deps.onSubtitleTrackListChange
|
||||
? (trackList: unknown[] | null) => deps.onSubtitleTrackListChange!(trackList)
|
||||
: undefined,
|
||||
@@ -182,10 +292,15 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
|
||||
},
|
||||
broadcastSubtitleAss: (text: string) =>
|
||||
deps.broadcastToOverlayWindows('subtitle-ass:set', text),
|
||||
broadcastSecondarySubtitle: (text: string) =>
|
||||
deps.broadcastToOverlayWindows('secondary-subtitle:set', text),
|
||||
broadcastSecondarySubtitle: (text: string) => {
|
||||
if (deps.onSecondarySubtitleChange) {
|
||||
deps.onSecondarySubtitleChange(text);
|
||||
return;
|
||||
}
|
||||
deps.broadcastToOverlayWindows('secondary-subtitle:set', text);
|
||||
},
|
||||
updateCurrentMediaPath: (path: string) => {
|
||||
immersionLineDedupGate.reset();
|
||||
resetSubtitleDeduplication();
|
||||
deps.updateCurrentMediaPath(path);
|
||||
},
|
||||
restoreMpvSubVisibility: () => deps.restoreMpvSubVisibility(),
|
||||
@@ -217,9 +332,22 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
|
||||
},
|
||||
reportJellyfinRemoteProgress: (forceImmediate: boolean) =>
|
||||
deps.reportJellyfinRemoteProgress(forceImmediate),
|
||||
onTimePosUpdate: deps.onTimePosUpdate
|
||||
? (time: number) => deps.onTimePosUpdate!(time)
|
||||
: undefined,
|
||||
onTimePosUpdate: (time: number) => {
|
||||
// Timing history is a viewing log: after a real backward seek, a rewatched
|
||||
// canonical line should enter it again. Immersion stats keep their
|
||||
// once-per-media deduplication and are not reset here.
|
||||
if (
|
||||
Number.isFinite(time) &&
|
||||
lastTimePosForTimingReset !== null &&
|
||||
time <= lastTimePosForTimingReset - SEEK_LIKE_TIME_DELTA_SECONDS
|
||||
) {
|
||||
recordedTimingCanonicalKeys.clear();
|
||||
}
|
||||
if (Number.isFinite(time)) {
|
||||
lastTimePosForTimingReset = time;
|
||||
}
|
||||
deps.onTimePosUpdate?.(time);
|
||||
},
|
||||
onFullscreenChange: deps.onFullscreenChange
|
||||
? (fullscreen: boolean) => deps.onFullscreenChange!(fullscreen)
|
||||
: undefined,
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { parseSubtitleCues } from '../../core/services/subtitle-cue-parser';
|
||||
import {
|
||||
resolveCanonicalPrimarySubtitle,
|
||||
resolvePrimarySubtitleText,
|
||||
stripCanonicalFragmentLines,
|
||||
} from './primary-subtitle-text';
|
||||
|
||||
test('resolvePrimarySubtitleText collapses full-span ASS style layers through parsed cues', () => {
|
||||
const ass = [
|
||||
'[Events]',
|
||||
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
|
||||
'Dialogue: 3,0:22:40.05,0:22:45.76,EDJP,,0,0,0,,{\\fad(400,400)\\bord0\\blur0.8}鏡の奥まで目を凝らして',
|
||||
'Dialogue: 2,0:22:40.05,0:22:45.76,EDJP,,0,0,0,,{\\fad(400,400)}鏡の奥まで目を凝らして',
|
||||
'Dialogue: 1,0:22:40.05,0:22:45.76,EDJP,,0,0,0,,{\\fad(400,400)\\bord6}鏡の奥まで目を凝らして',
|
||||
'Dialogue: 0,0:22:40.05,0:22:45.76,EDJP,,0,0,0,,{\\fad(400,400)\\bord8\\blur4}鏡の奥まで目を凝らして',
|
||||
].join('\n');
|
||||
const cues = parseSubtitleCues(ass, 'polar-opposites-s01e08.ass');
|
||||
|
||||
assert.deepEqual(cues, [
|
||||
{ startTime: 22 * 60 + 40.05, endTime: 22 * 60 + 45.76, text: '鏡の奥まで目を凝らして' },
|
||||
]);
|
||||
|
||||
assert.equal(
|
||||
resolvePrimarySubtitleText({
|
||||
liveText: [
|
||||
'鏡の奥まで目を凝らして',
|
||||
'鏡の奥まで目を凝らして',
|
||||
'鏡の奥まで目を凝らして',
|
||||
'鏡の奥まで目を凝らして',
|
||||
].join('\n'),
|
||||
currentTimeSec: 22 * 60 + 44,
|
||||
cues,
|
||||
}),
|
||||
'鏡の奥まで目を凝らして',
|
||||
);
|
||||
});
|
||||
|
||||
test('resolvePrimarySubtitleText keeps live text when active parsed cues do not explain it all', () => {
|
||||
const liveText = '普通のセリフ\n鏡の奥まで目を凝らして\n鏡の奥まで目を凝らして';
|
||||
|
||||
assert.equal(
|
||||
resolvePrimarySubtitleText({
|
||||
liveText,
|
||||
currentTimeSec: 2,
|
||||
cues: [{ startTime: 1, endTime: 3, text: '鏡の奥まで目を凝らして' }],
|
||||
}),
|
||||
liveText,
|
||||
);
|
||||
});
|
||||
|
||||
test('resolvePrimarySubtitleText combines unique simultaneous parsed cues', () => {
|
||||
assert.equal(
|
||||
resolvePrimarySubtitleText({
|
||||
liveText: '一行目\n一行目\n二行目\n二行目',
|
||||
currentTimeSec: 2,
|
||||
cues: [
|
||||
{ startTime: 1, endTime: 3, text: '一行目' },
|
||||
{ startTime: 1, endTime: 3, text: '二行目' },
|
||||
],
|
||||
}),
|
||||
'一行目\n二行目',
|
||||
);
|
||||
});
|
||||
|
||||
test('resolvePrimarySubtitleText tolerates stale time-pos at a parsed cue edge', () => {
|
||||
assert.equal(
|
||||
resolvePrimarySubtitleText({
|
||||
liveText: '新しい行\n新しい行',
|
||||
currentTimeSec: 0.8,
|
||||
cues: [{ startTime: 1, endTime: 3, text: '新しい行' }],
|
||||
}),
|
||||
'新しい行',
|
||||
);
|
||||
});
|
||||
|
||||
test('resolvePrimarySubtitleText prefers an active canonical cue over flattened mpv glyphs', () => {
|
||||
// mpv renders each simultaneously active ASS event on its own sub-text line.
|
||||
const text = resolvePrimarySubtitleText({
|
||||
liveText: '今\n今\n今\n手\n手\n手\nにある\nにある\nにある',
|
||||
currentTimeSec: 2,
|
||||
cues: [
|
||||
{
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
text: '今 手にある',
|
||||
source: 'canonical-ass',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
assert.equal(text, '今 手にある');
|
||||
});
|
||||
|
||||
test('resolvePrimarySubtitleText preserves live text outside canonical cue timing', () => {
|
||||
const text = resolvePrimarySubtitleText({
|
||||
liveText: '通常の会話',
|
||||
currentTimeSec: 8,
|
||||
cues: [
|
||||
{
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
text: '今 手にある',
|
||||
source: 'canonical-ass',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
assert.equal(text, '通常の会話');
|
||||
});
|
||||
|
||||
test('resolvePrimarySubtitleText keeps concurrent dialogue that is not part of the animation', () => {
|
||||
// An insert song's canonical window can overlap real dialogue on the same track.
|
||||
const text = resolvePrimarySubtitleText({
|
||||
liveText: '普通のセリフ\n今\n手にある',
|
||||
currentTimeSec: 2,
|
||||
cues: [
|
||||
{
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
text: '今 手にある',
|
||||
source: 'canonical-ass',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
assert.equal(text, '普通のセリフ\n今\n手にある');
|
||||
});
|
||||
|
||||
test('resolvePrimarySubtitleText keeps a fresh line starting just after the animation ended', () => {
|
||||
const text = resolvePrimarySubtitleText({
|
||||
liveText: '次のセリフ',
|
||||
currentTimeSec: 4.1,
|
||||
cues: [
|
||||
{
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
text: '今 手にある',
|
||||
source: 'canonical-ass',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
assert.equal(text, '次のセリフ');
|
||||
});
|
||||
|
||||
test('resolvePrimarySubtitleText survives overlapping frames of consecutive karaoke lines', () => {
|
||||
// Near a line boundary the previous line's exit frames and the next line's entrance
|
||||
// frames render together; neither line alone explains every live segment.
|
||||
const cues = [
|
||||
{ startTime: 1.2, endTime: 3.8, text: '今 手にある', source: 'canonical-ass' as const },
|
||||
{ startTime: 3.8, endTime: 6.4, text: '物差しでは', source: 'canonical-ass' as const },
|
||||
];
|
||||
|
||||
assert.equal(
|
||||
resolvePrimarySubtitleText({
|
||||
liveText: '手にある\n物差し\nでは',
|
||||
currentTimeSec: 3.6,
|
||||
cues,
|
||||
}),
|
||||
'今 手にある',
|
||||
);
|
||||
assert.equal(
|
||||
resolvePrimarySubtitleText({
|
||||
liveText: '手にある\n物差し\nでは',
|
||||
currentTimeSec: 3.9,
|
||||
cues,
|
||||
}),
|
||||
'物差しでは',
|
||||
);
|
||||
});
|
||||
|
||||
test('resolvePrimarySubtitleText combines simultaneous canonical cues in source order', () => {
|
||||
const text = resolvePrimarySubtitleText({
|
||||
liveText: 'fir\nst\nsecond',
|
||||
currentTimeSec: 2,
|
||||
cues: [
|
||||
{ startTime: 1, endTime: 3, text: 'first', source: 'canonical-ass' },
|
||||
{ startTime: 1.5, endTime: 2.5, text: 'second', source: 'canonical-ass' },
|
||||
],
|
||||
});
|
||||
|
||||
assert.equal(text, 'first\nsecond');
|
||||
});
|
||||
|
||||
test('resolveCanonicalPrimarySubtitle covers a nearby generated animation edge', () => {
|
||||
const cue = {
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
text: '今 手にある',
|
||||
source: 'canonical-ass' as const,
|
||||
};
|
||||
const resolved = resolveCanonicalPrimarySubtitle({
|
||||
liveText: '今\n手にある',
|
||||
currentTimeSec: 0.8,
|
||||
cues: [cue],
|
||||
});
|
||||
|
||||
assert.deepEqual(resolved, {
|
||||
text: '今 手にある',
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
cues: [cue],
|
||||
});
|
||||
});
|
||||
|
||||
test('resolveCanonicalPrimarySubtitle covers exit frames that outlive the authored timing', () => {
|
||||
// Real generated animations keep exit fragments on screen well past the authored
|
||||
// comment window; the recorded animation envelope is what makes them resolvable.
|
||||
const cue = {
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
text: '今 手にある',
|
||||
source: 'canonical-ass' as const,
|
||||
animationStartTime: 0.8,
|
||||
animationEndTime: 5.6,
|
||||
};
|
||||
const resolved = resolveCanonicalPrimarySubtitle({
|
||||
liveText: '今\n手にある',
|
||||
currentTimeSec: 5.4,
|
||||
cues: [cue],
|
||||
});
|
||||
|
||||
assert.deepEqual(resolved, {
|
||||
text: '今 手にある',
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
cues: [cue],
|
||||
});
|
||||
});
|
||||
|
||||
test('resolvePrimarySubtitleText handles late exit frames overlapping the next active line', () => {
|
||||
// The previous line's exit fragments can persist more than a second into the next
|
||||
// authored line. The next line supplies the text; the previous line's envelope
|
||||
// explains its lingering fragments.
|
||||
const cues = [
|
||||
{
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
text: '今 手にある',
|
||||
source: 'canonical-ass' as const,
|
||||
animationStartTime: 0.8,
|
||||
animationEndTime: 5.6,
|
||||
},
|
||||
{
|
||||
startTime: 3.8,
|
||||
endTime: 6.4,
|
||||
text: '物差しでは',
|
||||
source: 'canonical-ass' as const,
|
||||
animationStartTime: 3.4,
|
||||
animationEndTime: 7.0,
|
||||
},
|
||||
];
|
||||
|
||||
assert.equal(
|
||||
resolvePrimarySubtitleText({
|
||||
liveText: '手にある\n手にある\n物差し\nでは',
|
||||
currentTimeSec: 5.2,
|
||||
cues,
|
||||
}),
|
||||
'物差しでは',
|
||||
);
|
||||
});
|
||||
|
||||
test('resolveCanonicalPrimarySubtitle rejects unrelated live text at the animation edge', () => {
|
||||
const resolved = resolveCanonicalPrimarySubtitle({
|
||||
liveText: '次のセリフ',
|
||||
currentTimeSec: 4.1,
|
||||
cues: [
|
||||
{
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
text: '今 手にある',
|
||||
source: 'canonical-ass',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
assert.equal(resolved, null);
|
||||
});
|
||||
|
||||
test('stripCanonicalFragmentLines drops fragment lines but keeps concurrent dialogue', () => {
|
||||
const cues = [
|
||||
{
|
||||
startTime: 1.2,
|
||||
endTime: 3.8,
|
||||
text: '今 手にある',
|
||||
source: 'canonical-ass' as const,
|
||||
},
|
||||
];
|
||||
|
||||
assert.equal(
|
||||
stripCanonicalFragmentLines({
|
||||
liveText: '普通のセリフ\n今\n手にある',
|
||||
currentTimeSec: 2,
|
||||
cues,
|
||||
}),
|
||||
'普通のセリフ',
|
||||
);
|
||||
// No canonical cue nearby: nothing to strip.
|
||||
assert.equal(
|
||||
stripCanonicalFragmentLines({ liveText: '普通のセリフ\n今', currentTimeSec: 30, cues }),
|
||||
'普通のセリフ\n今',
|
||||
);
|
||||
// Everything matched (defensive): return the input rather than empty text.
|
||||
assert.equal(
|
||||
stripCanonicalFragmentLines({ liveText: '今\n手にある', currentTimeSec: 2, cues }),
|
||||
'今\n手にある',
|
||||
);
|
||||
});
|
||||
|
||||
test('resolveCanonicalPrimarySubtitle picks the cue its fragments spell, not the nearest', () => {
|
||||
// In the gap between two authored spans, the next line sits closer in time while only
|
||||
// the previous line's exit fragments are on screen: the fragments decide.
|
||||
const cues = [
|
||||
{
|
||||
startTime: 1,
|
||||
endTime: 3,
|
||||
text: '今 手にある',
|
||||
source: 'canonical-ass' as const,
|
||||
animationStartTime: 0.6,
|
||||
animationEndTime: 3.9,
|
||||
},
|
||||
{
|
||||
startTime: 4,
|
||||
endTime: 6,
|
||||
text: '物差しでは',
|
||||
source: 'canonical-ass' as const,
|
||||
animationStartTime: 3.5,
|
||||
animationEndTime: 6.4,
|
||||
},
|
||||
];
|
||||
|
||||
assert.equal(
|
||||
resolveCanonicalPrimarySubtitle({ liveText: '手にある', currentTimeSec: 3.8, cues })?.text,
|
||||
'今 手にある',
|
||||
);
|
||||
// Fragments of both lines in the gap: both envelopes cover the moment (distance 0),
|
||||
// and the earlier line wins the tie while it is still animating out.
|
||||
assert.equal(
|
||||
resolveCanonicalPrimarySubtitle({ liveText: '手にある\n物差し', currentTimeSec: 3.8, cues })
|
||||
?.text,
|
||||
'今 手にある',
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,221 @@
|
||||
import type { SubtitleCue } from '../../types';
|
||||
|
||||
// Slack on top of each cue's recorded animation envelope, for time-pos observation
|
||||
// staleness and small user sub-delay offsets. The envelope itself covers how far
|
||||
// entrance/exit frames actually run past the authored timing.
|
||||
const LIVE_CUE_EDGE_TOLERANCE_SECONDS = 1;
|
||||
|
||||
export interface ResolvedPrimarySubtitle {
|
||||
text: string;
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
/** The parsed cues behind `text`, for consumers that record lines individually. */
|
||||
cues: SubtitleCue[];
|
||||
}
|
||||
|
||||
function animationSpan(cue: SubtitleCue): { start: number; end: number } {
|
||||
return {
|
||||
start: cue.animationStartTime ?? cue.startTime,
|
||||
end: cue.animationEndTime ?? cue.endTime,
|
||||
};
|
||||
}
|
||||
|
||||
function nearbyCanonicalCues(
|
||||
cues: readonly SubtitleCue[] | null | undefined,
|
||||
currentTimeSec: number,
|
||||
): SubtitleCue[] {
|
||||
return (cues ?? []).filter((cue) => {
|
||||
if (cue.source !== 'canonical-ass') {
|
||||
return false;
|
||||
}
|
||||
const span = animationSpan(cue);
|
||||
return (
|
||||
span.end >= currentTimeSec - LIVE_CUE_EDGE_TOLERANCE_SECONDS &&
|
||||
span.start <= currentTimeSec + LIVE_CUE_EDGE_TOLERANCE_SECONDS
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function compactWhitespace(text: string): string {
|
||||
return text.replace(/\s+/gu, '');
|
||||
}
|
||||
|
||||
function compactLineSegments(text: string): string[] {
|
||||
return text.split('\n').map(compactWhitespace).filter(Boolean);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parsed cues have already collapsed exact ASS layers and animation runs. Trust that
|
||||
* cleaner view only when every live mpv line is accounted for by an active parsed cue.
|
||||
* This keeps unrelated concurrent dialogue on the live fallback while removing style
|
||||
* stacks where mpv repeats one full lyric for fill, border, blur, and shadow layers.
|
||||
*/
|
||||
function resolveActiveParsedPrimarySubtitle(options: {
|
||||
liveText: string;
|
||||
currentTimeSec: number;
|
||||
cues: readonly SubtitleCue[] | null | undefined;
|
||||
}): ResolvedPrimarySubtitle | null {
|
||||
if (!Number.isFinite(options.currentTimeSec)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const liveSegments = compactLineSegments(options.liveText);
|
||||
if (liveSegments.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const liveSegmentSet = new Set(liveSegments);
|
||||
const selected = (options.cues ?? []).filter((cue) => {
|
||||
if (
|
||||
cue.startTime > options.currentTimeSec + LIVE_CUE_EDGE_TOLERANCE_SECONDS ||
|
||||
cue.endTime <= options.currentTimeSec - LIVE_CUE_EDGE_TOLERANCE_SECONDS
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const cueSegments = compactLineSegments(cue.text);
|
||||
return cueSegments.length > 0 && cueSegments.every((segment) => liveSegmentSet.has(segment));
|
||||
});
|
||||
if (selected.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsedSegmentSet = new Set(selected.flatMap((cue) => compactLineSegments(cue.text)));
|
||||
if (!liveSegments.every((segment) => parsedSegmentSet.has(segment))) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const texts: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const cue of selected) {
|
||||
if (!seen.has(cue.text)) {
|
||||
seen.add(cue.text);
|
||||
texts.push(cue.text);
|
||||
}
|
||||
}
|
||||
return {
|
||||
text: texts.join('\n'),
|
||||
startTime: Math.min(...selected.map((cue) => cue.startTime)),
|
||||
endTime: Math.max(...selected.map((cue) => cue.endTime)),
|
||||
cues: selected,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* mpv's `sub-text` renders each simultaneously active ASS event on its own line, so
|
||||
* while a generated animation plays every live line is a contiguous piece of the
|
||||
* authored text. A line that is not -- concurrent dialogue during an insert song, or a
|
||||
* fresh line starting just after the animation ended -- proves the live text is not this
|
||||
* animation, and substituting the canonical line would swallow real dialogue.
|
||||
*/
|
||||
function liveTextIsFromCues(liveText: string, cues: readonly SubtitleCue[]): boolean {
|
||||
const compactCues = cues.map((cue) => compactWhitespace(cue.text));
|
||||
const segments = liveText.split('\n').map(compactWhitespace).filter(Boolean);
|
||||
return (
|
||||
segments.length > 0 &&
|
||||
segments.every((segment) => compactCues.some((cueText) => cueText.includes(segment)))
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveCanonicalPrimarySubtitle(options: {
|
||||
liveText: string;
|
||||
currentTimeSec: number;
|
||||
cues: readonly SubtitleCue[] | null | undefined;
|
||||
}): ResolvedPrimarySubtitle | null {
|
||||
if (!Number.isFinite(options.currentTimeSec)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Consecutive karaoke lines overlap: one line's exit frames are still on screen while
|
||||
// the next line's entrance frames appear. The fragment check therefore runs against
|
||||
// every canonical cue whose animation envelope reaches the current time, while only
|
||||
// the active (or single nearest) cue supplies the displayed text.
|
||||
const nearby = nearbyCanonicalCues(options.cues, options.currentTimeSec);
|
||||
const active = nearby.filter(
|
||||
(cue) => cue.startTime <= options.currentTimeSec && cue.endTime > options.currentTimeSec,
|
||||
);
|
||||
const liveSegments = options.liveText.split('\n').map(compactWhitespace).filter(Boolean);
|
||||
const selected =
|
||||
active.length > 0
|
||||
? active
|
||||
: nearby
|
||||
// Between authored spans, proximity alone can pick the wrong neighbor: the
|
||||
// next line can sit closer while only the previous line's exit fragments are
|
||||
// on screen. Only cues that explain at least one live line may be selected.
|
||||
.filter((cue) => {
|
||||
const cueText = compactWhitespace(cue.text);
|
||||
return liveSegments.some((segment) => cueText.includes(segment));
|
||||
})
|
||||
.map((cue) => {
|
||||
const span = animationSpan(cue);
|
||||
const distance =
|
||||
options.currentTimeSec < span.start
|
||||
? span.start - options.currentTimeSec
|
||||
: Math.max(0, options.currentTimeSec - span.end);
|
||||
return { cue, distance };
|
||||
})
|
||||
.sort((a, b) => a.distance - b.distance || a.cue.startTime - b.cue.startTime)
|
||||
.slice(0, 1)
|
||||
.map(({ cue }) => cue);
|
||||
if (selected.length === 0 || !liveTextIsFromCues(options.liveText, nearby)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const texts: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const cue of selected) {
|
||||
if (!seen.has(cue.text)) {
|
||||
seen.add(cue.text);
|
||||
texts.push(cue.text);
|
||||
}
|
||||
}
|
||||
return {
|
||||
text: texts.join('\n'),
|
||||
startTime: Math.min(...selected.map((cue) => cue.startTime)),
|
||||
endTime: Math.max(...selected.map((cue) => cue.endTime)),
|
||||
cues: selected,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Live text with generated-animation fragment lines removed. Recording paths use this
|
||||
* when full canonical substitution declined -- concurrent dialogue during an insert
|
||||
* song: the dialogue is worth recording, the glyph fragments beside it are not. Returns
|
||||
* the input unchanged when no canonical cue is near or nothing non-fragment remains.
|
||||
*/
|
||||
export function stripCanonicalFragmentLines(options: {
|
||||
liveText: string;
|
||||
currentTimeSec: number;
|
||||
cues: readonly SubtitleCue[] | null | undefined;
|
||||
}): string {
|
||||
if (!Number.isFinite(options.currentTimeSec)) {
|
||||
return options.liveText;
|
||||
}
|
||||
const nearby = nearbyCanonicalCues(options.cues, options.currentTimeSec);
|
||||
if (nearby.length === 0) {
|
||||
return options.liveText;
|
||||
}
|
||||
const compactCues = nearby.map((cue) => compactWhitespace(cue.text));
|
||||
const kept = options.liveText.split('\n').filter((line) => {
|
||||
const compact = compactWhitespace(line);
|
||||
return compact && !compactCues.some((cueText) => cueText.includes(compact));
|
||||
});
|
||||
return kept.length > 0 ? kept.join('\n') : options.liveText;
|
||||
}
|
||||
|
||||
export function resolvePrimarySubtitleText(options: {
|
||||
liveText: string;
|
||||
currentTimeSec: number;
|
||||
cues: readonly SubtitleCue[] | null | undefined;
|
||||
}): string {
|
||||
if (!options.liveText.trim()) {
|
||||
return options.liveText;
|
||||
}
|
||||
return (
|
||||
resolveCanonicalPrimarySubtitle({
|
||||
liveText: options.liveText,
|
||||
currentTimeSec: options.currentTimeSec,
|
||||
cues: options.cues,
|
||||
})?.text ??
|
||||
resolveActiveParsedPrimarySubtitle(options)?.text ??
|
||||
options.liveText
|
||||
);
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user