mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-14 01:55:58 -07:00
Compare commits
26 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
d2f9a91d39
|
|||
|
7cc9f72fc9
|
|||
|
de5b1b5d10
|
|||
| b98d4d65c7 | |||
|
046e74ea91
|
|||
| 8bf847503d | |||
|
47b5903392
|
|||
| bf85554d1e | |||
|
d74c7e1235
|
|||
| 57ddd19953 | |||
| ee25536d90 | |||
| 7b0fbdf254 | |||
| 2fefc83e3f | |||
| dbdf578c68 | |||
| 441ecf3c04 | |||
|
a0dde4ee3e
|
|||
| fe4dacc1e7 | |||
| b08cd0db35 | |||
| bffb1c5982 | |||
| 5b8848518a | |||
|
176edd67f1
|
|||
| 4d65dec340 | |||
|
6607c333bc
|
|||
|
b2bbf1ae12
|
|||
| b204d4dd6e | |||
| 89ed675935 |
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"name": "subminer-local",
|
||||
"interface": {
|
||||
"displayName": "SubMiner Local"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "subminer-workflow",
|
||||
"source": {
|
||||
"source": "local",
|
||||
"path": "./plugins/subminer-workflow"
|
||||
},
|
||||
"policy": {
|
||||
"installation": "AVAILABLE",
|
||||
"authentication": "ON_INSTALL"
|
||||
},
|
||||
"category": "Productivity"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,22 +1,45 @@
|
||||
---
|
||||
name: 'subminer-change-verification'
|
||||
description: 'Compatibility shim. Canonical SubMiner change verification workflow now lives in the repo-local subminer-workflow plugin.'
|
||||
name: subminer-change-verification
|
||||
description: Verify SubMiner changes with repo-native cheap-first test lanes. Use after code, config, launcher, plugin, runtime, stats, documentation, or workflow changes; do not use for read-only questions.
|
||||
---
|
||||
|
||||
# Compatibility Shim
|
||||
# SubMiner Change Verification
|
||||
|
||||
Canonical source:
|
||||
Verify the behavior claimed by a change without running unrelated expensive checks by default.
|
||||
|
||||
- `plugins/subminer-workflow/skills/subminer-change-verification/SKILL.md`
|
||||
## Workflow
|
||||
|
||||
Canonical helper scripts:
|
||||
1. Inspect the requested scope and changed paths with `git status --short` and `git diff`.
|
||||
2. Read `docs/workflow/verification.md` as the source of truth for maintained lanes.
|
||||
3. Run the cheapest lane or lanes that cover the changed behavior.
|
||||
4. Escalate to the full handoff gate only for substantial or cross-boundary changes.
|
||||
5. Report exact commands, results, skipped checks, blockers, and remaining risk.
|
||||
|
||||
- `plugins/subminer-workflow/skills/subminer-change-verification/scripts/classify_subminer_diff.sh`
|
||||
- `plugins/subminer-workflow/skills/subminer-change-verification/scripts/verify_subminer_change.sh`
|
||||
Do not use hidden wrapper commands. Verification commands are owned by `package.json` and the workflow documentation.
|
||||
|
||||
When this shim is invoked:
|
||||
## Lane Selection
|
||||
|
||||
1. Read the canonical plugin-owned skill.
|
||||
2. Follow the plugin-owned skill as the source of truth.
|
||||
3. Use the wrapper scripts in this shim directory only for compatibility with existing commands and docs.
|
||||
4. Do not duplicate workflow changes here; update the plugin-owned skill and scripts instead.
|
||||
- Internal docs, `AGENTS.md`, or `.agents/skills/**`: `bun run test:docs:kb`
|
||||
- User-facing `docs-site/**`: `bun run docs:test`, then `bun run docs:build`
|
||||
- Config/schema/defaults: `bun run test:config`
|
||||
- If defaults or templates changed, also run `bun run generate:config-example` and `bun run verify:config-example`.
|
||||
- General TypeScript source: `bun run typecheck`, then `bun run test:fast`
|
||||
- Launcher or mpv plugin: `bun run test:launcher` or `bun run test:env`, based on the behavior changed
|
||||
- Runtime compatibility or dist-sensitive wiring: `bun run test:runtime:compat`
|
||||
- Stats dashboard: `bun run test:stats`
|
||||
- Build/release scripts: `bun run test:scripts`
|
||||
|
||||
For substantial changes, use the full gate documented in `AGENTS.md` and `docs/workflow/verification.md`.
|
||||
|
||||
## Runtime Escalation
|
||||
|
||||
Real runtime checks are required when the claim depends on actual Electron, mpv, overlay, focus, window tracking, launch, or socket behavior. Run the relevant application flow when the environment supports it. Otherwise, report the missing runtime dependency and do not present cheaper checks as authoritative runtime validation.
|
||||
|
||||
## Pre-Handoff Checks
|
||||
|
||||
Before handoff, reconcile both questions:
|
||||
|
||||
1. Do behavior, defaults, flags, shortcuts, ports, APIs, architecture, or workflow changes require documentation updates?
|
||||
2. Does the change require a current-outcome fragment under `changes/` according to `changes/README.md`?
|
||||
|
||||
Complete required updates before handoff or report the blocker.
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
REPO_ROOT=$(cd "$SCRIPT_DIR/../../../.." && pwd)
|
||||
TARGET="$REPO_ROOT/plugins/subminer-workflow/skills/subminer-change-verification/scripts/classify_subminer_diff.sh"
|
||||
|
||||
if [[ ! -x "$TARGET" ]]; then
|
||||
echo "Missing canonical script: $TARGET" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec "$TARGET" "$@"
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
REPO_ROOT=$(cd "$SCRIPT_DIR/../../../.." && pwd)
|
||||
TARGET="$REPO_ROOT/plugins/subminer-workflow/skills/subminer-change-verification/scripts/verify_subminer_change.sh"
|
||||
|
||||
if [[ ! -x "$TARGET" ]]; then
|
||||
echo "Missing canonical script: $TARGET" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec "$TARGET" "$@"
|
||||
@@ -1,18 +0,0 @@
|
||||
---
|
||||
name: 'subminer-scrum-master'
|
||||
description: 'Compatibility shim. Canonical SubMiner scrum-master workflow now lives in the repo-local subminer-workflow plugin.'
|
||||
---
|
||||
|
||||
# Compatibility Shim
|
||||
|
||||
Canonical source:
|
||||
|
||||
- `plugins/subminer-workflow/skills/subminer-scrum-master/SKILL.md`
|
||||
|
||||
When this shim is invoked:
|
||||
|
||||
1. Read the canonical plugin-owned skill.
|
||||
2. Follow the plugin-owned skill as the source of truth.
|
||||
3. Do not duplicate workflow changes here; update the plugin-owned skill instead.
|
||||
|
||||
This shim exists so existing repo references and prompts keep resolving during the migration to the repo-local plugin workflow.
|
||||
@@ -1,4 +1,4 @@
|
||||
# AGENTS.MD
|
||||
# AGENTS.md
|
||||
|
||||
## Internal Docs
|
||||
|
||||
@@ -13,7 +13,7 @@ Start here, then leave this file.
|
||||
|
||||
`docs-site/` is user-facing. Do not treat it as the canonical internal source of truth.
|
||||
|
||||
`CLAUDE.md` is a symlink to this file — there is one project instruction file, not two.
|
||||
`CLAUDE.md` is a symlink to this file; there is one project instruction file, not two.
|
||||
|
||||
## Quick Start
|
||||
|
||||
@@ -25,8 +25,9 @@ Start here, then leave this file.
|
||||
|
||||
## Build / Test
|
||||
|
||||
- Runtime/package manager: Bun (`packageManager: bun@1.3.5`)
|
||||
- Default handoff gate:
|
||||
- Runtime/package manager: Bun; use the version pinned by `package.json`.
|
||||
- Follow [`docs/workflow/verification.md`](./docs/workflow/verification.md) and start with the cheapest sufficient lane.
|
||||
- Full handoff gate for substantial changes:
|
||||
`bun run typecheck`
|
||||
`bun run test:fast`
|
||||
`bun run test:env`
|
||||
@@ -44,13 +45,15 @@ Start here, then leave this file.
|
||||
- Runtime-compat / dist-sensitive: `bun run test:runtime:compat`
|
||||
- Stats dashboard UI (`stats/`): `bun run test:stats`
|
||||
- Build/release scripts (`scripts/**`): `bun run test:scripts`
|
||||
- Docs-only: `bun run docs:test`, then `bun run docs:build`
|
||||
- Internal docs, `AGENTS.md`, or repo skills: `bun run test:docs:kb`
|
||||
- User-facing `docs-site/`: `bun run docs:test`, then `bun run docs:build`
|
||||
- macOS mpv window helper: `bun test scripts/get-mpv-window-macos.test.ts`
|
||||
- Test lanes are directory-discovered via `scripts/test-lanes.ts`; never hand-list test files in `package.json`
|
||||
|
||||
## Docs Upkeep
|
||||
|
||||
- Docs ship with the change, not after. If a change alters behavior, defaults, flags, shortcuts, ports, or APIs, update the matching docs in the same PR. Touching code without reconciling its docs is an incomplete change.
|
||||
- Source of truth for config defaults is the generated `config.example.jsonc`. Never write a default value into prose you didn't read from it — and don't restate the same default across multiple docs; cite/link to one place so there's a single thing to update.
|
||||
- Source of truth for config defaults is the generated `config.example.jsonc`. Never write a default value into prose you didn't read from it, and don't restate the same default across multiple docs; cite/link to one place so there's a single thing to update.
|
||||
- Trigger map (touch left → update right):
|
||||
- `src/config/definitions/**` (schema/defaults/template) → `bun run generate:config-example`, then reconcile `docs-site/configuration.md` + any feature doc that cites that default
|
||||
- shortcuts/keybindings (`shortcuts.*`, `keybindings`, `stats.*Key`, `subtitleSidebar.toggleKey`, controller bindings) → `docs-site/shortcuts.md`
|
||||
@@ -71,16 +74,10 @@ Start here, then leave this file.
|
||||
|
||||
## Release / PR Notes
|
||||
|
||||
- User-visible PRs need reconciled current-outcome fragment(s) in `changes/*.md` — format and rules in [`changes/README.md`](./changes/README.md) (`type` + `area` keys required; inspect existing same-PR fragments, then update/remove stale bullets or add only genuinely separate outcomes; apply the `skip-changelog` label to opt out)
|
||||
- User-visible PRs need reconciled current-outcome fragment(s) in `changes/*.md`. Format and rules live in [`changes/README.md`](./changes/README.md) (`type` + `area` keys required; inspect existing same-PR fragments, then update/remove stale bullets or add only genuinely separate outcomes; apply the `skip-changelog` label to opt out).
|
||||
- User-visible docs changes get a `type: docs` fragment
|
||||
- CI enforces `bun run changelog:lint` and `bun run changelog:pr-check`
|
||||
- PR review helpers:
|
||||
- `gh pr view --json number,title,url --jq '"PR #\\(.number): \\(.title)\\n\\(.url)"'`
|
||||
- `gh pr view --json number,title --jq '"PR #\\(.number): \\(.title)"'`
|
||||
- `gh api repos/:owner/:repo/pulls/<num>/comments --paginate`
|
||||
|
||||
## Runtime Notes
|
||||
|
||||
- Use Codex background for long jobs; tmux only when persistence/interaction is required
|
||||
- CI red: `gh run list/view`, rerun, fix, repeat until green
|
||||
- TypeScript: keep files small; follow existing patterns
|
||||
- Only Swift is the `scripts/get-mpv-window-macos.swift` helper (macOS mpv window detection); validate via `bun test scripts/get-mpv-window-macos.test.ts`
|
||||
- For CI debugging, inspect runs with `gh run list/view`; rerun or fix only within the requested scope.
|
||||
|
||||
@@ -1,5 +1,56 @@
|
||||
# Changelog
|
||||
|
||||
## v0.19.3 (2026-08-13)
|
||||
|
||||
### Added
|
||||
- Changelog Modal: Adds an in-app changelog you can open from the tray ("View Changelog") or the "What's New" button on the update notification, so the notification stays reachable while you read. It shows the newest published release notes (falling back to the bundled changelog if that fetch fails), folds older versions while keeping the current one expanded, and supports keyboard navigation (`J`/`K`/arrows, `Enter`, `R`, `Esc`).
|
||||
|
||||
### Changed
|
||||
- Subtitle Tokenization Performance: Reworks subtitle dictionary lookups to cut per-line work roughly in half, cache repeated lookups across lines, and stop tokenization from competing with on-screen subtitle prefetching. Also fixes several accuracy issues along the way: dropped readings on trailing kana, character names being skipped after a dictionary sync, annotations not refreshing after mining a card, and halfwidth katakana character names losing their reading or being swallowed by other words.
|
||||
|
||||
### Fixed
|
||||
- Character Dictionary Large Imports: Large character dictionaries (e.g. One Piece) no longer fail to install from a fixed timeout budget; the import now scales its time budget to dictionary size and reports detailed progress (page/character counts, image download progress, elapsed time) instead of one static message.
|
||||
- Stats Delete Responsiveness: Deleting sessions, episodes, or library entries no longer freezes the stats page or an active video player; deletes are now batched into a single transaction.
|
||||
- Styled Subtitle Cue Parsing: Heavily typeset subtitles (karaoke, signs) no longer flood the subtitle sidebar with garbage; vector drawing commands are no longer shown as text, and duplicate/animation-burst cues now collapse into one.
|
||||
- X11 mpv Renderer: Fixes an mpv crash on the first fullscreen toggle for X11/XWayland users with `gpu-next` shaders (e.g. ArtCNN), which was caused by X11 mode forcing the legacy OpenGL renderer.
|
||||
- X11 Overlay Display Scaling: Fixes the overlay appearing oversized and offset from mpv on X11/XWayland under fractional or mixed-monitor display scaling.
|
||||
|
||||
<details>
|
||||
<summary>Internal changes</summary>
|
||||
|
||||
### Internal
|
||||
- Subtitle text is now decoded from ASS exactly once at ingest, so the renderer, timing tracker, and tokenizer all share one decoded value instead of each re-deriving it.
|
||||
- Added per-stage debug timings (`scanMs`, `mecabMs`, `frequencyMs`, `annotateMs`) to the subtitle tokenization pipeline log.
|
||||
|
||||
</details>
|
||||
|
||||
## v0.19.2 (2026-08-04)
|
||||
|
||||
### Changed
|
||||
- Subsync: The sync modal now lets you choose both the reference subtitle (correct timing) and the out-of-sync subtitle to retime, for both alass and ffsubsync. alass can also use the loaded video's audio as a reference for local files. Retiming the secondary track now reloads the result into the secondary slot instead of overwriting the primary subtitle.
|
||||
|
||||
### Fixed
|
||||
- Streaming Subtitle Tokenization: Jellyfin streams now seed subtitle tokenization directly from the downloaded subtitle file instead of relying on an mpv event that could be missed, and prefetching now runs to the end of the file and clears between episodes. The tokenization cache was raised from 256 to 2500 lines, and parsed cues are no longer lost when the active subtitle track briefly can't be resolved (e.g. switching to an embedded track). Together these prevent episodes from falling back to slow, line-by-line tokenization during playback.
|
||||
- Overlay: Subtitle lines now appear immediately at their cue time even on a tokenization cache miss, upgrading in place once tokens and annotations are ready, instead of waiting on a line still being processed. A failed tokenization is no longer cached as plain text, so repeated lines get another chance at annotations.
|
||||
- Background Logging: Background startup now respects the configured logging level when no explicit log level is passed.
|
||||
|
||||
<details>
|
||||
<summary>Internal changes</summary>
|
||||
|
||||
### Internal
|
||||
- Patched three high-severity dependency advisories (`undici`, `brace-expansion`, `fast-uri`).
|
||||
|
||||
</details>
|
||||
|
||||
## v0.19.1 (2026-08-01)
|
||||
|
||||
### Added
|
||||
- Word Card Type: Adds a setting (Settings > Mining/Anki > Kiku/Lapis Features > "Word Card Type") to choose which card-type flag SubMiner marks on Kiku/Lapis word cards — `word-and-sentence` (default), `click`, `sentence`, `audio`, or `none`. Click cards (`IsClickCard`) can now be flagged, and setting any card-type flag clears the others so a note can't claim two types at once.
|
||||
|
||||
### Fixed
|
||||
- Yomitan Popup: Fixes the macOS Yomitan popup going inert after mining a card — clicks outside the popup no longer pass through to mpv, and scrolling over the popup scrolls its definitions instead of seeking playback.
|
||||
- YouTube Playlist Links: Fixes opening a video from a playlist URL (e.g. a Watch Later link with `list=`/`index=`) timing out while probing subtitles, metadata, or the playback URL.
|
||||
|
||||
## v0.19.0 (2026-07-29)
|
||||
|
||||
### Added
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
"eslint": "^10.8.0",
|
||||
"prettier": "^3.8.1",
|
||||
"typescript": "^5.9.3",
|
||||
"undici": "7.28.0",
|
||||
"undici": "7.29.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -36,16 +36,18 @@
|
||||
"overrides": {
|
||||
"@xmldom/xmldom": "0.8.13",
|
||||
"app-builder-lib": "26.15.3",
|
||||
"brace-expansion": "5.0.8",
|
||||
"brace-expansion": "5.0.9",
|
||||
"electron-builder-squirrel-windows": "26.15.3",
|
||||
"fast-uri": "3.1.5",
|
||||
"form-data": "4.0.6",
|
||||
"ip-address": "10.2.0",
|
||||
"js-yaml": "4.3.0",
|
||||
"js-yaml": "4.3.1",
|
||||
"lodash": "4.18.0",
|
||||
"minimatch": "10.2.5",
|
||||
"picomatch": "4.0.4",
|
||||
"tar": "7.5.21",
|
||||
"tmp": "0.2.7",
|
||||
"undici": "7.29.0",
|
||||
},
|
||||
"packages": {
|
||||
"@discordjs/collection": ["@discordjs/collection@2.1.1", "", {}, "sha512-LiSusze9Tc7qF03sLCujF5iZp7K+vRNEDBZ86FT9aQAv3vxMLihUvKvpsCWiQ2DJq1tVckopKm1rxomgNUc9hg=="],
|
||||
@@ -266,7 +268,7 @@
|
||||
|
||||
"boolean": ["boolean@3.2.0", "", {}, "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw=="],
|
||||
|
||||
"brace-expansion": ["brace-expansion@5.0.8", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg=="],
|
||||
"brace-expansion": ["brace-expansion@5.0.9", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg=="],
|
||||
|
||||
"buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="],
|
||||
|
||||
@@ -404,7 +406,7 @@
|
||||
|
||||
"fast-levenshtein": ["fast-levenshtein@2.0.6", "", {}, "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="],
|
||||
|
||||
"fast-uri": ["fast-uri@3.1.4", "", {}, "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw=="],
|
||||
"fast-uri": ["fast-uri@3.1.5", "", {}, "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw=="],
|
||||
|
||||
"fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="],
|
||||
|
||||
@@ -496,7 +498,7 @@
|
||||
|
||||
"jiti": ["jiti@2.6.1", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ=="],
|
||||
|
||||
"js-yaml": ["js-yaml@4.3.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q=="],
|
||||
"js-yaml": ["js-yaml@4.3.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ=="],
|
||||
|
||||
"json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="],
|
||||
|
||||
@@ -714,7 +716,7 @@
|
||||
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"undici": ["undici@7.28.0", "", {}, "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA=="],
|
||||
"undici": ["undici@7.29.0", "", {}, "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw=="],
|
||||
|
||||
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
|
||||
|
||||
@@ -754,8 +756,6 @@
|
||||
|
||||
"@discordjs/rest/discord-api-types": ["discord-api-types@0.38.43", "", {}, "sha512-sSoBf/nK6m7BGtw65mi+QBuvEWaHE8MMziFLqWL+gT6ME/BLg34dRSVKS3Husx40uU06bvxUc3/X+D9Y6/zAbw=="],
|
||||
|
||||
"@discordjs/rest/undici": ["undici@6.27.0", "", {}, "sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg=="],
|
||||
|
||||
"@discordjs/util/discord-api-types": ["discord-api-types@0.38.43", "", {}, "sha512-sSoBf/nK6m7BGtw65mi+QBuvEWaHE8MMziFLqWL+gT6ME/BLg34dRSVKS3Husx40uU06bvxUc3/X+D9Y6/zAbw=="],
|
||||
|
||||
"@electron/asar/commander": ["commander@5.1.0", "", {}, "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg=="],
|
||||
@@ -808,8 +808,6 @@
|
||||
|
||||
"node-gyp/env-paths": ["env-paths@2.2.1", "", {}, "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A=="],
|
||||
|
||||
"node-gyp/undici": ["undici@6.27.0", "", {}, "sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg=="],
|
||||
|
||||
"node-gyp/which": ["which@6.0.1", "", { "dependencies": { "isexe": "^4.0.0" }, "bin": { "node-which": "bin/which.js" } }, "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg=="],
|
||||
|
||||
"pkijs/@noble/hashes": ["@noble/hashes@1.4.0", "", {}, "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg=="],
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
type: fixed
|
||||
area: Anki media
|
||||
|
||||
- Fixed sentence-audio generation timing out on slow network-mounted MKV files with many subtitle and font-attachment streams. Selected audio tracks now use bounded FFmpeg probing and a two-minute extraction budget, and missing output reports a clear FFmpeg error instead of raw `ENOENT`.
|
||||
@@ -0,0 +1,5 @@
|
||||
type: fixed
|
||||
area: stats
|
||||
|
||||
- Typeset subtitles no longer flood the stats. Karaoke openings and animated signs are authored as one subtitle event per animation frame, and immersion tracking counted every frame, which was enough to put an OP lyric at the top of "Top Repeated Words" for good. Lines are now collapsed on the way in using the same rules the subtitle sidebar already applies: matching parsed timings record exactly the cues the sidebar shows, while shifted, changing, or unparsed sources use a strict fallback where identical, contiguous, sub-0.1s lines stop counting after a few frames. Ordinary repeated dialogue and rewatches are unaffected.
|
||||
- Added a cleanup for stats already affected. The Vocabulary tab has a **Duplicates** button that scans a chosen window (7 days through all time), shows the bursts it found and the word and kanji counts they added, and collapses each run to one line once confirmed. `subminer stats cleanup --duplicate-lines` does the same from the terminal, with `--dry-run` and `--lookback-days <n>`. Only subtitle lines and the vocabulary counts they feed are touched; watch time and lines-seen totals are left as recorded.
|
||||
@@ -0,0 +1,8 @@
|
||||
type: fixed
|
||||
area: stats
|
||||
|
||||
- Stats deletes no longer freeze the stats dashboard: the delete worker module now resolves when running from source, so deletes actually run off the serving thread instead of silently falling back to it.
|
||||
- Deletes now subtract their exact contribution from lifetime summaries instead of rebuilding them from retained sessions, making delete cost proportional to what is deleted and preserving lifetime totals older than the session retention window.
|
||||
- If the delete worker crashes, the delete now retries on the current thread instead of failing.
|
||||
- Library merges, video moves, AniList reassignments, and `subminer stats cleanup -l` also stopped rebuilding lifetime summaries from retained sessions; they now recompute from per-episode history, so those operations are faster and no longer erase lifetime totals older than the session retention window.
|
||||
- Deleting content that contains very common words no longer rescans every occurrence of those words across the whole library; first/last-seen dates are refreshed with index seeks instead.
|
||||
@@ -523,7 +523,7 @@
|
||||
// ==========================================
|
||||
// AnkiConnect Integration
|
||||
// Automatic Anki updates and media generation options.
|
||||
// Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, and isKiku.fieldGrouping update live while SubMiner is running.
|
||||
// Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running.
|
||||
// Shared AI provider transport settings are read from top-level ai and typically require restart.
|
||||
// Most other AnkiConnect settings still require restart.
|
||||
// ==========================================
|
||||
@@ -605,7 +605,10 @@
|
||||
"enabled": false, // Enable Kiku-specific mining behaviors (duplicate handling, field grouping). Values: true | false
|
||||
"fieldGrouping": "disabled", // Kiku duplicate-card field grouping mode. Values: auto | manual | disabled
|
||||
"deleteDuplicateInAuto": true // When Kiku field grouping is "auto", delete the duplicate source card after grouping completes. Values: true | false
|
||||
} // Is kiku setting.
|
||||
}, // Is kiku setting.
|
||||
"lapisKiku": {
|
||||
"wordCardKind": "word-and-sentence" // Card-type flag SubMiner marks on Kiku/Lapis word cards. Only one flag is set at a time; the others are cleared. Requires isKiku.enabled or isLapis.enabled. Values: word-and-sentence | click | sentence | audio | none
|
||||
} // Lapis kiku setting.
|
||||
}, // Automatic Anki updates and media generation options.
|
||||
|
||||
// ==========================================
|
||||
|
||||
@@ -289,6 +289,21 @@ Trigger with the mine sentence shortcut (`Ctrl/Cmd+S` by default). The card is c
|
||||
|
||||
To mine multiple subtitle lines as one sentence card, use `Ctrl/Cmd+Shift+S` followed by a digit (1–9) to select how many recent lines to combine.
|
||||
|
||||
## Word Card Type (Kiku/Lapis)
|
||||
|
||||
Word cards get a card-type flag when SubMiner fills their sentence, whether that comes from Yomitan auto-enrichment, a manual clipboard update, or stats-dashboard word mining. By default the flag is `IsWordAndSentenceCard`; pick a different one with `ankiConnect.lapisKiku.wordCardKind`.
|
||||
|
||||
```jsonc
|
||||
"ankiConnect": {
|
||||
"isKiku": { "enabled": true },
|
||||
"lapisKiku": {
|
||||
"wordCardKind": "click" // word-and-sentence (default), click, sentence, audio, none
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`click` marks `IsClickCard`, `sentence` marks `IsSentenceCard`, `audio` marks `IsAudioCard`, and `none` leaves the flags untouched for templates that manage them elsewhere. Whichever flag is chosen, the other card-type flags are cleared so the note never claims two card types. The setting is only read when `isKiku` or `isLapis` is enabled, and cards mined with Mine Sentence or Mine Audio keep their own flag.
|
||||
|
||||
## Field Grouping (Kiku)
|
||||
|
||||
When you mine the same word multiple times, SubMiner can merge the cards instead of creating duplicates. This is designed for note types like [Kiku](https://github.com/youyoumu/kiku) that support grouped sentence/audio/image fields.
|
||||
@@ -313,11 +328,11 @@ When you mine the same word multiple times, SubMiner can merge the cards instead
|
||||
|
||||
### What Gets Merged
|
||||
|
||||
| Field | Merge behavior |
|
||||
| -------- | ---------------------------------------- |
|
||||
| Field | Merge behavior |
|
||||
| -------- | --------------------------------------------- |
|
||||
| Sentence | Both cards' sentences kept as grouped entries |
|
||||
| Audio | Both cards' `[sound:...]` entries kept |
|
||||
| Image | Both cards' images kept |
|
||||
| Audio | Both cards' `[sound:...]` entries kept |
|
||||
| Image | Both cards' images kept |
|
||||
|
||||
Identical values from both cards are kept as separate grouped entries; the merge does not deduplicate.
|
||||
|
||||
|
||||
@@ -75,8 +75,8 @@ src/
|
||||
renderer/ # Overlay renderer (modularized UI/runtime)
|
||||
handlers/ # Keyboard/mouse/gamepad interaction modules
|
||||
modals/ # Modal flows (Jimaku, Kiku, subsync, runtime options, session help,
|
||||
# character dictionary, playlist browser, subtitle sidebar,
|
||||
# YouTube track picker, controller config/debug/select)
|
||||
# changelog, character dictionary, playlist browser, subtitle
|
||||
# sidebar, YouTube track picker, controller config/debug/select)
|
||||
positioning/ # Subtitle position controller (drag-to-reposition)
|
||||
settings/ # Settings window UI (model, controls, markup)
|
||||
types/ # Domain type modules (anki, config, integrations, ...)
|
||||
|
||||
@@ -1,5 +1,56 @@
|
||||
# Changelog
|
||||
|
||||
## v0.19.3 (2026-08-13)
|
||||
|
||||
**Added**
|
||||
- Changelog Modal: Adds an in-app changelog you can open from the tray ("View Changelog") or the "What's New" button on the update notification, so the notification stays reachable while you read. It shows the newest published release notes (falling back to the bundled changelog if that fetch fails), folds older versions while keeping the current one expanded, and supports keyboard navigation (`J`/`K`/arrows, `Enter`, `R`, `Esc`).
|
||||
|
||||
**Changed**
|
||||
- Subtitle Tokenization Performance: Reworks subtitle dictionary lookups to cut per-line work roughly in half, cache repeated lookups across lines, and stop tokenization from competing with on-screen subtitle prefetching. Also fixes several accuracy issues along the way: dropped readings on trailing kana, character names being skipped after a dictionary sync, annotations not refreshing after mining a card, and halfwidth katakana character names losing their reading or being swallowed by other words.
|
||||
|
||||
**Fixed**
|
||||
- Character Dictionary Large Imports: Large character dictionaries (e.g. One Piece) no longer fail to install from a fixed timeout budget; the import now scales its time budget to dictionary size and reports detailed progress (page/character counts, image download progress, elapsed time) instead of one static message.
|
||||
- Stats Delete Responsiveness: Deleting sessions, episodes, or library entries no longer freezes the stats page or an active video player; deletes are now batched into a single transaction.
|
||||
- Styled Subtitle Cue Parsing: Heavily typeset subtitles (karaoke, signs) no longer flood the subtitle sidebar with garbage; vector drawing commands are no longer shown as text, and duplicate/animation-burst cues now collapse into one.
|
||||
- X11 mpv Renderer: Fixes an mpv crash on the first fullscreen toggle for X11/XWayland users with `gpu-next` shaders (e.g. ArtCNN), which was caused by X11 mode forcing the legacy OpenGL renderer.
|
||||
- X11 Overlay Display Scaling: Fixes the overlay appearing oversized and offset from mpv on X11/XWayland under fractional or mixed-monitor display scaling.
|
||||
|
||||
<details>
|
||||
<summary>Internal changes</summary>
|
||||
|
||||
**Internal**
|
||||
- Subtitle text is now decoded from ASS exactly once at ingest, so the renderer, timing tracker, and tokenizer all share one decoded value instead of each re-deriving it.
|
||||
- Added per-stage debug timings (`scanMs`, `mecabMs`, `frequencyMs`, `annotateMs`) to the subtitle tokenization pipeline log.
|
||||
|
||||
</details>
|
||||
|
||||
## v0.19.2 (2026-08-04)
|
||||
|
||||
**Changed**
|
||||
- Subsync: The sync modal now lets you choose both the reference subtitle (correct timing) and the out-of-sync subtitle to retime, for both alass and ffsubsync. alass can also use the loaded video's audio as a reference for local files. Retiming the secondary track now reloads the result into the secondary slot instead of overwriting the primary subtitle.
|
||||
|
||||
**Fixed**
|
||||
- Streaming Subtitle Tokenization: Jellyfin streams now seed subtitle tokenization directly from the downloaded subtitle file instead of relying on an mpv event that could be missed, and prefetching now runs to the end of the file and clears between episodes. The tokenization cache was raised from 256 to 2500 lines, and parsed cues are no longer lost when the active subtitle track briefly can't be resolved (e.g. switching to an embedded track). Together these prevent episodes from falling back to slow, line-by-line tokenization during playback.
|
||||
- Overlay: Subtitle lines now appear immediately at their cue time even on a tokenization cache miss, upgrading in place once tokens and annotations are ready, instead of waiting on a line still being processed. A failed tokenization is no longer cached as plain text, so repeated lines get another chance at annotations.
|
||||
- Background Logging: Background startup now respects the configured logging level when no explicit log level is passed.
|
||||
|
||||
<details>
|
||||
<summary>Internal changes</summary>
|
||||
|
||||
**Internal**
|
||||
- Patched three high-severity dependency advisories (`undici`, `brace-expansion`, `fast-uri`).
|
||||
|
||||
</details>
|
||||
|
||||
## v0.19.1 (2026-08-01)
|
||||
|
||||
**Added**
|
||||
- Word Card Type: Adds a setting (Settings > Mining/Anki > Kiku/Lapis Features > "Word Card Type") to choose which card-type flag SubMiner marks on Kiku/Lapis word cards — `word-and-sentence` (default), `click`, `sentence`, `audio`, or `none`. Click cards (`IsClickCard`) can now be flagged, and setting any card-type flag clears the others so a note can't claim two types at once.
|
||||
|
||||
**Fixed**
|
||||
- Yomitan Popup: Fixes the macOS Yomitan popup going inert after mining a card — clicks outside the popup no longer pass through to mpv, and scrolling over the popup scrolls its definitions instead of seeking playback.
|
||||
- YouTube Playlist Links: Fixes opening a video from a playlist URL (e.g. a Watch Later link with `list=`/`index=`) timing out while probing subtitles, metadata, or the playback URL.
|
||||
|
||||
## v0.19.0 (2026-07-29)
|
||||
|
||||
**Added**
|
||||
|
||||
+74
-56
@@ -398,30 +398,30 @@ See `config.example.jsonc` for detailed configuration options.
|
||||
}
|
||||
```
|
||||
|
||||
| Option | Values | Description |
|
||||
| ---------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `primaryDefaultMode` | string | Default primary subtitle bar visibility mode: `"hidden"`, `"visible"`, or `"hover"` (default: `"visible"`) |
|
||||
| `subtitleStyle.css` | object | CSS declaration object applied to primary subtitles after normal style defaults. Use CSS property names such as `font-size`. |
|
||||
| `secondary.css` | object | CSS declaration object applied to secondary subtitles after normal secondary style defaults. |
|
||||
| `enableJlpt` | boolean | Enable JLPT level underline styling (`false` by default) |
|
||||
| `preserveLineBreaks` | boolean | Preserve line breaks in visible overlay subtitle rendering (`false` by default). Enable to mirror mpv line layout. |
|
||||
| `autoPauseVideoOnHover` | boolean | Pause playback while mouse hovers subtitle text, then resume on leave (`true` by default). |
|
||||
| `autoPauseVideoOnYomitanPopup` | boolean | Pause playback while the Yomitan popup is open, then resume when the popup closes (`true` by default). |
|
||||
| `primaryVisibleOnYomitanPopup` | boolean | Keep hover-mode primary subtitles visible while the Yomitan popup is open (`true` by default). |
|
||||
| `nameMatchEnabled` | boolean | Enable character dictionary sync and subtitle token coloring for character-name matches (`false` by default) |
|
||||
| `nameMatchImagesEnabled` | boolean | Show small cached AniList character portraits beside matched character-name tokens (`false` by default) |
|
||||
| `nameMatchColor` | string | Hex color used for subtitle tokens matched from the SubMiner character dictionary (default: `#f5bde6`) |
|
||||
| `knownWordColor` | string | Hex color used for known-word subtitle highlights (default: `#a6da95`) |
|
||||
| Option | Values | Description |
|
||||
| ---------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `primaryDefaultMode` | string | Default primary subtitle bar visibility mode: `"hidden"`, `"visible"`, or `"hover"` (default: `"visible"`) |
|
||||
| `subtitleStyle.css` | object | CSS declaration object applied to primary subtitles after normal style defaults. Use CSS property names such as `font-size`. |
|
||||
| `secondary.css` | object | CSS declaration object applied to secondary subtitles after normal secondary style defaults. |
|
||||
| `enableJlpt` | boolean | Enable JLPT level underline styling (`false` by default) |
|
||||
| `preserveLineBreaks` | boolean | Preserve line breaks in visible overlay subtitle rendering (`false` by default). Enable to mirror mpv line layout. |
|
||||
| `autoPauseVideoOnHover` | boolean | Pause playback while mouse hovers subtitle text, then resume on leave (`true` by default). |
|
||||
| `autoPauseVideoOnYomitanPopup` | boolean | Pause playback while the Yomitan popup is open, then resume when the popup closes (`true` by default). |
|
||||
| `primaryVisibleOnYomitanPopup` | boolean | Keep hover-mode primary subtitles visible while the Yomitan popup is open (`true` by default). |
|
||||
| `nameMatchEnabled` | boolean | Enable character dictionary sync and subtitle token coloring for character-name matches (`false` by default) |
|
||||
| `nameMatchImagesEnabled` | boolean | Show small cached AniList character portraits beside matched character-name tokens (`false` by default) |
|
||||
| `nameMatchColor` | string | Hex color used for subtitle tokens matched from the SubMiner character dictionary (default: `#f5bde6`) |
|
||||
| `knownWordColor` | string | Hex color used for known-word subtitle highlights (default: `#a6da95`) |
|
||||
| `knownWordMaturityColors` | object | Per-tier known-word colors used when `ankiConnect.knownWords.maturityEnabled` is on: `new` (`#ee99a0`), `learning` (`#b7bdf8`), `young` (`#91d7e3`), `mature` (`#a6da95`) |
|
||||
| `nPlusOneColor` | string | Hex color used for the single N+1 target subtitle highlight (default: `#c6a0f6`) |
|
||||
| `frequencyDictionary.enabled` | boolean | Enable frequency highlighting from dictionary lookups (`false` by default) |
|
||||
| `frequencyDictionary.sourcePath` | string | Path to a local frequency dictionary root. Leave empty or omit to use installed/default frequency-dictionary search paths. |
|
||||
| `frequencyDictionary.topX` | number | Only color tokens whose frequency rank is `<= topX` (`10000` by default) |
|
||||
| `frequencyDictionary.mode` | string | `"single"` or `"banded"` (`"single"` by default) |
|
||||
| `frequencyDictionary.matchMode` | string | `"headword"` or `"surface"` (`"headword"` by default) |
|
||||
| `frequencyDictionary.singleColor` | string | Color used for all highlighted tokens in single mode |
|
||||
| `frequencyDictionary.bandedColors` | string[] | Array of five hex colors used for ranked bands in banded mode |
|
||||
| `jlptColors` | object | JLPT level underline colors object (`N1`..`N5`) |
|
||||
| `nPlusOneColor` | string | Hex color used for the single N+1 target subtitle highlight (default: `#c6a0f6`) |
|
||||
| `frequencyDictionary.enabled` | boolean | Enable frequency highlighting from dictionary lookups (`false` by default) |
|
||||
| `frequencyDictionary.sourcePath` | string | Path to a local frequency dictionary root. Leave empty or omit to use installed/default frequency-dictionary search paths. |
|
||||
| `frequencyDictionary.topX` | number | Only color tokens whose frequency rank is `<= topX` (`10000` by default) |
|
||||
| `frequencyDictionary.mode` | string | `"single"` or `"banded"` (`"single"` by default) |
|
||||
| `frequencyDictionary.matchMode` | string | `"headword"` or `"surface"` (`"headword"` by default) |
|
||||
| `frequencyDictionary.singleColor` | string | Color used for all highlighted tokens in single mode |
|
||||
| `frequencyDictionary.bandedColors` | string[] | Array of five hex colors used for ranked bands in banded mode |
|
||||
| `jlptColors` | object | JLPT level underline colors object (`N1`..`N5`) |
|
||||
|
||||
Subtitle CSS custom properties:
|
||||
|
||||
@@ -555,11 +555,11 @@ Secondary subtitles do **not** auto-load by default. To turn them on for local a
|
||||
}
|
||||
```
|
||||
|
||||
| Option | Values | Description |
|
||||
| ----------------------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `secondarySubLanguages` | string[] | Language codes to auto-load (e.g., `["eng", "en"]`); non-Signs/Songs tracks are preferred when several tracks match. Default is empty (`[]`). |
|
||||
| `autoLoadSecondarySub` | `true`, `false` | Auto-detect and load a matching secondary subtitle track for local/Jellyfin sidecar files (default: `false`) |
|
||||
| `defaultMode` | `"hidden"`, `"visible"`, `"hover"` | Initial display mode (default: `"hover"`) |
|
||||
| Option | Values | Description |
|
||||
| ----------------------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `secondarySubLanguages` | string[] | Language codes to auto-load (e.g., `["eng", "en"]`); non-Signs/Songs tracks are preferred when several tracks match. Default is empty (`[]`). |
|
||||
| `autoLoadSecondarySub` | `true`, `false` | Auto-detect and load a matching secondary subtitle track for local/Jellyfin sidecar files (default: `false`) |
|
||||
| `defaultMode` | `"hidden"`, `"visible"`, `"hover"` | Initial display mode (default: `"hover"`) |
|
||||
|
||||
These two settings apply to local and Jellyfin playback only. YouTube secondary selection is fixed to English and ignores them; see [YouTube Integration](/youtube-integration#secondary-subtitle-languages). `defaultMode` still controls how the loaded secondary bar is displayed in every case.
|
||||
|
||||
@@ -1043,7 +1043,7 @@ This example is intentionally compact. The option table below documents availabl
|
||||
| `ankiConnect.knownWords.refreshMinutes` | number | Minutes between known-word cache refreshes (default: `1440`) |
|
||||
| `ankiConnect.knownWords.decks` | object | Deck→fields mapping used for known-word cache query scope (e.g. `{ "Kaishi 1.5k": ["Word"] }`). |
|
||||
| `ankiConnect.knownWords.maturityEnabled` | `true`, `false` | Color known words by Anki card maturity (new/learning/young/mature) instead of one color. Requires `knownWords.highlightEnabled` (default: `false`). Tier colors come from `subtitleStyle.knownWordMaturityColors`. |
|
||||
| `ankiConnect.knownWords.matureThresholdDays` | number | Card interval in days at which a known word counts as mature (default: `21`, matching Anki's own convention) |
|
||||
| `ankiConnect.knownWords.matureThresholdDays` | number | Card interval in days at which a known word counts as mature (default: `21`, matching Anki's own convention) |
|
||||
| `ankiConnect.nPlusOne.enabled` | `true`, `false` | Enable N+1 subtitle highlighting (highlights the one unknown word in a sentence). Independent from `knownWords.highlightEnabled`. Requires known-word cache data (default: `false`). |
|
||||
| `ankiConnect.nPlusOne.minSentenceWords` | number | Minimum number of words required in a sentence before single unknown-word N+1 highlighting can trigger (default: `3`). |
|
||||
| `behavior.notificationType` | `"overlay"`, `"system"`, `"both"`, `"none"` | Notification type on card update (default: `"overlay"`). `"both"` means overlay + system. `osd` and `osd-system` are legacy config-file-only values; use `"osd-system"` to keep the old OSD + system behavior. |
|
||||
@@ -1069,6 +1069,9 @@ SubMiner is intentionally built for [Kiku](https://kiku.youyoumu.my.id/) and [La
|
||||
"enabled": true,
|
||||
"fieldGrouping": "manual",
|
||||
"deleteDuplicateInAuto": true
|
||||
},
|
||||
"lapisKiku": {
|
||||
"wordCardKind": "word-and-sentence"
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -1077,6 +1080,21 @@ SubMiner is intentionally built for [Kiku](https://kiku.youyoumu.my.id/) and [La
|
||||
- Enable `isKiku` to turn on duplicate merge behavior for mined Word/Expression hits.
|
||||
- When both are enabled, Kiku behavior is applied for grouping while sentence-card model settings are still read from `isLapis`.
|
||||
- `isKiku.fieldGrouping` supports `disabled`, `auto`, and `manual` merge modes; see [Field Grouping Modes](#field-grouping-modes).
|
||||
- `lapisKiku.wordCardKind` picks the card-type flag set on word cards; see [Word Card Type](#word-card-type). It is read only while `isLapis` or `isKiku` is enabled.
|
||||
|
||||
### Word Card Type
|
||||
|
||||
When SubMiner fills the sentence on a mined word card - from Yomitan auto-enrichment, a manual clipboard update, or stats-dashboard word mining - it marks which card that note should generate. `ankiConnect.lapisKiku.wordCardKind` chooses the flag:
|
||||
|
||||
| Value | Flag set |
|
||||
| ----------------------------- | ----------------------- |
|
||||
| `word-and-sentence` (default) | `IsWordAndSentenceCard` |
|
||||
| `click` | `IsClickCard` |
|
||||
| `sentence` | `IsSentenceCard` |
|
||||
| `audio` | `IsAudioCard` |
|
||||
| `none` | none; flags left as-is |
|
||||
|
||||
The other card-type flags are cleared so a note never claims two card types at once. Notes are skipped when the note type has no field for the chosen flag, and when the note was already mined as a sentence or audio card. Cards created by Mine Sentence and Mine Audio keep their own flag regardless of this setting.
|
||||
|
||||
### N+1 Word Highlighting
|
||||
|
||||
@@ -1167,10 +1185,10 @@ TsukiHime subtitle search works out of the box and needs no account or API key.
|
||||
}
|
||||
```
|
||||
|
||||
| Option | Values | Description |
|
||||
| ---------------------------- | ------------ | ---------------------------------------------------------------------------------------------------- |
|
||||
| Option | Values | Description |
|
||||
| ---------------------------- | ------------ | ----------------------------------------------------------------------------------------------------- |
|
||||
| `tsukihime.apiBaseUrl` | string (URL) | Base URL of the TsukiHime API (default: `https://api.tsukihime.org/v1`). Only change it for a mirror. |
|
||||
| `tsukihime.maxSearchResults` | number | Maximum releases returned per search (default: `10`; the API caps this at 100) |
|
||||
| `tsukihime.maxSearchResults` | number | Maximum releases returned per search (default: `10`; the API caps this at 100) |
|
||||
|
||||
The keyboard shortcut lives under `shortcuts.openTsukihime` (default `Ctrl+Shift+T`; set to `null` to disable). The older `animetosho` section and `shortcuts.openAnimetosho` are still accepted as deprecated aliases, with the current names taking precedence when both are set.
|
||||
|
||||
@@ -1178,9 +1196,9 @@ See [TsukiHime Integration](/tsukihime-integration) for the modal workflow, lang
|
||||
|
||||
### Subtitle Sync
|
||||
|
||||
Sync the active subtitle track from the overlay picker using `alass` or `ffsubsync`. Both are **optional external tools** that must be installed separately and available on your `PATH` (or configured via the path options below).
|
||||
Sync a subtitle track from the overlay picker using `alass` or `ffsubsync`. The picker lets you choose which track gets retimed (the active primary track by default) and, for alass, which reference it is aligned against (the secondary subtitle track by default). Both are **optional external tools** that must be installed separately and available on your `PATH` (or configured via the path options below).
|
||||
|
||||
- [`alass`](https://github.com/kaegi/alass) - fast, audio-independent sync using a secondary subtitle as reference
|
||||
- [`alass`](https://github.com/kaegi/alass) - fast, audio-independent sync using another subtitle as reference; it can also take the local video file as reference (alass extracts the audio itself)
|
||||
- [`ffsubsync`](https://github.com/smacke/ffsubsync) - audio-based sync using the video file as reference
|
||||
|
||||
```json
|
||||
@@ -1228,17 +1246,17 @@ AniList integration is opt-in and disabled by default. Enable it to allow SubMin
|
||||
}
|
||||
```
|
||||
|
||||
| Option | Values | Description |
|
||||
| -------------------------------------------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------- |
|
||||
| `anilist.enabled` | `true`, `false` | Enable AniList post-watch progress updates (default: `false`) |
|
||||
| `accessToken` | string | Optional explicit AniList access token override (default: empty string) |
|
||||
| `characterDictionary.maxLoaded` | number | Maximum number of most-recently-used AniList media snapshots included in the merged dictionary (default: `3`) |
|
||||
| `characterDictionary.refreshTtlHours` | number | Hours before a cached media snapshot is refreshed (default: `168`, clamped to 1–8760) |
|
||||
| `characterDictionary.evictionPolicy` | `"delete"`, `"disable"` | What happens to snapshots evicted beyond `maxLoaded` (default: `"delete"`) |
|
||||
| `characterDictionary.collapsibleSections.description` | `true`, `false` | Open the Description section by default in generated dictionary entries |
|
||||
| `characterDictionary.collapsibleSections.characterInformation` | `true`, `false` | Open the Character Information section by default in generated dictionary entries |
|
||||
| `characterDictionary.collapsibleSections.voicedBy` | `true`, `false` | Open the Voiced by section by default in generated dictionary entries |
|
||||
| `characterDictionary.profileScope` | `"all"`, `"active"` | Apply dictionary settings updates to all Yomitan profiles or only active profile |
|
||||
| Option | Values | Description |
|
||||
| -------------------------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------- |
|
||||
| `anilist.enabled` | `true`, `false` | Enable AniList post-watch progress updates (default: `false`) |
|
||||
| `accessToken` | string | Optional explicit AniList access token override (default: empty string) |
|
||||
| `characterDictionary.maxLoaded` | number | Maximum number of most-recently-used AniList media snapshots included in the merged dictionary (default: `3`) |
|
||||
| `characterDictionary.refreshTtlHours` | number | Hours before a cached media snapshot is refreshed (default: `168`, clamped to 1–8760) |
|
||||
| `characterDictionary.evictionPolicy` | `"delete"`, `"disable"` | What happens to snapshots evicted beyond `maxLoaded` (default: `"delete"`) |
|
||||
| `characterDictionary.collapsibleSections.description` | `true`, `false` | Open the Description section by default in generated dictionary entries |
|
||||
| `characterDictionary.collapsibleSections.characterInformation` | `true`, `false` | Open the Character Information section by default in generated dictionary entries |
|
||||
| `characterDictionary.collapsibleSections.voicedBy` | `true`, `false` | Open the Voiced by section by default in generated dictionary entries |
|
||||
| `characterDictionary.profileScope` | `"all"`, `"active"` | Apply dictionary settings updates to all Yomitan profiles or only active profile |
|
||||
|
||||
When `enabled` is `true` and `accessToken` is empty, SubMiner opens an AniList setup helper window. Keep `enabled` as `false` to disable all AniList setup/update behavior.
|
||||
|
||||
@@ -1539,18 +1557,18 @@ Configure the mpv executable, profile, and window state for SubMiner-managed mpv
|
||||
}
|
||||
```
|
||||
|
||||
| Option | Values | Description |
|
||||
| ------------------------ | --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `executablePath` | string | Absolute path to `mpv.exe` for Windows launch flows. Leave empty to auto-discover from `SUBMINER_MPV_PATH` or `PATH` (default `""`) |
|
||||
| `profile` | string | mpv profile name passed as `--profile=<name>`. Leave empty to pass no profile (default `""`) |
|
||||
| `launchMode` | `"normal"` \| `"maximized"` \| `"fullscreen"` | Window state when SubMiner spawns mpv (default `"normal"`) |
|
||||
| Option | Values | Description |
|
||||
| ------------------------ | --------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `executablePath` | string | Absolute path to `mpv.exe` for Windows launch flows. Leave empty to auto-discover from `SUBMINER_MPV_PATH` or `PATH` (default `""`) |
|
||||
| `profile` | string | mpv profile name passed as `--profile=<name>`. Leave empty to pass no profile (default `""`) |
|
||||
| `launchMode` | `"normal"` \| `"maximized"` \| `"fullscreen"` | Window state when SubMiner spawns mpv (default `"normal"`) |
|
||||
| `socketPath` | string | mpv IPC socket path used by SubMiner-managed playback and the bundled mpv plugin (platform-dependent default: `/tmp/subminer-socket`, or `\\\\.\\pipe\\subminer-socket` on Windows) |
|
||||
| `backend` | `"auto"` \| `"hyprland"` \| `"sway"` \| `"x11"` \| `"macos"` \| `"windows"` | Window tracking backend passed to the bundled mpv plugin. Auto detects the current platform (default: `"auto"`) |
|
||||
| `autoStartSubMiner` | `true`, `false` | Start SubMiner in the background when SubMiner-managed mpv loads a file (default: `true`) |
|
||||
| `pauseUntilOverlayReady` | `true`, `false` | Pause mpv on visible-overlay auto-start until SubMiner signals subtitle tokenization readiness, with a 30-second fallback (default: `true`) |
|
||||
| `subminerBinaryPath` | string | SubMiner app binary path passed to the bundled mpv plugin. Leave empty to use the launcher-detected app path (default: `""`) |
|
||||
| `aniskipEnabled` | `true`, `false` | Enable AniSkip intro detection, chapter markers, and the skip-intro key (default: `true`) |
|
||||
| `aniskipButtonKey` | string | mpv key used to skip the detected intro while the skip prompt is visible (default: `"TAB"`) |
|
||||
| `backend` | `"auto"` \| `"hyprland"` \| `"sway"` \| `"x11"` \| `"macos"` \| `"windows"` | Window tracking backend passed to the bundled mpv plugin. Auto detects the current platform (default: `"auto"`) |
|
||||
| `autoStartSubMiner` | `true`, `false` | Start SubMiner in the background when SubMiner-managed mpv loads a file (default: `true`) |
|
||||
| `pauseUntilOverlayReady` | `true`, `false` | Pause mpv on visible-overlay auto-start until SubMiner signals subtitle tokenization readiness, with a 30-second fallback (default: `true`) |
|
||||
| `subminerBinaryPath` | string | SubMiner app binary path passed to the bundled mpv plugin. Leave empty to use the launcher-detected app path (default: `""`) |
|
||||
| `aniskipEnabled` | `true`, `false` | Enable AniSkip intro detection, chapter markers, and the skip-intro key (default: `true`) |
|
||||
| `aniskipButtonKey` | string | mpv key used to skip the detected intro while the skip prompt is visible (default: `"TAB"`) |
|
||||
|
||||
If `mpv.profile` is configured and the launcher also receives `--profile`, SubMiner passes both as a comma-separated mpv profile list.
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ The same immersion data powers the stats dashboard.
|
||||
- In-app overlay: focus the visible overlay, then press the key from `stats.toggleKey` (default: `` ` `` / `Backquote`).
|
||||
- Launcher command: run `subminer stats` to start the local stats server on demand (it also opens the dashboard in your browser when `stats.autoOpenBrowser` is enabled; the default is `false`).
|
||||
- Background server: run `subminer stats -b` to start or reuse a dedicated background stats daemon without keeping the launcher attached, and `subminer stats -s` to stop that daemon.
|
||||
- Maintenance commands: run `subminer stats cleanup` or `subminer stats cleanup -v` to backfill/repair vocabulary metadata (`headword`, `reading`, POS) and purge stale or excluded rows from `imm_words` on demand; `subminer stats cleanup -l` repairs lifetime summary tables. `subminer stats rebuild` and `subminer stats backfill` rebuild or backfill rollup data.
|
||||
- Maintenance commands: run `subminer stats cleanup` or `subminer stats cleanup -v` to backfill/repair vocabulary metadata (`headword`, `reading`, POS) and purge stale or excluded rows from `imm_words` on demand; `subminer stats cleanup -l` repairs lifetime summary tables non-destructively (recomputed from per-episode history, so lifetime totals older than the session retention window are kept); `subminer stats cleanup --duplicate-lines` collapses repeated lines left behind by typeset subtitles (see [Repeated Line Cleanup](#repeated-line-cleanup)). `subminer stats rebuild` and `subminer stats backfill` rebuild or backfill rollup data.
|
||||
- Browser page: open `http://127.0.0.1:6969` directly if the local stats server is already running.
|
||||
|
||||
### Dashboard Tabs
|
||||
@@ -125,6 +125,34 @@ Secondary subtitle text (typically English translations) is stored alongside pri
|
||||
|
||||
The Vocabulary tab toolbar includes an **Exclusions** button for hiding words from all vocabulary views. Excluded words are stored in the immersion database, with older browser localStorage exclusions imported on first load after upgrade. They can be managed (restored or cleared) from the exclusion modal. Exclusions affect stat cards, charts, the frequency rank table, and the word list.
|
||||
|
||||
### Repeated Line Cleanup
|
||||
|
||||
Karaoke openings and animated signs are authored as one subtitle event per animation frame, all carrying the same text. Playback reports every one of those frames, so a single OP lyric could be recorded hundreds of times and dominate "Top Repeated Words".
|
||||
|
||||
Recording now collapses those runs as they happen, matching what the subtitle sidebar shows:
|
||||
|
||||
- When the active subtitle source has been parsed, its cue list has already had duplicate events and animation bursts merged. A line landing inside a surviving cue but after that cue's start is a frame the sidebar merged away, and is not recorded.
|
||||
- When no parsed cue covers the live timing, including while a subtitle source is changing or shifted, the strict metadata-free rule applies: a run of identical, contiguous lines each shorter than 0.1s stops being recorded after a few frames. Runs are tracked per line of text, so dual-line karaoke (a kanji and a romaji line frame-flipped together) collapses both lines. Ordinary repeated dialogue, and lines held for a normal beat, always record.
|
||||
|
||||
For stats recorded before this, the Vocabulary tab toolbar has a **Duplicates** button:
|
||||
|
||||
- Pick how far back to look (7 days, 30 days, 90 days, 1 year, or all time). A narrower window does less work and keeps older history untouched.
|
||||
- **Scan** reports the bursts found, the lines they added, and the word and kanji counts they inflated, without writing anything.
|
||||
- **Clean Up** applies exactly what the scan reported: each run collapses to its first line (extended to cover the run), and the removed lines' word and kanji occurrences are subtracted from the vocabulary aggregates.
|
||||
|
||||
The same thing runs from the terminal:
|
||||
|
||||
```bash
|
||||
subminer stats cleanup --duplicate-lines --dry-run --lookback-days 30
|
||||
subminer stats cleanup --duplicate-lines --lookback-days 30
|
||||
```
|
||||
|
||||
`--duplicate-lines` (short: `-d`) picks the cleanup mode, so it cannot be combined with `--vocab` or `--lifetime`, and `--dry-run` and `--lookback-days <days>` only apply to it. Omitting `--lookback-days` scans all history; the value must be at least one day.
|
||||
|
||||
The cleanup chains runs per line of text, so interleaved dual-line karaoke collapses each of its lines. It also removes the short residue the live rule stores before a run is long enough to recognize: a run one frame short of the usual minimum qualifies when every event is under the strict 0.1s bound.
|
||||
|
||||
Runs never cross a session boundary, so rewatching an episode keeps both watches. Session telemetry (watch time, lines seen, tokens seen) and the rollups derived from it are left as recorded: they are cumulative samples taken during playback, and cannot be recomputed for sessions whose raw rows have since been pruned.
|
||||
|
||||
## Retention Defaults
|
||||
|
||||
By default, SubMiner keeps all retention tables and raw data (`0` means keep all) while continuing daily/monthly rollup maintenance:
|
||||
|
||||
@@ -151,6 +151,7 @@ subminer stats -b # start background stats daemon
|
||||
| `subminer stats` | Start the stats server (opens the dashboard when `stats.autoOpenBrowser` is on) |
|
||||
| `subminer stats -b` / `-s` | Start/reuse or stop the background stats daemon |
|
||||
| `subminer stats cleanup` | Backfill vocabulary metadata and prune stale rows (`-v` vocab, `-l` lifetime summaries) |
|
||||
| `subminer stats cleanup -d` | Collapse repeated lines from typeset subs (`--dry-run`, `--lookback-days <n>`) |
|
||||
| `subminer stats rebuild` / `backfill` | Rebuild or backfill rollup data |
|
||||
| `subminer doctor` | Dependency + config + socket diagnostics (`--refresh-known-words` refreshes the known-word cache) |
|
||||
| `subminer settings` | Open the SubMiner settings window |
|
||||
|
||||
@@ -161,10 +161,13 @@ If your subtitle file is out of sync with the audio, SubMiner can resynchronize
|
||||
|
||||
1. Open the subsync modal from the overlay.
|
||||
2. Select the sync engine (alass or ffsubsync).
|
||||
3. For alass, select a reference subtitle track from the video.
|
||||
4. SubMiner runs the sync and reloads the corrected subtitle.
|
||||
3. For alass, pick the **reference** - the subtitle with correct timing. This defaults to the secondary subtitle track. The loaded video file can also be used as the reference (alass extracts the audio itself), but it is never the default.
|
||||
4. Pick the **out-of-sync subtitle** - the track that gets retimed. This defaults to the active primary subtitle track and applies to both engines.
|
||||
5. SubMiner runs the sync and reloads the corrected subtitle into the slot the out-of-sync track came from: retiming the secondary track keeps it secondary and leaves the primary track selected.
|
||||
|
||||
For remote streams, including Jellyfin playback, the modal only offers alass. Jellyfin subtitle URLs are cached as temporary subtitle files so alass can read them, but the video stream is not downloaded. ffsubsync needs direct access to the local media file and is unavailable for stream URLs.
|
||||
The reference and the out-of-sync subtitle must be different tracks; the reference list hides whichever track is selected as the target.
|
||||
|
||||
For remote streams, including Jellyfin playback, the modal only offers alass with a subtitle reference. Jellyfin subtitle URLs are cached as temporary subtitle files so alass can read them, but the video stream is not downloaded. ffsubsync and the video-file reference need direct access to the local media file and are unavailable for stream URLs.
|
||||
|
||||
When you mine a sentence card from the stats dashboard, SubMiner can also use `alass` automatically to align a local English sidecar against the matching local Japanese sidecar before filling the card translation field. The source subtitle files are not modified; SubMiner writes a temporary retimed copy and reuses it while the stats server is running.
|
||||
|
||||
|
||||
@@ -523,7 +523,7 @@
|
||||
// ==========================================
|
||||
// AnkiConnect Integration
|
||||
// Automatic Anki updates and media generation options.
|
||||
// Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, and isKiku.fieldGrouping update live while SubMiner is running.
|
||||
// Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running.
|
||||
// Shared AI provider transport settings are read from top-level ai and typically require restart.
|
||||
// Most other AnkiConnect settings still require restart.
|
||||
// ==========================================
|
||||
@@ -605,7 +605,10 @@
|
||||
"enabled": false, // Enable Kiku-specific mining behaviors (duplicate handling, field grouping). Values: true | false
|
||||
"fieldGrouping": "disabled", // Kiku duplicate-card field grouping mode. Values: auto | manual | disabled
|
||||
"deleteDuplicateInAuto": true // When Kiku field grouping is "auto", delete the duplicate source card after grouping completes. Values: true | false
|
||||
} // Is kiku setting.
|
||||
}, // Is kiku setting.
|
||||
"lapisKiku": {
|
||||
"wordCardKind": "word-and-sentence" // Card-type flag SubMiner marks on Kiku/Lapis word cards. Only one flag is set at a time; the others are cleared. Requires isKiku.enabled or isLapis.enabled. Values: word-and-sentence | click | sentence | audio | none
|
||||
} // Lapis kiku setting.
|
||||
}, // Automatic Anki updates and media generation options.
|
||||
|
||||
// ==========================================
|
||||
|
||||
@@ -171,7 +171,9 @@ Without FFmpeg, card creation still works but audio and image fields will be emp
|
||||
|
||||
**Audio or screenshot generation hangs**
|
||||
|
||||
Media generation has a 30-second timeout (60 seconds for animated AVIF). If your video file is on a slow network mount or the codec requires software decoding, generation may time out. Try:
|
||||
Audio extraction has a 2-minute timeout. SubMiner also limits FFmpeg probing when mpv provides the selected audio stream, which avoids scanning unrelated subtitle and font-attachment streams in large MKV files. Screenshots retain a 30-second timeout, and animated AVIF uses 60 seconds.
|
||||
|
||||
If your video file is on a slow or unresponsive network mount, generation may still time out. Try:
|
||||
|
||||
- Using a local copy of the video file.
|
||||
- Reducing `ankiConnect.media.imageQuality` or switching from `avif` to `static` image type.
|
||||
@@ -227,7 +229,7 @@ Install ffsubsync or configure the path:
|
||||
|
||||
If subtitle sync fails (the error message is prefixed with the engine name):
|
||||
|
||||
- Ensure the reference subtitle track exists in the video (alass requires a source track).
|
||||
- Ensure a reference is selected (alass needs either a second subtitle track or the local video file, and it cannot be the same track that is being retimed).
|
||||
- Check that `ffmpeg` is available (used to extract the internal subtitle track).
|
||||
- Try running the sync tool manually to see detailed error output.
|
||||
- ffsubsync requires local files and cannot handle remote media streams (e.g., streaming URLs).
|
||||
@@ -405,8 +407,9 @@ On any Wayland session that is not Hyprland or Sway (KDE Plasma, GNOME, and othe
|
||||
|
||||
SubMiner handles this automatically:
|
||||
|
||||
- It launches its own window under XWayland (it sets `--ozone-platform-hint=x11`).
|
||||
- Every mpv it launches (via the `subminer` launcher, Jellyfin, or YouTube) is pinned to XWayland too - Wayland environment hints are stripped and an X11 GPU context (`--gpu-context=x11egl,x11`) is applied.
|
||||
- It launches its own window under XWayland (it sets `--ozone-platform=x11`).
|
||||
- Every mpv it launches (via the `subminer` launcher, Jellyfin, or YouTube) is pinned to XWayland too - Wayland environment hints are stripped and an X11 GPU context (`--gpu-context=x11vk,x11egl,x11`) is applied. Only the window context is overridden; your `vo`/`gpu-api` and user shaders are left alone.
|
||||
- Fractional and mixed-monitor display scaling is handled per screen when SubMiner maps XWayland mpv coordinates to the overlay.
|
||||
- While mpv is windowed, the overlay is a managed X11 window owned by the tracked mpv window (`WM_TRANSIENT_FOR`), so it stays above mpv while other foreground X11/Xwayland apps can still cover both windows.
|
||||
- While tracked mpv is fullscreen, SubMiner swaps the visible overlay to a focusable-false X11 override-redirect window. That path can stay above the active fullscreen mpv window without requiring a KDE/KWin-specific rule, and SubMiner hides/releases it when mpv is no longer the active X11/Xwayland window.
|
||||
- The visible overlay is shown inactive on Linux, so normal hover should not steal keyboard focus from mpv.
|
||||
@@ -420,7 +423,7 @@ Requirements: `xdotool`, `xprop`, and `xwininfo` must be installed. SubMiner use
|
||||
This almost always means mpv came up as a **native Wayland** window that the XWayland overlay cannot cover. It happens when mpv is launched **manually** (your own command), because SubMiner can only force XWayland on the mpv processes it launches itself. Fix it one of these ways:
|
||||
|
||||
- Launch playback through SubMiner (the `subminer` launcher or the tray), which forces XWayland for you, or
|
||||
- Force XWayland in your own mpv invocation, e.g. `mpv --gpu-context=x11egl …`, or launch with `WAYLAND_DISPLAY= mpv …`, or set `gpu-context=x11egl` in your `mpv.conf`.
|
||||
- Force XWayland in your own mpv invocation, e.g. `mpv --gpu-context=x11vk,x11egl,x11 …`, or launch with `WAYLAND_DISPLAY= mpv …`, or set `gpu-context=x11vk` (Vulkan) / `gpu-context=x11egl` (OpenGL) in your `mpv.conf`.
|
||||
|
||||
To confirm mpv is on XWayland, `xdotool search --class mpv` should return a window id (a native Wayland mpv returns nothing).
|
||||
|
||||
|
||||
+10
-2
@@ -95,6 +95,8 @@ subminer texthooker # Texthooker-only mode (-o also opens the brow
|
||||
subminer stats -b # Start/reuse the background stats daemon
|
||||
subminer stats -s # Stop the background stats daemon
|
||||
subminer stats cleanup # Backfill vocabulary metadata, prune stale rows
|
||||
subminer stats cleanup -d --dry-run # Preview cleanup of repeated typeset subtitle lines
|
||||
subminer stats cleanup -d --lookback-days 30 # Clean only lines recorded in the last 30 days
|
||||
subminer stats rebuild # Rebuild rollup data
|
||||
subminer doctor --refresh-known-words # Refresh the known-word cache
|
||||
subminer logs -e # Export a sanitized log ZIP and print its path
|
||||
@@ -107,6 +109,8 @@ subminer app --stop # Stop the background app
|
||||
subminer --version # Print the launcher's version
|
||||
```
|
||||
|
||||
`stats cleanup` runs one mode per invocation: `-v`/`--vocab` (the default), `-l`/`--lifetime`, or `-d`/`--duplicate-lines`; explicitly selected modes cannot be combined. `--dry-run` and `--lookback-days <days>` apply to `--duplicate-lines` only and are rejected without it; `--lookback-days` must be at least one day, and leaving it off scans all history.
|
||||
|
||||
Jellyfin, cross-machine sync, and character-dictionary commands have their own sections: [Jellyfin](/jellyfin-integration), [Sync Between Machines](/launcher-script#sync-between-machines), and [Character Dictionary](/character-dictionary).
|
||||
|
||||
</details>
|
||||
@@ -137,7 +141,7 @@ SubMiner.AppImage --start --log-level debug # Verbose logging without dev mode
|
||||
SubMiner.AppImage --help # Show all options
|
||||
```
|
||||
|
||||
The remaining flags are internal or scripting-only surfaces: the `--jellyfin-*` family (login, library listing, item playback, cast announce), `--sync-cli` (the app's headless sync entrypoint that `subminer sync` proxies to), `--dictionary-candidates` / `--dictionary-select`, and `--playback-feedback <text>`. Run `SubMiner.AppImage --help` for the complete list. The previous `--open-animetosho` flag is still accepted as a deprecated alias for `--open-tsukihime`.
|
||||
The remaining flags are internal or scripting-only surfaces: the `--jellyfin-*` family (login, library listing, item playback, cast announce), `--sync-cli` (the app's headless sync entrypoint that `subminer sync` proxies to), the `--stats-cleanup-*` family that `subminer stats cleanup` forwards (`--stats-cleanup-vocab`, `--stats-cleanup-lifetime`, `--stats-cleanup-duplicate-lines`, and its `--stats-cleanup-dry-run` / `--stats-cleanup-lookback-days <days>` modifiers), `--dictionary-candidates` / `--dictionary-select`, and `--playback-feedback <text>`. Run `SubMiner.AppImage --help` for the complete list. The previous `--open-animetosho` flag is still accepted as a deprecated alias for `--open-tsukihime`.
|
||||
|
||||
</details>
|
||||
|
||||
@@ -145,11 +149,13 @@ The tray menu includes `Export Logs`, which creates the same sanitized local-dat
|
||||
|
||||
Once Jellyfin is configured, the tray menu includes `Jellyfin Discovery` for starting or stopping cast discovery in the current app session without changing config.
|
||||
|
||||
The tray menu also includes `View Changelog`, which opens the in-app changelog modal. It fetches the changelog from the newest published release, so you see release notes for versions newer than the one you run; if the download fails it falls back to the changelog bundled with your install and says so. Versions in the current `0.x` line are expanded by default and older lines are folded, matching this site's [Changelog](/changelog). A badge marks the version you have installed, and newer versions are tagged `New`. The same modal opens from the `What's New` button on the update-available overlay notification.
|
||||
|
||||
### Logging and App Mode
|
||||
|
||||
- `--log-level` controls logger verbosity.
|
||||
- `--dev` and `--debug` are app/dev-mode switches; they are not log-level aliases.
|
||||
- `--background` defaults to quieter logging (`warn`) unless `--log-level` is set.
|
||||
- `--background` starts at the default quieter logging level (`warn`), then follows `logging.level` after config loads. An explicit `--log-level` remains the override.
|
||||
- `--background` launched from a terminal detaches and returns the prompt; stop it with tray Quit or `SubMiner.AppImage --stop` (`SubMiner.exe --stop` on Windows).
|
||||
- Linux desktop launcher starts SubMiner with `--background` by default (via electron-builder `linux.executableArgs`).
|
||||
- On Hyprland and other Wayland compositors, the tray icon appears only when your panel provides a StatusNotifier/AppIndicator tray host.
|
||||
@@ -368,6 +374,8 @@ Press `V` to cycle the primary SubMiner subtitle bar through hidden → visible
|
||||
|
||||
`Ctrl/Cmd+/` opens the session help modal with the current overlay and mpv keybindings. The same help view is also available through the `y-h` chord in mpv.
|
||||
|
||||
The changelog modal (tray > `View Changelog`) works the same way: it renders over mpv when a video is playing and in its own window otherwise. Use `J`/`K` or the arrow keys to move between versions, `Enter` to fold or unfold one, `R` to refetch, and `Esc` to close.
|
||||
|
||||
Hovering over subtitle text pauses mpv by default; leaving resumes it. Yomitan popups also pause playback by default. Set `subtitleStyle.autoPauseVideoOnHover: false` or `subtitleStyle.autoPauseVideoOnYomitanPopup: false` to disable either behavior.
|
||||
|
||||
### Drag-and-Drop
|
||||
|
||||
@@ -64,7 +64,7 @@ Use the basic subtitle websocket when you only need the current subtitle line as
|
||||
- **Client auth:** none
|
||||
- **Reconnects:** client-managed
|
||||
|
||||
When a client connects, SubMiner immediately sends the latest subtitle payload if one is available. After that, it pushes a new message each time the current subtitle changes.
|
||||
When a client connects, SubMiner immediately sends the latest subtitle payload if one is available. After that, it pushes a new message each time the current subtitle changes. Annotation-only upgrades do not repeat the same line on this basic stream.
|
||||
|
||||
#### Message shape
|
||||
|
||||
@@ -96,6 +96,8 @@ Use the annotation websocket for custom clients that want the same structured to
|
||||
|
||||
In practice, if you are building a new client, prefer `annotationWebsocket` unless you specifically need compatibility with an existing `websocket` consumer.
|
||||
|
||||
On a tokenization cache miss, this stream first sends the cue as plain text with an empty `tokens` array, then sends the annotated replacement when tokenization finishes. Treat each message as the complete current state, replacing the previous payload.
|
||||
|
||||
#### Message shape
|
||||
|
||||
```json
|
||||
|
||||
@@ -11,7 +11,7 @@ SubMiner auto-loads Japanese subtitles when you play a YouTube URL, giving you t
|
||||
|
||||
When SubMiner detects a YouTube URL (or `ytsearch:` target), it pauses mpv at startup and runs a subtitle pipeline before resuming playback:
|
||||
|
||||
1. **Probe** --- `yt-dlp --dump-single-json` extracts all available subtitle tracks (manual uploads and auto-generated captions) along with video metadata.
|
||||
1. **Probe** --- `yt-dlp --dump-single-json` extracts all available subtitle tracks (manual uploads and auto-generated captions) along with video metadata. Every yt-dlp call passes `--no-playlist`, so playlist links (for example a Watch Later URL with `list=`/`index=`) resolve to the single video instead of the whole playlist.
|
||||
2. **Discover** --- Each track is normalized into a `YoutubeTrackOption` with language code, kind (`manual` or `auto`), display label, and direct download URL.
|
||||
3. **Select** --- SubMiner picks the best primary track (Japanese, preferring manual over auto) and secondary track (English, preferring manual over auto).
|
||||
4. **Download** --- Selected tracks are fetched via direct URL when available, falling back to `yt-dlp --write-subs` / `--write-auto-subs`. YouTube TimedText XML formats (`srv1`/`srv2`/`srv3`) are converted to VTT on the fly. Auto-generated VTT captions are normalized to remove rolling-caption duplication.
|
||||
|
||||
@@ -64,18 +64,23 @@ External subtitle files only (SRT, VTT, ASS). Embedded subtitle tracks are out o
|
||||
A cue parser extracts both timing and text content from subtitle files for prefetching.
|
||||
|
||||
**Parsed cue structure:**
|
||||
|
||||
```typescript
|
||||
interface SubtitleCue {
|
||||
startTime: number; // seconds
|
||||
endTime: number; // seconds
|
||||
text: string; // raw subtitle text
|
||||
startTime: number; // seconds
|
||||
endTime: number; // seconds
|
||||
text: string; // plain text, decoded from the source format
|
||||
}
|
||||
```
|
||||
|
||||
**Supported formats:**
|
||||
|
||||
- SRT/VTT: Regex-based parsing of timing lines + text content between timing blocks.
|
||||
- ASS: Parse `[Events]` section, extract `Dialogue:` lines, split on the first 9 commas only (ASS v4+ has 10 fields; the last field is Text which can itself contain commas). Strip ASS override tags (`{\...}`) from the text before storing.
|
||||
ASS text fields contain inline override tags like `{\b1}`, `{\an8}`, `{\fad(200,300)}`. The cue parser strips these during extraction so the tokenizer receives clean text.
|
||||
- ASS: Parse `[Events]` section, extract `Dialogue:` lines, read the field order from the `Format:` row, and take everything after the Text field index as the text (Text can itself contain commas).
|
||||
|
||||
**ASS decoding.** The parser is where ASS text is decoded, once, via `assToPlainText()` in `src/core/services/ass-text.ts`. That decoder mirrors mpv's `ass_to_plaintext` so a cue read from a file reads identically to the same line arriving live on `sub-text`: `{...}` override blocks are markup, `\pN … \p0` vector drawing runs are dropped rather than shown as text, `\N`/`\n`/`\h` are the only escapes (`\{`, `\}` and `\\` are not), and an unclosed `{` is rendered verbatim. Every layer downstream — renderer, timing tracker, tokenizer, tokenization cache keys — receives plain text and uses `normalizePlainSubtitleText()` for whitespace only, so nothing decodes the same string twice and one authored line always maps to one cache key.
|
||||
|
||||
**Duplicate collapsing.** Typeset scripts emit one `Dialogue:` event per animation frame, plus layered copies of the same line. The parser collapses identical text over an identical span unconditionally, and collapses contiguous same-text runs of at least three events when the run looks like an animation. For ASS that means shared style and actor plus authoring evidence: a temporal tag (`\t`, `\move`, `\k`/`\kf`/`\ko`/`\K`, or anything wrapped in `\t(...)`), an animated `Effect` column (`Karaoke`, `Banner`, `Scroll`), or override values that change across the run. Static tags shared by every event (`\pos`, an identical `\clip`) are not evidence. SRT/VTT carry no such metadata, so there collapsing needs at least five contiguous events all under 0.1s — the frame timing left behind by ASS-to-SRT conversion. The parser keeps this authoring metadata (style, actor, layer, `Effect`, parsed override commands, source order) private; `parseSubtitleCues()` returns only `SubtitleCue`.
|
||||
|
||||
#### Prefetch Service Lifecycle
|
||||
|
||||
@@ -153,6 +158,7 @@ tokens (already have frequencyRank values from parser-level applyFrequencyRanks)
|
||||
### Dependency Analysis
|
||||
|
||||
All annotations either depend on MeCab POS data or benefit from running after it:
|
||||
|
||||
- **Known word marking:** Needs base tokens (surface/headword). No POS dependency, but no reason to run separately.
|
||||
- **Frequency filtering:** Uses `pos1Exclusions` and `pos2Exclusions` to clear frequency ranks on excluded tokens (particles, noise). Depends on MeCab POS data.
|
||||
- **JLPT marking:** Uses `shouldIgnoreJlptForMecabPos1` to filter. Depends on MeCab POS data.
|
||||
@@ -169,18 +175,14 @@ function annotateTokens(tokens, deps, options): MergedToken[] {
|
||||
|
||||
// Single pass: known word + frequency filtering + JLPT computed together
|
||||
const annotated = tokens.map((token) => {
|
||||
const isKnown = nPlusOneEnabled
|
||||
? token.isKnown || computeIsKnown(token, deps)
|
||||
: false;
|
||||
const isKnown = nPlusOneEnabled ? token.isKnown || computeIsKnown(token, deps) : false;
|
||||
|
||||
// Filter frequency rank using POS exclusions (rank values already set at parser level)
|
||||
const frequencyRank = frequencyEnabled
|
||||
? filterFrequencyRank(token, pos1Exclusions, pos2Exclusions)
|
||||
: undefined;
|
||||
|
||||
const jlptLevel = jlptEnabled
|
||||
? computeJlptLevel(token, deps.getJlptLevel)
|
||||
: undefined;
|
||||
const jlptLevel = jlptEnabled ? computeJlptLevel(token, deps.getJlptLevel) : undefined;
|
||||
|
||||
return { ...token, isKnown, frequencyRank, jlptLevel };
|
||||
});
|
||||
@@ -221,6 +223,7 @@ Replace `document.createElement('span')` calls in the renderer with `templateSpa
|
||||
### Current Behavior
|
||||
|
||||
In `renderWithTokens` (`subtitle-render.ts`), each render cycle:
|
||||
|
||||
1. Clears DOM with `innerHTML = ''`
|
||||
2. Creates a `DocumentFragment`
|
||||
3. Calls `document.createElement('span')` for each token (~10-15 per subtitle)
|
||||
@@ -256,27 +259,30 @@ Full recycling (collecting old nodes, clearing attributes, reusing them) require
|
||||
|
||||
## Combined Impact Summary
|
||||
|
||||
| Scenario | Before | After | Improvement |
|
||||
|----------|--------|-------|-------------|
|
||||
| Normal playback (prefetch-warmed) | ~200-320ms | ~30-50ms | ~80-85% |
|
||||
| Cache hit (repeated subtitle) | ~72ms | ~55-65ms | ~10-20% |
|
||||
| Cache miss (immediate seek) | ~200-320ms | ~150-260ms | ~20-25% |
|
||||
| Scenario | Before | After | Improvement |
|
||||
| --------------------------------- | ---------- | ---------- | ----------- |
|
||||
| Normal playback (prefetch-warmed) | ~200-320ms | ~30-50ms | ~80-85% |
|
||||
| Cache hit (repeated subtitle) | ~72ms | ~55-65ms | ~10-20% |
|
||||
| Cache miss (immediate seek) | ~200-320ms | ~150-260ms | ~20-25% |
|
||||
|
||||
---
|
||||
|
||||
## Files Summary
|
||||
|
||||
### New Files
|
||||
|
||||
- `src/core/services/subtitle-prefetch.ts`
|
||||
- `src/core/services/subtitle-cue-parser.ts`
|
||||
|
||||
### Modified Files
|
||||
|
||||
- `src/core/services/subtitle-processing-controller.ts` (expose `preCacheTokenization`)
|
||||
- `src/core/services/tokenizer/annotation-stage.ts` (batched single-pass)
|
||||
- `src/renderer/subtitle-render.ts` (template cloneNode)
|
||||
- `src/main.ts` (wire up prefetch service)
|
||||
|
||||
### Test Files
|
||||
|
||||
- New tests for subtitle cue parser (SRT, VTT, ASS formats)
|
||||
- New tests for subtitle prefetch service (priority window, seek, pause/resume)
|
||||
- Updated tests for annotation stage (same behavior, new implementation)
|
||||
|
||||
@@ -25,6 +25,7 @@ Read when: you need to find the owner module for a behavior or test surface
|
||||
- Anki workflow: `src/anki-integration/`, `src/core/services/anki-jimaku*.ts`
|
||||
- Immersion tracking: `src/core/services/immersion-tracker/`
|
||||
Includes stats storage/query schema such as `imm_videos`, `imm_media_art`, and `imm_youtube_videos` for per-video and YouTube-specific library metadata.
|
||||
`delete-maintenance-scheduler.ts` coalesces and serializes stats deletes; the expensive work runs in `delete-maintenance-worker-thread.ts` while the tracker queues playback writes. Each batch uses one transaction, lexical update, rollup refresh, and incremental lifetime subtraction (`planLifetimeRemovals`/`applyLifetimeRemovals` in `lifetime.ts`). Merges, moves, AniList reassignments, and `stats cleanup -l` use `repairLifetimeSummariesFromMedia` (recompute from the per-video media ledger). The full lifetime rebuild survives only as the empty-table bootstrap — anywhere else it would collapse lifetime totals to the session retention window.
|
||||
- AniList tracking + character dictionary: `src/core/services/anilist/`, `src/main/runtime/composers/anilist-*`, `src/main/character-dictionary-runtime.ts`, `src/main/character-dictionary-runtime/`
|
||||
- Jellyfin integration: `src/core/services/jellyfin*.ts`, `src/main/runtime/composers/jellyfin-*`
|
||||
- Window trackers: `src/window-trackers/`
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# Subtitle Overlay Priming
|
||||
|
||||
Status: active
|
||||
Last verified: 2026-06-14
|
||||
Last verified: 2026-08-04
|
||||
Owner: Kyle Yasuda
|
||||
Read when: debugging subtitle state or blank Linux/X11 overlay windows when the visible overlay is shown or recreated
|
||||
|
||||
@@ -47,18 +47,43 @@ subtitles do not draw.
|
||||
`emitSubtitle(payload)` and `refreshCurrentSubtitle(text)`, then prime secondary subtitles.
|
||||
6. Tokenization cache hit: call `consumeCachedSubtitle(text)`, `onSubtitleChange(text)`, and
|
||||
`emitSubtitle(cachedPayload)`, then prime secondary subtitles.
|
||||
7. Cache miss: call `refreshCurrentSubtitle(text)` and let normal tokenization emit the final
|
||||
payload.
|
||||
7. Cache miss: call `refreshCurrentSubtitle(text)`. Normal processing emits a plain payload
|
||||
synchronously, then replaces it with the tokenized payload when ready.
|
||||
|
||||
In `src/main.ts`, both `onSubtitleChange` and `refreshCurrentSubtitle` pause
|
||||
`subtitlePrefetchService`, notify it with `onSeek(lastObservedTimePos)`, and then call the matching
|
||||
`subtitleProcessingController` method. This gives the visible overlay priority over background
|
||||
prefetch work and re-centers prefetch around the live playback time.
|
||||
Both `onSubtitleChange` and `refreshCurrentSubtitle` pause `subtitlePrefetchService` and then call
|
||||
the matching `subtitleProcessingController` method, giving the visible overlay priority over
|
||||
background prefetch work. Prefetch is not re-centered here: restarting the run per line
|
||||
(`onSeek`) discarded the in-flight tokenization every time the subtitle changed, so only real
|
||||
seeks restart it (see `onTimePosUpdate` in `src/main.ts`).
|
||||
|
||||
On an uncached autoplay prime the raw payload is emitted here and reported to the controller with
|
||||
`notePlainSubtitleEmitted`, so the controller skips its own plain emit for that line and the
|
||||
overlay receives one plain payload followed by the annotated one.
|
||||
|
||||
The pause is released by the controller's `onProcessingSettled` callback, which fires once it has
|
||||
no work left. Emits do not release it: the first emit for an uncached line is the plain payload
|
||||
that precedes tokenization, and a run can finish without emitting at all (a suppressed duplicate,
|
||||
a failed tokenization). Both controller methods return whether processing is now pending, and the
|
||||
caller resumes immediately when it is not — a repeated subtitle schedules no work, so no settle is
|
||||
coming and prefetching would otherwise idle for the rest of the cue.
|
||||
|
||||
## Live Cue Delivery
|
||||
|
||||
- A tokenization cache miss emits the plain cue synchronously. Tokenization remains serialized so
|
||||
live work does not contend for Yomitan state.
|
||||
- If a newer cue arrives while an older line is still tokenizing, the newer plain cue or empty
|
||||
clear payload is emitted immediately. The older tokenization result is dropped before it can
|
||||
replace the current cue.
|
||||
- The current cue upgrades in place when its tokens and annotations are ready. This can reflow text
|
||||
or character images, but cue visibility does not wait for that work.
|
||||
|
||||
## Emitted State
|
||||
|
||||
- `emitSubtitle(payload)` maps to `emitSubtitlePayload(payload)`, which sends the normal
|
||||
annotated subtitle payload to overlay windows and subtitle websocket listeners.
|
||||
- `emitSubtitle(payload)` maps to `emitSubtitlePayload(payload)`. Overlay windows and annotation
|
||||
websocket listeners receive both the immediate plain cue and its later annotation upgrade.
|
||||
- The basic subtitle websocket receives the immediate plain cue only. Because its serialized
|
||||
payload discards annotations, the later upgrade would be an identical duplicate and is skipped
|
||||
when text and cue timing match.
|
||||
- Secondary priming reads mpv `secondary-sub-text`, stores it in
|
||||
`mpvClient.currentSecondarySubText`, and broadcasts `secondary-subtitle:set` to overlay windows.
|
||||
- If secondary `requestProperty` fails, the primary flow stays complete and only a debug line is
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# Documentation Catalog
|
||||
|
||||
Status: active
|
||||
Last verified: 2026-05-23
|
||||
Last verified: 2026-08-13
|
||||
Owner: Kyle Yasuda
|
||||
Read when: finding internal docs or checking verification status
|
||||
|
||||
@@ -17,10 +17,10 @@ Read when: finding internal docs or checking verification status
|
||||
| KB rules | `docs/knowledge-base/README.md` | active | 2026-05-23 | maintenance policy |
|
||||
| Core beliefs | `docs/knowledge-base/core-beliefs.md` | active | 2026-03-13 | agent-first principles |
|
||||
| Quality scorecard | `docs/knowledge-base/quality.md` | active | 2026-03-13 | quality grades and gaps |
|
||||
| Workflow index | `docs/workflow/README.md` | active | 2026-05-23 | execution map |
|
||||
| Workflow index | `docs/workflow/README.md` | active | 2026-08-13 | execution map |
|
||||
| Planning guide | `docs/workflow/planning.md` | active | 2026-05-23 | lightweight vs execution plans |
|
||||
| Agent plugins | `docs/workflow/agent-plugins.md` | active | 2026-05-23 | repo-local agent workflow plugin ownership |
|
||||
| Verification guide | `docs/workflow/verification.md` | active | 2026-05-23 | maintained verification lanes |
|
||||
| Agent skills | `docs/workflow/agent-skills.md` | active | 2026-08-13 | repo-local workflow skill ownership |
|
||||
| Verification guide | `docs/workflow/verification.md` | active | 2026-08-13 | maintained verification lanes |
|
||||
| Release guide | `docs/RELEASING.md` | active | 2026-05-23 | release checklist |
|
||||
|
||||
## Update Rules
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,184 +0,0 @@
|
||||
# Library Summary Replaces Per-Day Trends — Design
|
||||
|
||||
**Status:** Draft
|
||||
**Date:** 2026-04-09
|
||||
**Scope:** `stats/` frontend, `src/core/services/immersion-tracker/query-trends.ts` backend
|
||||
|
||||
## Problem
|
||||
|
||||
The "Library — Per Day" section on the stats Trends tab (`stats/src/components/trends/TrendsTab.tsx:224-254`) renders six stacked-area charts — Videos, Watch Time, Cards, Words, Lookups, and Lookups/100w, each broken down per title per day.
|
||||
|
||||
In practice these charts are not useful:
|
||||
|
||||
- Most titles only have activity on one or two days in a window, so they render as isolated bumps on a noisy baseline.
|
||||
- Stacking 7+ titles with mostly-zero days makes individual lines hard to follow.
|
||||
- The top "Activity" and "Period Trends" sections already answer "what am I doing per day" globally.
|
||||
- The "Library — Cumulative" section directly below already answers "which titles am I progressing through" with less noise.
|
||||
|
||||
The per-day section occupies significant vertical space without carrying its weight, and the user has confirmed it should be replaced.
|
||||
|
||||
## Goal
|
||||
|
||||
Replace the six per-day stacked charts with a single "Library — Summary" section that surfaces per-title aggregate statistics over the selected date range. The new view should make it trivially easy to answer: "For the selected window, which titles am I spending time on, how much mining output have they produced, and how efficient is my lookup rate on each?"
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Changing the "Library — Cumulative" section (stays as-is).
|
||||
- Changing the "Activity", "Period Trends", or "Patterns" sections.
|
||||
- Adding a new API endpoint — the existing dashboard endpoint is extended in place.
|
||||
- Renaming internal `anime*` data-model identifiers (`animeId`, `imm_anime`, etc.). Those stay per the convention established in `c5e778d7`; only new fields/types/user-visible strings use generic "title"/"library" wording.
|
||||
- Supporting a true all-time library view on the Trends tab. If that's ever wanted, it belongs on a different tab.
|
||||
|
||||
## Solution Overview
|
||||
|
||||
Delete the "Library — Per Day" section. In its place, add "Library — Summary", composed of:
|
||||
|
||||
1. A horizontal-bar leaderboard chart of watch time per title (top 10, descending).
|
||||
2. A sortable table of every title with activity in the selected window, with columns: Title, Watch Time, Videos, Sessions, Cards, Words, Lookups, Lookups/100w, Date Range.
|
||||
|
||||
Both controls are scoped to the top-of-page date range selector. The existing shared Anime Visibility filter continues to work — it now gates Summary + Cumulative instead of Per-Day + Cumulative.
|
||||
|
||||
## Backend
|
||||
|
||||
### New type
|
||||
|
||||
Add to `stats/src/types/stats.ts` and the backend query module:
|
||||
|
||||
```ts
|
||||
type LibrarySummaryRow = {
|
||||
title: string; // display title — anime series, YouTube video title, etc.
|
||||
watchTimeMin: number; // sum(total_active_min) across the window
|
||||
videos: number; // distinct video_id count
|
||||
sessions: number; // session count from imm_sessions
|
||||
cards: number; // sum(total_cards)
|
||||
words: number; // sum(total_tokens_seen)
|
||||
lookups: number; // sum(lookup_count) from imm_sessions
|
||||
lookupsPerHundred: number | null; // lookups / words * 100, null when words == 0
|
||||
firstWatched: number; // min(rollup_day) as epoch day, within the window
|
||||
lastWatched: number; // max(rollup_day) as epoch day, within the window
|
||||
};
|
||||
```
|
||||
|
||||
### Query changes in `src/core/services/immersion-tracker/query-trends.ts`
|
||||
|
||||
- Add `librarySummary: LibrarySummaryRow[]` to `TrendsDashboardQueryResult`.
|
||||
- Populate it from a single aggregating query over `imm_daily_rollups` joined to `imm_videos` → `imm_anime`, filtered by `rollup_day` within the selected window. Session count and lookup count come from `imm_sessions` aggregated by `video_id` and then grouped by the parent library entry. Use a single query (or at most two joined/unioned) — no N+1.
|
||||
- `imm_anime` is the generic library-grouping table; anime series, YouTube videos, and yt-dlp imports all land there. The internal table name stays `imm_anime`; only the new field uses generic naming.
|
||||
- Return rows pre-sorted by `watchTimeMin` descending so the leaderboard is zero-cost and the table default sort matches.
|
||||
- Emit `lookupsPerHundred: null` when `words == 0`.
|
||||
|
||||
### Removed from API response
|
||||
|
||||
Drop the entire `animePerDay` field from `TrendsDashboardQueryResult` (both backend in `src/core/services/immersion-tracker/query-trends.ts` and frontend in `stats/src/types/stats.ts`).
|
||||
|
||||
Internally, the existing helpers (`buildPerAnimeFromDailyRollups`, `buildEpisodesPerAnimeFromDailyRollups`) are still used as intermediates to build `animeCumulative.*` via `buildCumulativePerAnime`. Keep those helpers — just scope their output to local variables inside `getTrendsDashboard` instead of exposing them on the response. The `buildPerAnimeFromSessions` call for lookups and the `buildLookupsPerHundredPerAnime` helper become unused and can be deleted.
|
||||
|
||||
Before removing `animePerDay` from the frontend type, verify no other file under `stats/src/` references it. Based on current inspection, only `TrendsTab.tsx` and `stats/src/types/stats.ts` touch it.
|
||||
|
||||
## Frontend
|
||||
|
||||
### New component: `stats/src/components/trends/LibrarySummarySection.tsx`
|
||||
|
||||
Owns the header, leaderboard chart, visibility-filtered data, and the table. Keeps `TrendsTab.tsx` from growing. Component props: `{ rows: LibrarySummaryRow[]; hiddenTitles: ReadonlySet<string>; windowStart: Date; windowEnd: Date }`.
|
||||
|
||||
Internal state: `useState<{ column: ColumnId; direction: 'asc' | 'desc' }>` for sort, defaulting to `{ column: 'watchTimeMin', direction: 'desc' }`.
|
||||
|
||||
### Layout
|
||||
|
||||
Replaces `TrendsTab.tsx:224-254`:
|
||||
|
||||
```
|
||||
[SectionHeader: "Library — Summary"]
|
||||
[AnimeVisibilityFilter — unchanged, shared with Cumulative below]
|
||||
[Card, col-span-full: Leaderboard — horizontal bar chart, ~260px tall]
|
||||
[Card, col-span-full: Sortable table, auto height up to ~480px with internal scroll]
|
||||
```
|
||||
|
||||
Both cards use the existing chart/card wrapper styling.
|
||||
|
||||
### Leaderboard chart
|
||||
|
||||
- Recharts horizontal bar chart (matches the rest of the page — existing charts use `recharts`, not ECharts).
|
||||
- Top 10 titles by watch time. If fewer titles have activity, render what's there.
|
||||
- Y-axis: title (category), truncated with ellipsis at container width; full title visible in the Recharts tooltip.
|
||||
- X-axis: minutes (number).
|
||||
- Use `layout="vertical"` with `YAxis dataKey="title" type="category"` and `XAxis type="number"`.
|
||||
- Single series color: `#8aadf4` (matching the existing Watch Time color).
|
||||
- Reuse `CHART_DEFAULTS`, `CHART_THEME`, `TOOLTIP_CONTENT_STYLE` from `stats/src/lib/chart-theme.ts` so theming matches the rest of the dashboard.
|
||||
- Chart order is fixed at watch-time desc regardless of table sort — the leaderboard's meaning is fixed.
|
||||
|
||||
### Table
|
||||
|
||||
- Plain HTML `<table>` with Tailwind classes. No new deps.
|
||||
- Columns, in order:
|
||||
1. **Title** — left-aligned, sticky, truncated with ellipsis, full title on hover.
|
||||
2. **Watch Time** — formatted `Xh Ym` when ≥60 min, else `Xm`.
|
||||
3. **Videos** — integer.
|
||||
4. **Sessions** — integer.
|
||||
5. **Cards** — integer.
|
||||
6. **Words** — integer.
|
||||
7. **Lookups** — integer.
|
||||
8. **Lookups/100w** — one decimal place, `—` when null.
|
||||
9. **Date Range** — `Mon D → Mon D` using the title's `firstWatched` / `lastWatched` within the window.
|
||||
- Click a column header to sort; click again to reverse. Visual arrow on the active column.
|
||||
- Numeric columns right-aligned.
|
||||
- Null `lookupsPerHundred` sorts as the lowest value in both directions (consistent with "no data").
|
||||
- Row hover highlight; no row click action (read-only view).
|
||||
- Empty state: "No library activity in the selected window."
|
||||
|
||||
### Visibility filter integration
|
||||
|
||||
Hiding a title via `AnimeVisibilityFilter` removes it from both the leaderboard and the table. The filter's set of available titles is built from the union of titles that appear in `librarySummary` and the existing `animeCumulative.*` arrays (matches current behavior in `buildAnimeVisibilityOptions`).
|
||||
|
||||
### `TrendsTab.tsx` changes
|
||||
|
||||
- Remove the `filteredEpisodesPerAnime`, `filteredWatchTimePerAnime`, `filteredCardsPerAnime`, `filteredWordsPerAnime`, `filteredLookupsPerAnime`, `filteredLookupsPerHundredPerAnime` locals.
|
||||
- Remove the six `<StackedTrendChart>` calls in the "Library — Per Day" section.
|
||||
- Remove the `<SectionHeader>Library — Per Day</SectionHeader>` and the `<AnimeVisibilityFilter>` from that position.
|
||||
- Insert `<SectionHeader>Library — Summary</SectionHeader>` + `<AnimeVisibilityFilter>` + `<LibrarySummarySection>` in the same place.
|
||||
- Update `buildAnimeVisibilityOptions` input to use `librarySummary` titles instead of the six dropped `animePerDay.*` arrays.
|
||||
|
||||
## Data flow
|
||||
|
||||
1. `useTrends(range, groupBy)` calls `/api/stats/trends/dashboard`.
|
||||
2. Response now includes `librarySummary` (sorted by watch time desc).
|
||||
3. `TrendsTab` holds the shared `hiddenAnime` set (unchanged).
|
||||
4. `LibrarySummarySection` receives `librarySummary` + `hiddenAnime`, filters out hidden rows, renders the leaderboard from the top-10 slice of the filtered list, renders the table from the filtered list with local sort state applied.
|
||||
5. Date-range selector changes trigger a new fetch; `groupBy` toggle does not affect the summary section (it's always window-total).
|
||||
|
||||
## Edge cases
|
||||
|
||||
- **No activity in window:** Section renders header + empty-state card. Leaderboard card hidden. Visibility filter hidden.
|
||||
- **One title only:** Leaderboard renders a single bar; table renders one row. No special-casing.
|
||||
- **Title with zero words but non-zero lookups:** `lookupsPerHundred` is `null`, rendered as `—`. Sort treats null as lowest.
|
||||
- **Title with zero cards/lookups/words but non-zero watch time:** Normal zero rendering, still shown.
|
||||
- **Very long titles:** Ellipsis in chart y-axis labels and table title column; full title in `title` attribute / ECharts tooltip.
|
||||
- **Mixed sources (anime + YouTube):** No special case — both land in `imm_anime` and are grouped uniformly.
|
||||
|
||||
## Testing
|
||||
|
||||
### Backend (`query-trends.ts`)
|
||||
|
||||
New unit tests, following the existing pattern:
|
||||
|
||||
1. Empty window returns `librarySummary: []`.
|
||||
2. Single title with a few rollups: all aggregates are correct; `firstWatched`/`lastWatched` match the bounding days within the window.
|
||||
3. Multiple titles: rows returned sorted by watch time desc.
|
||||
4. Mixed sources (anime-style + YouTube-style entries in `imm_anime`): both appear in the summary with their own aggregates.
|
||||
5. Title with `words == 0`: `lookupsPerHundred` is `null`.
|
||||
6. Date range excludes some rollups: excluded rollups are not counted; `firstWatched`/`lastWatched` reflect only within-window activity.
|
||||
7. `sessions` and `lookups` come from `imm_sessions`, not `imm_daily_rollups`, and are correctly attributed to the parent library entry.
|
||||
|
||||
### Frontend
|
||||
|
||||
- Existing Trends tab smoke test should continue to pass after wiring.
|
||||
- Optional: a targeted render test for `LibrarySummarySection` (empty state, single title, sort toggle, visibility filter interaction). Not required for merge if the smoke test exercises the happy path.
|
||||
|
||||
## Release / docs
|
||||
|
||||
- One fragment in `changes/*.md` summarizing the replacement.
|
||||
- No user-facing docs (`docs-site/`) changes unless the per-day section was documented there — verify during implementation.
|
||||
|
||||
## Open items
|
||||
|
||||
None.
|
||||
@@ -1,347 +0,0 @@
|
||||
# Stats Dashboard Feedback Pass — Design
|
||||
|
||||
Date: 2026-04-09
|
||||
Scope: Stats dashboard UX follow-ups from user feedback (items 1–7).
|
||||
Delivery: **Single PR**, broken into logically scoped commits.
|
||||
|
||||
## Goals
|
||||
|
||||
Address seven concrete pieces of feedback against the Statistics menu:
|
||||
|
||||
1. Library — collapse episodes behind a per-series dropdown.
|
||||
2. Sessions — roll up multiple sessions of the same episode within a day.
|
||||
3. Trends — add a 365d range option.
|
||||
4. Library — delete an episode (video) from its detail view.
|
||||
5. Vocabulary — tighten spacing between word and reading in the Top 50 table.
|
||||
6. Episode detail — hide cards whose Anki notes have been deleted.
|
||||
7. Trend/watch charts — add gridlines, fix tick legibility, unify theming.
|
||||
|
||||
Out of scope for this pass: English-token ingestion cleanup and Overview stat-card drill-downs (feedback items 8 and 9). Those require a larger design decision and a migration respectively.
|
||||
|
||||
## Files touched (inventory)
|
||||
|
||||
Dashboard (`stats/src/`):
|
||||
- `components/library/LibraryTab.tsx` — collapsible groups (item 1).
|
||||
- `components/library/MediaDetailView.tsx`, `components/library/MediaHeader.tsx` — delete-episode action (item 4).
|
||||
- `components/sessions/SessionsTab.tsx`, `components/library/MediaSessionList.tsx` — episode rollup (item 2).
|
||||
- `components/trends/DateRangeSelector.tsx`, `hooks/useTrends.ts`, `lib/api-client.ts`, `lib/api-client.test.ts` — 365d (item 3).
|
||||
- `components/vocabulary/FrequencyRankTable.tsx` — word/reading column collapse (item 5).
|
||||
- `components/anime/EpisodeDetail.tsx` — filter deleted Anki cards (item 6).
|
||||
- `components/trends/TrendChart.tsx`, `components/trends/StackedTrendChart.tsx`, `components/overview/WatchTimeChart.tsx`, `lib/chart-theme.ts` — chart clarity (item 7).
|
||||
- New file: `stats/src/lib/session-grouping.ts` + `session-grouping.test.ts`.
|
||||
|
||||
Backend (`src/core/services/`):
|
||||
- `immersion-tracker/query-trends.ts` — extend `TrendRange` and `TREND_DAY_LIMITS` (item 3).
|
||||
- `immersion-tracker/__tests__/query.test.ts` — 365d coverage (item 3).
|
||||
- `stats-server.ts` — passthrough if range validation lives here (check before editing).
|
||||
- `__tests__/stats-server.test.ts` — 365d coverage (item 3).
|
||||
|
||||
## Commit plan
|
||||
|
||||
One PR, one feature per commit. Order picks low-risk mechanical changes first so failures in later commits don't block merging of earlier ones.
|
||||
|
||||
1. `feat(stats): add 365d range to trends dashboard` (item 3)
|
||||
2. `fix(stats): tighten word/reading column in Top 50 table` (item 5)
|
||||
3. `fix(stats): hide cards deleted from Anki in episode detail` (item 6)
|
||||
4. `feat(stats): delete episode from library detail view` (item 4)
|
||||
5. `feat(stats): collapsible series groups in library` (item 1)
|
||||
6. `feat(stats): roll up same-episode sessions within a day` (item 2)
|
||||
7. `feat(stats): gridlines and unified theme for trend charts` (item 7)
|
||||
|
||||
Each commit must pass `bun run typecheck`, `bun run test:fast`, and any change-specific checks listed below.
|
||||
|
||||
---
|
||||
|
||||
## Item 1 — Library collapsible series groups
|
||||
|
||||
### Current behavior
|
||||
|
||||
`LibraryTab.tsx` groups media via `groupMediaLibraryItems` and always renders the full grid of `MediaCard`s beneath each group header.
|
||||
|
||||
### Target behavior
|
||||
|
||||
Each group header becomes clickable. Groups with `items.length > 1` default to **collapsed**; single-video groups stay expanded (collapsing them would be visual noise).
|
||||
|
||||
### Implementation
|
||||
|
||||
- State: `const [collapsedGroups, setCollapsedGroups] = useState<Set<string>>(...)`. Initialize from `grouped` where `items.length > 1`.
|
||||
- Toggle helper: `toggleGroup(key: string)` adds/removes from the set.
|
||||
- Group header: wrap in a `<button>` with `aria-expanded` and a chevron icon (`▶`/`▼`). Keep the existing cover + title + subtitle layout inside the button.
|
||||
- Children grid is conditionally rendered on `!collapsedGroups.has(group.key)`.
|
||||
- Header summary (`N videos · duration · cards`) stays visible in both states so collapsed groups remain informative.
|
||||
|
||||
### Tests
|
||||
|
||||
- New `LibraryTab.test.tsx` (if not already present — check first) covering:
|
||||
- Multi-video group renders collapsed on first mount.
|
||||
- Single-video group renders expanded on first mount.
|
||||
- Clicking the header toggles visibility.
|
||||
- Header summary is visible in both states.
|
||||
|
||||
---
|
||||
|
||||
## Item 2 — Sessions episode rollup within a day
|
||||
|
||||
### Current behavior
|
||||
|
||||
`SessionsTab.tsx:10-24` groups sessions by day label only (`formatSessionDayLabel(startedAtMs)`). Multiple sessions of the same episode on the same day show as independent rows. `MediaSessionList.tsx` has the same problem inside the library detail view.
|
||||
|
||||
### Target behavior
|
||||
|
||||
Within each day, sessions with the same `videoId` collapse into one parent row showing combined totals. A chevron reveals the individual sessions. Single-session buckets render flat (no pointless nesting).
|
||||
|
||||
### Implementation
|
||||
|
||||
- New helper in `stats/src/lib/session-grouping.ts`:
|
||||
```ts
|
||||
export interface SessionBucket {
|
||||
key: string; // videoId as string, or `s-${sessionId}` for singletons
|
||||
videoId: number | null;
|
||||
sessions: SessionSummary[];
|
||||
totalActiveMs: number;
|
||||
totalCardsMined: number;
|
||||
representativeSession: SessionSummary; // most recent, for header display
|
||||
}
|
||||
export function groupSessionsByVideo(sessions: SessionSummary[]): SessionBucket[];
|
||||
```
|
||||
Sessions missing a `videoId` become singleton buckets.
|
||||
|
||||
- `SessionsTab.tsx`: after day grouping, pipe each `daySessions` through `groupSessionsByVideo`. Render each bucket:
|
||||
- `sessions.length === 1`: existing `SessionRow` behavior, unchanged.
|
||||
- `sessions.length >= 2`: render a **bucket row** that looks like `SessionRow` but shows combined totals and session count (e.g. `3 sessions · 1h 24m · 12 cards`). Chevron state stored in a second `Set<string>` on bucket key. Expanded buckets render the child `SessionRow`s indented (`pl-8`) beneath the header.
|
||||
- `MediaSessionList.tsx`: within the media detail view, a single video's sessions are all the same `videoId` by definition — grouping here is by day only, and within a day multiple sessions render nested under a day header. Re-use the same visual pattern; factor the bucket row into a shared `SessionBucketRow` component.
|
||||
|
||||
### Delete semantics
|
||||
|
||||
- Deleting a bucket header offers "Delete all N sessions in this group" (reuse `confirmDayGroupDelete` pattern with a bucket-specific message, or add `confirmBucketDelete`).
|
||||
- Deleting an individual session from inside an expanded bucket keeps the existing single-delete flow.
|
||||
|
||||
### Tests
|
||||
|
||||
- `session-grouping.test.ts`:
|
||||
- Empty input → empty output.
|
||||
- All unique videos → N singleton buckets.
|
||||
- Two sessions same videoId → one bucket with correct totals and representative (most recent start time).
|
||||
- Missing videoId → singleton bucket keyed by sessionId.
|
||||
- `SessionsTab.test.tsx` (extend or add) verifying the rendered bucket rows expand/collapse and delete hooks fire with the right ID set.
|
||||
|
||||
---
|
||||
|
||||
## Item 3 — 365d trends range
|
||||
|
||||
### Backend
|
||||
|
||||
`src/core/services/immersion-tracker/query-trends.ts`:
|
||||
- `type TrendRange = '7d' | '30d' | '90d' | '365d' | 'all';`
|
||||
- Add `'365d': 365` to `TREND_DAY_LIMITS`.
|
||||
- `getTrendDayLimit` picks up the new key automatically because of the `Exclude<TrendRange, 'all'>` generic.
|
||||
|
||||
`src/core/services/stats-server.ts`:
|
||||
- Search for any hardcoded range validation (e.g. allow-list in the trends route handler) and extend it.
|
||||
|
||||
### Frontend
|
||||
|
||||
- `hooks/useTrends.ts`: widen the `TimeRange` union.
|
||||
- `components/trends/DateRangeSelector.tsx`: add `'365d'` to the options list. Display label stays as `365d`.
|
||||
- `lib/api-client.ts` / `api-client.test.ts`: if the client validates ranges, add `365d`.
|
||||
|
||||
### Tests
|
||||
|
||||
- `query.test.ts`: extend the existing range table to cover `365d` returning 365 days of data.
|
||||
- `stats-server.test.ts`: ensure the route accepts `range=365d`.
|
||||
- `api-client.test.ts`: ensure the client emits the new range.
|
||||
|
||||
### Change-specific checks
|
||||
|
||||
- `bun run test:config` is not required here (no schema/defaults change).
|
||||
- Run `bun run typecheck` + `bun run test:fast`.
|
||||
|
||||
---
|
||||
|
||||
## Item 4 — Delete episode from library detail
|
||||
|
||||
### Current behavior
|
||||
|
||||
`MediaDetailView.tsx` provides session-level delete only. The backend `deleteVideo` exists (`query-maintenance.ts:509`), the API is exposed at `stats-server.ts:559`, and `api-client.deleteVideo` is already wired (`stats/src/lib/api-client.ts:146`). `EpisodeList.tsx:46` already uses it from the anime tab.
|
||||
|
||||
### Target behavior
|
||||
|
||||
A "Delete Episode" action in `MediaHeader` (top-right, small, `text-ctp-red`), gated by `confirmEpisodeDelete(title)`. On success, call `onBack()` and make sure the parent `LibraryTab` refetches.
|
||||
|
||||
### Implementation
|
||||
|
||||
- Add an `onDeleteEpisode?: () => void` prop to `MediaHeader` and render the button only if provided.
|
||||
- In `MediaDetailView`:
|
||||
- New handler `handleDeleteEpisode` that calls `apiClient.deleteVideo(videoId)`, then `onBack()`.
|
||||
- Reuse `confirmEpisodeDelete` from `stats/src/lib/delete-confirm.ts`.
|
||||
- In `LibraryTab`:
|
||||
- `useMediaLibrary` returns fresh data on mount. The simplest fix: pass a `refresh` function from the hook (extend the hook if it doesn't already expose one) and call it when the detail view signals back.
|
||||
- Alternative: force a remount by incrementing a `libraryVersion` key on the library list. Prefer `refresh` for clarity.
|
||||
|
||||
### Tests
|
||||
|
||||
- Extend the existing `MediaDetailView.test.tsx`: mock `apiClient.deleteVideo`, click the new button, confirm `onBack` fires after success.
|
||||
- `useMediaLibrary.test.ts`: if we add a `refresh` method, cover it.
|
||||
|
||||
---
|
||||
|
||||
## Item 5 — Vocabulary word/reading column collapse
|
||||
|
||||
### Current behavior
|
||||
|
||||
`FrequencyRankTable.tsx:110-144` uses a 5-column table: `Rank | Word | Reading | POS | Seen`. Word and Reading are auto-sized, producing a large gap.
|
||||
|
||||
### Target behavior
|
||||
|
||||
Merge Word + Reading into a single column titled "Word". Reading sits immediately after the headword in a muted, smaller style.
|
||||
|
||||
### Implementation
|
||||
|
||||
- Drop the `<th>Reading</th>` header and cell.
|
||||
- Word cell becomes:
|
||||
```tsx
|
||||
<td className="py-1.5 pr-3">
|
||||
<span className="text-ctp-text font-medium">{w.headword}</span>
|
||||
{reading && (
|
||||
<span className="text-ctp-subtext0 text-xs ml-1.5">
|
||||
【{reading}】
|
||||
</span>
|
||||
)}
|
||||
</td>
|
||||
```
|
||||
where `reading = fullReading(w.headword, w.reading)` and differs from `headword`.
|
||||
- Keep `fullReading` import from `reading-utils`.
|
||||
|
||||
### Tests
|
||||
|
||||
- Extend `FrequencyRankTable.test.tsx` (if present — otherwise add a focused test) to assert:
|
||||
- Headword renders.
|
||||
- Reading renders when different from headword.
|
||||
- Reading does not render when equal to headword.
|
||||
|
||||
---
|
||||
|
||||
## Item 6 — Hide Anki-deleted cards in Cards Mined
|
||||
|
||||
### Current behavior
|
||||
|
||||
`EpisodeDetail.tsx:109-147` iterates `cardEvents`, fetches note info via `ankiNotesInfo(allNoteIds)`, and for each `noteId` renders a row even if no matching `info` came back — the user sees an empty word with an "Open in Anki" button that leads nowhere.
|
||||
|
||||
### Target behavior
|
||||
|
||||
After `ankiNotesInfo` resolves:
|
||||
- Drop `noteId`s that are not in the resolved map.
|
||||
- Drop `cardEvents` whose `noteIds` list was non-empty but is now empty after filtering.
|
||||
- Card events with a positive `cardsDelta` but no `noteIds` (legacy rollup path) still render as `+N cards` — we have no way to cross-reference them, so leave them alone.
|
||||
|
||||
### Implementation
|
||||
|
||||
- Compute `filteredCardEvents` as a `useMemo` depending on `data.cardEvents` and `noteInfos`.
|
||||
- Iterate `filteredCardEvents` instead of `cardEvents` in the render.
|
||||
- Surface a subtle note (optional, muted) "N cards hidden (deleted from Anki)" at the end of the list if any were filtered — helps the user understand why counts here diverge from session totals. Final decision on the note can be made at PR review; default: **show it**.
|
||||
|
||||
### Tests
|
||||
|
||||
- Add a test in `EpisodeDetail.test.tsx` (add the file if not present) that stubs `ankiNotesInfo` to return only a subset of notes and verifies the missing ones are not rendered.
|
||||
|
||||
### Other call sites
|
||||
|
||||
- Grep so far shows `ankiNotesInfo` is only used in `EpisodeDetail.tsx`. Re-verify before landing the commit; if another call site appears, apply the same filter.
|
||||
|
||||
---
|
||||
|
||||
## Item 7 — Trend/watch chart clarity pass
|
||||
|
||||
### Current behavior
|
||||
|
||||
`TrendChart.tsx`, `StackedTrendChart.tsx`, and `WatchTimeChart.tsx` render Recharts components with:
|
||||
- No `CartesianGrid` → no horizontal reference lines.
|
||||
- 9px axis ticks → borderline unreadable.
|
||||
- Height 120 → cramped.
|
||||
- Tooltip uses raw labels (`04/04` etc.).
|
||||
- No shared theme object; each chart redefines colors and tooltip styles inline.
|
||||
|
||||
`stats/src/lib/chart-theme.ts` already exists and currently exports a single `CHART_THEME` constant with tick/tooltip colors and `barFill`. It will be extended, not replaced, to preserve existing consumers.
|
||||
|
||||
### Target behavior
|
||||
|
||||
All three charts share a theme, have horizontal gridlines, readable ticks, and sensible tooltips.
|
||||
|
||||
### Implementation
|
||||
|
||||
Extend `stats/src/lib/chart-theme.ts` with the additional shared defaults (keeping the existing `CHART_THEME` export intact so current consumers don't break):
|
||||
```ts
|
||||
export const CHART_THEME = {
|
||||
tick: '#a5adcb',
|
||||
tooltipBg: '#363a4f',
|
||||
tooltipBorder: '#494d64',
|
||||
tooltipText: '#cad3f5',
|
||||
tooltipLabel: '#b8c0e0',
|
||||
barFill: '#8aadf4',
|
||||
grid: '#494d64',
|
||||
axisLine: '#494d64',
|
||||
} as const;
|
||||
|
||||
export const CHART_DEFAULTS = {
|
||||
height: 160,
|
||||
tickFontSize: 11,
|
||||
margin: { top: 8, right: 8, bottom: 0, left: 0 },
|
||||
grid: { strokeDasharray: '3 3', vertical: false },
|
||||
} as const;
|
||||
|
||||
export const TOOLTIP_CONTENT_STYLE = {
|
||||
background: CHART_THEME.tooltipBg,
|
||||
border: `1px solid ${CHART_THEME.tooltipBorder}`,
|
||||
borderRadius: 6,
|
||||
color: CHART_THEME.tooltipText,
|
||||
fontSize: 12,
|
||||
};
|
||||
```
|
||||
|
||||
Apply to each chart:
|
||||
- Import `CartesianGrid` from recharts.
|
||||
- Insert `<CartesianGrid stroke={CHART_THEME.grid} {...CHART_DEFAULTS.grid} />` inside each chart container.
|
||||
- `<XAxis tick={{ fontSize: CHART_DEFAULTS.tickFontSize, fill: CHART_THEME.tick }} />` and equivalent `YAxis`.
|
||||
- `YAxis` gains `axisLine={{ stroke: CHART_THEME.axisLine }}`.
|
||||
- `ResponsiveContainer` height changes from 120 → `CHART_DEFAULTS.height`.
|
||||
- `Tooltip` `contentStyle` uses `TOOLTIP_CONTENT_STYLE`, and charts pass a `labelFormatter` when the label is a date key (e.g. show `Fri Apr 4`).
|
||||
|
||||
### Unit formatters
|
||||
|
||||
- `TrendChart` already accepts a `formatter` prop — extend usage sites to pass unit-aware formatters where they aren't already (`formatDuration`, `formatNumber`, etc.).
|
||||
|
||||
### Tests
|
||||
|
||||
- `chart-theme.test.ts` (if present — otherwise add a trivial snapshot to keep the shape stable).
|
||||
- `TrendChart` snapshot/render tests: no regression, gridline element present.
|
||||
|
||||
---
|
||||
|
||||
## Verification gate
|
||||
|
||||
Before requesting code review, run:
|
||||
|
||||
```
|
||||
bun run typecheck
|
||||
bun run test:fast
|
||||
bun run test:env
|
||||
bun run test:runtime:compat # dist-sensitive check for the charts
|
||||
bun run build
|
||||
bun run test:smoke:dist
|
||||
```
|
||||
|
||||
No docs-site changes are planned in this spec; if `docs-site/` ends up touched (e.g. screenshots), also run `bun run docs:test` and `bun run docs:build`.
|
||||
|
||||
No config schema changes → `bun run test:config` and `bun run generate:config-example` are not required.
|
||||
|
||||
## Risks and open questions
|
||||
|
||||
- **MediaDetailView refresh**: `useMediaLibrary` may not expose a `refresh` function. If it doesn't, the simplest path is adding one; the alternative (keying a remount) works but is harder to test. Decide during implementation.
|
||||
- **Session bucket delete UX**: "Delete all N sessions in this group" is powerful. The copy must make it clear the underlying sessions are being removed, not just the grouping. Reuse `confirmBucketDelete` wording from existing confirm helpers if possible.
|
||||
- **Anki-deleted-cards hidden notice**: Showing a subtle "N cards hidden" footer is a call that can be made at PR review.
|
||||
- **Bucket delete helper**: `confirmBucketDelete` does not currently exist in `delete-confirm.ts`. Implementation either adds it or reuses `confirmDayGroupDelete` with bucket-specific wording — decide during the session-rollup commit.
|
||||
|
||||
## Changelog entry
|
||||
|
||||
User-visible PR → needs a fragment under `changes/*.md`. Suggested title:
|
||||
`Stats dashboard: collapsible series, session rollups, 365d trends, chart polish, episode delete.`
|
||||
@@ -3,7 +3,7 @@
|
||||
# Workflow
|
||||
|
||||
Status: active
|
||||
Last verified: 2026-05-23
|
||||
Last verified: 2026-08-13
|
||||
Owner: Kyle Yasuda
|
||||
Read when: planning or executing nontrivial work in this repo
|
||||
|
||||
@@ -13,7 +13,7 @@ This section is the internal workflow map for contributors and agents.
|
||||
|
||||
- [Planning](./planning.md) - when to write a lightweight plan vs a full execution plan
|
||||
- [Verification](./verification.md) - maintained test/build lanes and handoff gate
|
||||
- [Agent Plugins](./agent-plugins.md) - repo-local plugin ownership for agent workflow skills
|
||||
- [Agent Skills](./agent-skills.md) - repo-local workflow skill ownership
|
||||
- [Release Guide](../RELEASING.md) - tagged release workflow
|
||||
|
||||
## Default Flow
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
<!-- read_when: using or modifying repo-local agent plugins -->
|
||||
|
||||
# Agent Plugins
|
||||
|
||||
Status: active
|
||||
Last verified: 2026-05-23
|
||||
Owner: Kyle Yasuda
|
||||
Read when: packaging or migrating repo-local agent workflow skills into plugins
|
||||
|
||||
## SubMiner Workflow Plugin
|
||||
|
||||
- Canonical plugin path: `plugins/subminer-workflow/`
|
||||
- Marketplace catalog: `.agents/plugins/marketplace.json`
|
||||
- Canonical skill sources:
|
||||
- `plugins/subminer-workflow/skills/subminer-scrum-master/`
|
||||
- `plugins/subminer-workflow/skills/subminer-change-verification/`
|
||||
|
||||
## Migration Rule
|
||||
|
||||
- Plugin-owned skills are the source of truth.
|
||||
- `.agents/skills/subminer-*` remain only as compatibility shims.
|
||||
- Existing script entrypoints under `.agents/skills/subminer-change-verification/scripts/` stay as wrappers so historical commands do not break.
|
||||
|
||||
## Verification
|
||||
|
||||
- For plugin/docs-only changes, start with `bun run test:docs:kb`.
|
||||
- Use the plugin-owned verifier when the change crosses from docs into scripts or workflow logic.
|
||||
@@ -0,0 +1,31 @@
|
||||
<!-- read_when: using or modifying repo-local agent skills -->
|
||||
|
||||
# Agent Skills
|
||||
|
||||
Status: active
|
||||
Last verified: 2026-08-13
|
||||
Owner: Kyle Yasuda
|
||||
Read when: using, adding, or changing a repo-local agent workflow skill
|
||||
|
||||
## Canonical Skills
|
||||
|
||||
- `.agents/skills/subminer-change-verification/`
|
||||
- Selects the cheapest sufficient repo-native verification lane.
|
||||
- Defers command ownership to `package.json` and `docs/workflow/verification.md`.
|
||||
|
||||
Repo-local workflows stay as standalone skills. Do not add plugin packaging, marketplace metadata, or compatibility shims unless the workflow is intentionally being distributed beyond this repository.
|
||||
|
||||
## Rules
|
||||
|
||||
- Keep each skill focused on one repeatable repository task.
|
||||
- Prefer instructions over helper scripts unless deterministic tooling provides clear value beyond existing package commands.
|
||||
- Keep trigger descriptions narrow enough to avoid invoking skills for unrelated requests.
|
||||
- Update this page and the documentation catalog when skill ownership changes.
|
||||
|
||||
## Verification
|
||||
|
||||
For skill or internal workflow documentation changes, run:
|
||||
|
||||
```bash
|
||||
bun run test:docs:kb
|
||||
```
|
||||
@@ -3,14 +3,14 @@
|
||||
# Verification
|
||||
|
||||
Status: active
|
||||
Last verified: 2026-07-06
|
||||
Last verified: 2026-08-13
|
||||
Owner: Kyle Yasuda
|
||||
Read when: selecting the right verification lane for a change
|
||||
|
||||
## Lane Infrastructure
|
||||
|
||||
- Lane membership is defined once in `scripts/test-lanes.ts` and discovered by
|
||||
directory — new test files join their lane automatically; never hand-list test
|
||||
directory, so new test files join their lane automatically; never hand-list test
|
||||
files in `package.json`.
|
||||
- `scripts/run-test-lane.mjs` runs each test file in its own `bun test` process
|
||||
(per-file isolation with a wall timeout) so a hanging test or leaked global in
|
||||
@@ -43,8 +43,8 @@ bun run docs:build
|
||||
|
||||
## Cheap-First Lane Selection
|
||||
|
||||
- Docs-only boundary/content changes: `bun run docs:test`, `bun run docs:build`
|
||||
- Internal KB / `AGENTS.md` changes: `bun run test:docs:kb`
|
||||
- User-facing `docs-site/` changes: `bun run docs:test`, `bun run docs:build`
|
||||
- Internal KB, `AGENTS.md`, or `.agents/skills/**` changes: `bun run test:docs:kb`
|
||||
- Config/schema/defaults: `bun run test:config`, then `bun run generate:config-example` if template/defaults changed
|
||||
- Launcher/plugin: `bun run test:launcher` or `bun run test:env`
|
||||
- Runtime-compat / compiled behavior: `bun run test:runtime:compat`
|
||||
|
||||
@@ -157,6 +157,15 @@ export async function runStatsCommand(
|
||||
if (args.statsCleanupLifetime) {
|
||||
forwarded.push('--stats-cleanup-lifetime');
|
||||
}
|
||||
if (args.statsCleanupDuplicateLines) {
|
||||
forwarded.push('--stats-cleanup-duplicate-lines');
|
||||
}
|
||||
if (args.statsCleanupDryRun) {
|
||||
forwarded.push('--stats-cleanup-dry-run');
|
||||
}
|
||||
if (args.statsCleanupLookbackDays) {
|
||||
forwarded.push('--stats-cleanup-lookback-days', String(args.statsCleanupLookbackDays));
|
||||
}
|
||||
if (shouldForwardLogLevel(args.logLevel)) {
|
||||
forwarded.push('--log-level', args.logLevel);
|
||||
}
|
||||
|
||||
@@ -134,6 +134,9 @@ test('applyInvocationsToArgs maps config and jellyfin invocation state', () => {
|
||||
statsCleanup: false,
|
||||
statsCleanupVocab: false,
|
||||
statsCleanupLifetime: false,
|
||||
statsCleanupDuplicateLines: false,
|
||||
statsCleanupDryRun: false,
|
||||
statsCleanupLookbackDays: null,
|
||||
statsLogLevel: null,
|
||||
syncTriggered: false,
|
||||
syncCliTokens: [],
|
||||
@@ -185,6 +188,9 @@ test('applyInvocationsToArgs maps settings invocation to settings window', () =>
|
||||
statsCleanup: false,
|
||||
statsCleanupVocab: false,
|
||||
statsCleanupLifetime: false,
|
||||
statsCleanupDuplicateLines: false,
|
||||
statsCleanupDryRun: false,
|
||||
statsCleanupLookbackDays: null,
|
||||
statsLogLevel: null,
|
||||
syncTriggered: false,
|
||||
syncCliTokens: [],
|
||||
@@ -229,6 +235,9 @@ test('applyInvocationsToArgs fails when config invocation has no action', () =>
|
||||
statsCleanup: false,
|
||||
statsCleanupVocab: false,
|
||||
statsCleanupLifetime: false,
|
||||
statsCleanupDuplicateLines: false,
|
||||
statsCleanupDryRun: false,
|
||||
statsCleanupLookbackDays: null,
|
||||
statsLogLevel: null,
|
||||
syncTriggered: false,
|
||||
syncCliTokens: [],
|
||||
@@ -271,6 +280,9 @@ test('applyInvocationsToArgs maps texthooker browser-open request', () => {
|
||||
statsCleanup: false,
|
||||
statsCleanupVocab: false,
|
||||
statsCleanupLifetime: false,
|
||||
statsCleanupDuplicateLines: false,
|
||||
statsCleanupDryRun: false,
|
||||
statsCleanupLookbackDays: null,
|
||||
statsLogLevel: null,
|
||||
syncTriggered: false,
|
||||
syncCliTokens: [],
|
||||
|
||||
@@ -162,6 +162,8 @@ export function createDefaultArgs(
|
||||
statsCleanup: false,
|
||||
statsCleanupVocab: false,
|
||||
statsCleanupLifetime: false,
|
||||
statsCleanupDuplicateLines: false,
|
||||
statsCleanupDryRun: false,
|
||||
doctor: false,
|
||||
doctorRefreshKnownWords: false,
|
||||
logsExport: false,
|
||||
@@ -258,6 +260,11 @@ export function applyInvocationsToArgs(parsed: Args, invocations: CliInvocations
|
||||
if (invocations.statsCleanup) parsed.statsCleanup = true;
|
||||
if (invocations.statsCleanupVocab) parsed.statsCleanupVocab = true;
|
||||
if (invocations.statsCleanupLifetime) parsed.statsCleanupLifetime = true;
|
||||
if (invocations.statsCleanupDuplicateLines) parsed.statsCleanupDuplicateLines = true;
|
||||
if (invocations.statsCleanupDryRun) parsed.statsCleanupDryRun = true;
|
||||
if (invocations.statsCleanupLookbackDays !== null) {
|
||||
parsed.statsCleanupLookbackDays = invocations.statsCleanupLookbackDays;
|
||||
}
|
||||
if (invocations.dictionaryTarget) {
|
||||
parsed.dictionaryTarget = parseDictionaryTarget(invocations.dictionaryTarget);
|
||||
} else if (
|
||||
|
||||
@@ -37,6 +37,9 @@ export interface CliInvocations {
|
||||
statsCleanup: boolean;
|
||||
statsCleanupVocab: boolean;
|
||||
statsCleanupLifetime: boolean;
|
||||
statsCleanupDuplicateLines: boolean;
|
||||
statsCleanupDryRun: boolean;
|
||||
statsCleanupLookbackDays: number | null;
|
||||
statsLogLevel: string | null;
|
||||
syncTriggered: boolean;
|
||||
syncCliTokens: string[];
|
||||
@@ -53,6 +56,16 @@ export interface CliInvocations {
|
||||
texthookerOpenBrowser: boolean;
|
||||
}
|
||||
|
||||
/** `--lookback-days` narrows the duplicate-line cleanup; fractions are floored. */
|
||||
function parseStatsLookbackDays(value: unknown): number | null {
|
||||
if (typeof value !== 'string' && typeof value !== 'number') return null;
|
||||
const days = Number(value);
|
||||
if (!Number.isFinite(days) || days < 1) {
|
||||
throw new Error('Stats --lookback-days must be at least one day.');
|
||||
}
|
||||
return Math.floor(days);
|
||||
}
|
||||
|
||||
function applyRootOptions(program: Command): void {
|
||||
program
|
||||
.option(
|
||||
@@ -169,6 +182,9 @@ export function parseCliPrograms(
|
||||
let statsCleanup = false;
|
||||
let statsCleanupVocab = false;
|
||||
let statsCleanupLifetime = false;
|
||||
let statsCleanupDuplicateLines = false;
|
||||
let statsCleanupDryRun = false;
|
||||
let statsCleanupLookbackDays: number | null = null;
|
||||
let statsLogLevel: string | null = null;
|
||||
let syncTriggered = false;
|
||||
let syncCliTokens: string[] = [];
|
||||
@@ -269,6 +285,9 @@ export function parseCliPrograms(
|
||||
.option('-s, --stop', 'Stop the background stats server')
|
||||
.option('-v, --vocab', 'Clean vocabulary rows in the stats database')
|
||||
.option('-l, --lifetime', 'Rebuild lifetime summary rows from retained data')
|
||||
.option('-d, --duplicate-lines', 'Collapse repeated subtitle lines from typeset animations')
|
||||
.option('--dry-run', 'Report what a cleanup would remove without changing anything')
|
||||
.option('--lookback-days <days>', 'Only clean lines recorded in the last N days')
|
||||
.option('--log-level <level>', 'Log level')
|
||||
.action((action: string | undefined, options: Record<string, unknown>) => {
|
||||
statsTriggered = true;
|
||||
@@ -289,13 +308,35 @@ export function parseCliPrograms(
|
||||
if (normalizedAction && (statsBackground || statsStop)) {
|
||||
throw new Error('Stats background and stop flags cannot be combined with stats actions.');
|
||||
}
|
||||
if (normalizedAction !== 'cleanup' && (options.vocab === true || options.lifetime === true)) {
|
||||
throw new Error('Stats --vocab and --lifetime flags require the cleanup action.');
|
||||
if (
|
||||
normalizedAction !== 'cleanup' &&
|
||||
(options.vocab === true || options.lifetime === true || options.duplicateLines === true)
|
||||
) {
|
||||
throw new Error(
|
||||
'Stats --vocab, --lifetime and --duplicate-lines flags require the cleanup action.',
|
||||
);
|
||||
}
|
||||
if (
|
||||
options.duplicateLines !== true &&
|
||||
(options.dryRun === true || options.lookbackDays !== undefined)
|
||||
) {
|
||||
throw new Error('Stats --dry-run and --lookback-days require --duplicate-lines.');
|
||||
}
|
||||
if (normalizedAction === 'cleanup') {
|
||||
statsCleanup = true;
|
||||
statsCleanupLifetime = options.lifetime === true;
|
||||
statsCleanupVocab = statsCleanupLifetime ? false : options.vocab !== false;
|
||||
statsCleanupDuplicateLines = options.duplicateLines === true;
|
||||
const explicitModeCount = [options.vocab, options.lifetime, options.duplicateLines].filter(
|
||||
(value) => value === true,
|
||||
).length;
|
||||
if (explicitModeCount > 1) {
|
||||
throw new Error('Stats cleanup runs one mode at a time.');
|
||||
}
|
||||
// Vocabulary cleanup stays the default so `stats cleanup` keeps its old meaning.
|
||||
statsCleanupVocab =
|
||||
statsCleanupLifetime || statsCleanupDuplicateLines ? false : options.vocab !== false;
|
||||
statsCleanupDryRun = options.dryRun === true;
|
||||
statsCleanupLookbackDays = parseStatsLookbackDays(options.lookbackDays);
|
||||
} else if (normalizedAction === 'rebuild' || normalizedAction === 'backfill') {
|
||||
statsCleanup = true;
|
||||
statsCleanupLifetime = true;
|
||||
@@ -483,6 +524,9 @@ export function parseCliPrograms(
|
||||
statsCleanup,
|
||||
statsCleanupVocab,
|
||||
statsCleanupLifetime,
|
||||
statsCleanupDuplicateLines,
|
||||
statsCleanupDryRun,
|
||||
statsCleanupLookbackDays,
|
||||
statsLogLevel,
|
||||
syncTriggered,
|
||||
syncCliTokens,
|
||||
|
||||
@@ -222,7 +222,7 @@ test('buildMpvEnv preserves native Wayland env for supported Hyprland and Sway a
|
||||
});
|
||||
});
|
||||
|
||||
test('buildMpvBackendArgs forces an explicit X11 renderer stack when backend resolves to x11', () => {
|
||||
test('buildMpvBackendArgs pins the X11 window context when backend resolves to x11', () => {
|
||||
withPlatform('linux', () => {
|
||||
assert.deepEqual(
|
||||
buildMpvBackendArgs(makeArgs({ backend: 'x11' }), {
|
||||
@@ -230,12 +230,12 @@ test('buildMpvBackendArgs forces an explicit X11 renderer stack when backend res
|
||||
WAYLAND_DISPLAY: 'wayland-0',
|
||||
XDG_SESSION_TYPE: 'wayland',
|
||||
}),
|
||||
['--vo=gpu', '--gpu-api=opengl', '--gpu-context=x11egl,x11'],
|
||||
['--gpu-context=x11vk,x11egl,x11'],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('buildMpvBackendArgs forces the same X11 renderer stack for unsupported Wayland auto fallback', () => {
|
||||
test('buildMpvBackendArgs pins the same X11 window context for unsupported Wayland auto fallback', () => {
|
||||
withPlatform('linux', () => {
|
||||
assert.deepEqual(
|
||||
buildMpvBackendArgs(makeArgs({ backend: 'auto' }), {
|
||||
@@ -245,7 +245,7 @@ test('buildMpvBackendArgs forces the same X11 renderer stack for unsupported Way
|
||||
XDG_CURRENT_DESKTOP: 'KDE',
|
||||
XDG_SESSION_DESKTOP: 'plasma',
|
||||
}),
|
||||
['--vo=gpu', '--gpu-api=opengl', '--gpu-context=x11egl,x11'],
|
||||
['--gpu-context=x11vk,x11egl,x11'],
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -292,9 +292,7 @@ test('buildConfiguredMpvDefaultArgs appends maximized launch mode to configured
|
||||
'--secondary-sub-visibility=no',
|
||||
'--alang=ja,jp,jpn,japanese,en,eng,english,enus,en-us',
|
||||
'--slang=ja,jp,jpn,japanese,en,eng,english,enus,en-us',
|
||||
'--vo=gpu',
|
||||
'--gpu-api=opengl',
|
||||
'--gpu-context=x11egl,x11',
|
||||
'--gpu-context=x11vk,x11egl,x11',
|
||||
'--window-maximized=yes',
|
||||
],
|
||||
);
|
||||
|
||||
@@ -232,6 +232,75 @@ test('parseArgs maps lifetime stats cleanup flag', () => {
|
||||
assert.equal(parsed.statsCleanupLifetime, true);
|
||||
});
|
||||
|
||||
test('parseArgs maps duplicate-line stats cleanup flags', () => {
|
||||
const parsed = parseArgs(
|
||||
['stats', 'cleanup', '--duplicate-lines', '--dry-run', '--lookback-days', '30'],
|
||||
'subminer',
|
||||
{},
|
||||
);
|
||||
|
||||
assert.equal(parsed.statsCleanup, true);
|
||||
assert.equal(parsed.statsCleanupVocab, false);
|
||||
assert.equal(parsed.statsCleanupDuplicateLines, true);
|
||||
assert.equal(parsed.statsCleanupDryRun, true);
|
||||
assert.equal(parsed.statsCleanupLookbackDays, 30);
|
||||
|
||||
const fractional = parseArgs(
|
||||
['stats', 'cleanup', '--duplicate-lines', '--lookback-days', '1.5'],
|
||||
'subminer',
|
||||
{},
|
||||
);
|
||||
assert.equal(fractional.statsCleanupLookbackDays, 1);
|
||||
});
|
||||
|
||||
test('parseArgs rejects duplicate-line flags without the duplicate-lines mode', () => {
|
||||
const error = withProcessExitIntercept(() => {
|
||||
parseArgs(['stats', 'cleanup', '--dry-run'], 'subminer', {});
|
||||
});
|
||||
|
||||
assert.equal(error.code, 1);
|
||||
assert.match(error.stderr, /--dry-run and --lookback-days require --duplicate-lines/);
|
||||
});
|
||||
|
||||
test('parseArgs rejects an empty lookback value outside duplicate-line cleanup', () => {
|
||||
const error = withProcessExitIntercept(() => {
|
||||
parseArgs(['stats', '--lookback-days', ''], 'subminer', {});
|
||||
});
|
||||
|
||||
assert.equal(error.code, 1);
|
||||
assert.match(error.stderr, /--dry-run and --lookback-days require --duplicate-lines/);
|
||||
});
|
||||
|
||||
test('parseArgs rejects combining explicit cleanup modes', () => {
|
||||
for (const modes of [
|
||||
['--lifetime', '--duplicate-lines'],
|
||||
['--vocab', '--duplicate-lines'],
|
||||
['--vocab', '--lifetime'],
|
||||
]) {
|
||||
const error = withProcessExitIntercept(() => {
|
||||
parseArgs(['stats', 'cleanup', ...modes], 'subminer', {});
|
||||
});
|
||||
|
||||
assert.equal(error.code, 1);
|
||||
assert.match(error.stderr, /Stats cleanup runs one mode at a time/);
|
||||
}
|
||||
});
|
||||
|
||||
test('parseArgs rejects unusable lookback windows', () => {
|
||||
for (const value of ['0', '0.5', '-5', 'soon']) {
|
||||
const error = withProcessExitIntercept(() => {
|
||||
parseArgs(
|
||||
['stats', 'cleanup', '--duplicate-lines', '--lookback-days', value],
|
||||
'subminer',
|
||||
{},
|
||||
);
|
||||
});
|
||||
|
||||
assert.equal(error.code, 1);
|
||||
assert.match(error.stderr, /--lookback-days must be at least one day/);
|
||||
}
|
||||
});
|
||||
|
||||
test('parseArgs rejects cleanup-only stats flags without cleanup action', () => {
|
||||
const error = withProcessExitIntercept(() => {
|
||||
parseArgs(['stats', '--vocab'], 'subminer', {});
|
||||
@@ -239,7 +308,10 @@ test('parseArgs rejects cleanup-only stats flags without cleanup action', () =>
|
||||
|
||||
assert.equal(error.code, 1);
|
||||
assert.match(error.message, /exit:1/);
|
||||
assert.match(error.stderr, /Stats --vocab and --lifetime flags require the cleanup action/);
|
||||
assert.match(
|
||||
error.stderr,
|
||||
/Stats --vocab, --lifetime and --duplicate-lines flags require the cleanup action/,
|
||||
);
|
||||
});
|
||||
|
||||
test('parseArgs maps stats rebuild action to cleanup lifetime mode', () => {
|
||||
|
||||
@@ -142,6 +142,9 @@ export interface Args {
|
||||
statsCleanup?: boolean;
|
||||
statsCleanupVocab?: boolean;
|
||||
statsCleanupLifetime?: boolean;
|
||||
statsCleanupDuplicateLines?: boolean;
|
||||
statsCleanupDryRun?: boolean;
|
||||
statsCleanupLookbackDays?: number;
|
||||
dictionaryTarget?: string;
|
||||
doctor: boolean;
|
||||
doctorRefreshKnownWords: boolean;
|
||||
|
||||
+11
-5
@@ -2,7 +2,7 @@
|
||||
"name": "subminer",
|
||||
"productName": "SubMiner",
|
||||
"desktopName": "SubMiner.desktop",
|
||||
"version": "0.19.0",
|
||||
"version": "0.19.3",
|
||||
"description": "All-in-one sentence mining overlay with AnkiConnect and dictionary integration",
|
||||
"packageManager": "bun@1.3.5",
|
||||
"main": "dist/main-entry.js",
|
||||
@@ -84,16 +84,18 @@
|
||||
"overrides": {
|
||||
"@xmldom/xmldom": "0.8.13",
|
||||
"app-builder-lib": "26.15.3",
|
||||
"brace-expansion": "5.0.8",
|
||||
"brace-expansion": "5.0.9",
|
||||
"electron-builder-squirrel-windows": "26.15.3",
|
||||
"fast-uri": "3.1.5",
|
||||
"form-data": "4.0.6",
|
||||
"ip-address": "10.2.0",
|
||||
"js-yaml": "4.3.0",
|
||||
"js-yaml": "4.3.1",
|
||||
"lodash": "4.18.0",
|
||||
"minimatch": "10.2.5",
|
||||
"picomatch": "4.0.4",
|
||||
"tar": "7.5.21",
|
||||
"tmp": "0.2.7"
|
||||
"tmp": "0.2.7",
|
||||
"undici": "7.29.0"
|
||||
},
|
||||
"keywords": [
|
||||
"anki",
|
||||
@@ -125,7 +127,7 @@
|
||||
"@types/ws": "^8.18.1",
|
||||
"electron": "42.6.0",
|
||||
"electron-builder": "26.15.3",
|
||||
"undici": "7.28.0",
|
||||
"undici": "7.29.0",
|
||||
"esbuild": "^0.25.12",
|
||||
"eslint": "^10.8.0",
|
||||
"prettier": "^3.8.1",
|
||||
@@ -258,6 +260,10 @@
|
||||
{
|
||||
"from": "dist/launcher/subminer",
|
||||
"to": "launcher/subminer"
|
||||
},
|
||||
{
|
||||
"from": "CHANGELOG.md",
|
||||
"to": "CHANGELOG.md"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"name": "subminer-workflow",
|
||||
"version": "0.1.0",
|
||||
"description": "Repo-local SubMiner agent workflow plugin for orchestration and change verification.",
|
||||
"author": {
|
||||
"name": "Kyle Yasuda",
|
||||
"email": "suda@sudacode.com",
|
||||
"url": "https://github.com/sudacode"
|
||||
},
|
||||
"homepage": "https://github.com/sudacode/SubMiner/tree/main/plugins/subminer-workflow",
|
||||
"repository": "https://github.com/sudacode/SubMiner",
|
||||
"license": "GPL-3.0-or-later",
|
||||
"keywords": ["subminer", "workflow", "verification", "skills"],
|
||||
"skills": "./skills/",
|
||||
"interface": {
|
||||
"displayName": "SubMiner Workflow",
|
||||
"shortDescription": "SubMiner orchestration and verification.",
|
||||
"longDescription": "Canonical repo-local plugin for SubMiner agent workflow packaging. Owns the scrum-master and change-verification skills plus helper scripts used to plan, verify, and validate changes reproducibly inside this repo.",
|
||||
"developerName": "Kyle Yasuda",
|
||||
"category": "Productivity",
|
||||
"capabilities": ["Interactive", "Write"],
|
||||
"websiteURL": "https://github.com/sudacode/SubMiner",
|
||||
"defaultPrompt": [
|
||||
"Use SubMiner workflow to plan and ship a feature.",
|
||||
"Verify a SubMiner change with the plugin-owned verifier.",
|
||||
"Plan and ship this SubMiner task."
|
||||
],
|
||||
"brandColor": "#2F6B4F"
|
||||
}
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
<!-- read_when: migrating or using the repo-local SubMiner workflow plugin -->
|
||||
|
||||
# SubMiner Workflow Plugin
|
||||
|
||||
Status: active
|
||||
Last verified: 2026-03-26
|
||||
Owner: Kyle Yasuda
|
||||
Read when: using or updating the repo-local plugin that owns SubMiner agent workflow skills
|
||||
|
||||
This plugin is the canonical source of truth for the SubMiner agent workflow packaging.
|
||||
|
||||
## Contents
|
||||
|
||||
- `skills/subminer-scrum-master/`
|
||||
- intake, planning, dispatch, and handoff workflow
|
||||
- `skills/subminer-change-verification/`
|
||||
- cheap-first verification workflow plus helper scripts
|
||||
|
||||
## Compatibility
|
||||
|
||||
- `.agents/skills/subminer-scrum-master/` is a compatibility shim that redirects to the plugin-owned skill.
|
||||
- `.agents/skills/subminer-change-verification/` is a compatibility shim.
|
||||
- `.agents/skills/subminer-change-verification/scripts/*.sh` remain as wrapper entrypoints so existing docs and shell history keep working.
|
||||
|
||||
## Verification
|
||||
|
||||
For plugin/doc/shim changes, prefer:
|
||||
|
||||
```bash
|
||||
bun run test:docs:kb
|
||||
bash plugins/subminer-workflow/skills/subminer-change-verification/scripts/verify_subminer_change.sh --lane docs --lane core \
|
||||
plugins/subminer-workflow \
|
||||
.agents/skills/subminer-scrum-master/SKILL.md \
|
||||
.agents/skills/subminer-change-verification/SKILL.md \
|
||||
.agents/skills/subminer-change-verification/scripts/classify_subminer_diff.sh \
|
||||
.agents/skills/subminer-change-verification/scripts/verify_subminer_change.sh \
|
||||
.agents/plugins/marketplace.json \
|
||||
docs/workflow/README.md \
|
||||
docs/workflow/agent-plugins.md
|
||||
```
|
||||
@@ -1,143 +0,0 @@
|
||||
---
|
||||
name: 'subminer-change-verification'
|
||||
description: 'Use when working in the SubMiner repo and you need to verify code changes actually work. Covers targeted regression checks during debugging and pre-handoff verification, with cheap-first lane selection for config, docs, launcher/plugin, runtime-compat, and optional real-runtime escalation.'
|
||||
---
|
||||
|
||||
# SubMiner Change Verification
|
||||
|
||||
Canonical source: this plugin path.
|
||||
|
||||
Use this skill for SubMiner code changes. Default to cheap, repo-native verification first. Escalate only when the changed behavior actually depends on Electron, mpv, overlay/window tracking, or other GUI-sensitive runtime behavior.
|
||||
|
||||
## Scripts
|
||||
|
||||
- `scripts/classify_subminer_diff.sh`
|
||||
- Emits suggested lanes and flags from explicit paths or current git changes.
|
||||
- `scripts/verify_subminer_change.sh`
|
||||
- Runs selected lanes, captures artifacts, and writes a compact summary.
|
||||
|
||||
If you need an explicit installed path, use the directory that contains this `SKILL.md`. The helper scripts live under:
|
||||
|
||||
```bash
|
||||
export SUBMINER_VERIFY_SKILL="<path-to-plugin-skill>"
|
||||
```
|
||||
|
||||
## Default workflow
|
||||
|
||||
1. Inspect the changed files or user-requested area.
|
||||
2. Run the classifier unless you already know the right lane.
|
||||
3. Run the verifier with the cheapest sufficient lane set.
|
||||
4. If the classifier emits `flag:real-runtime-candidate`, do not jump straight to runtime verification. First run the non-runtime lanes.
|
||||
5. Escalate to explicit `--lane real-runtime --allow-real-runtime` only when cheaper lanes cannot validate the behavior claim.
|
||||
6. Return:
|
||||
- verification summary
|
||||
- exact commands run
|
||||
- artifact paths
|
||||
- skipped lanes and blockers
|
||||
|
||||
## Quick start
|
||||
|
||||
Plugin-source quick start:
|
||||
|
||||
```bash
|
||||
bash plugins/subminer-workflow/skills/subminer-change-verification/scripts/classify_subminer_diff.sh
|
||||
```
|
||||
|
||||
Installed-skill quick start:
|
||||
|
||||
```bash
|
||||
bash "$SUBMINER_VERIFY_SKILL/scripts/classify_subminer_diff.sh"
|
||||
```
|
||||
|
||||
Compatibility entrypoint:
|
||||
|
||||
```bash
|
||||
bash .agents/skills/subminer-change-verification/scripts/classify_subminer_diff.sh
|
||||
```
|
||||
|
||||
Classify explicit files:
|
||||
|
||||
```bash
|
||||
bash plugins/subminer-workflow/skills/subminer-change-verification/scripts/classify_subminer_diff.sh \
|
||||
launcher/main.ts \
|
||||
plugin/subminer/lifecycle.lua \
|
||||
src/main/runtime/mpv-client-runtime-service.ts
|
||||
```
|
||||
|
||||
Run automatic lane selection:
|
||||
|
||||
```bash
|
||||
bash plugins/subminer-workflow/skills/subminer-change-verification/scripts/verify_subminer_change.sh
|
||||
```
|
||||
|
||||
Installed-skill form:
|
||||
|
||||
```bash
|
||||
bash "$SUBMINER_VERIFY_SKILL/scripts/verify_subminer_change.sh"
|
||||
```
|
||||
|
||||
Compatibility entrypoint:
|
||||
|
||||
```bash
|
||||
bash .agents/skills/subminer-change-verification/scripts/verify_subminer_change.sh
|
||||
```
|
||||
|
||||
Run targeted lanes:
|
||||
|
||||
```bash
|
||||
bash plugins/subminer-workflow/skills/subminer-change-verification/scripts/verify_subminer_change.sh \
|
||||
--lane launcher-plugin \
|
||||
--lane runtime-compat
|
||||
```
|
||||
|
||||
Dry-run to inspect planned commands and artifact layout:
|
||||
|
||||
```bash
|
||||
bash plugins/subminer-workflow/skills/subminer-change-verification/scripts/verify_subminer_change.sh \
|
||||
--dry-run \
|
||||
launcher/main.ts \
|
||||
src/main.ts
|
||||
```
|
||||
|
||||
## Lane guidance
|
||||
|
||||
- `docs`
|
||||
- For `docs-site/`, `docs/`, and doc-only edits.
|
||||
- `config`
|
||||
- For `src/config/` and config-template-sensitive edits.
|
||||
- `stats`
|
||||
- For `stats/` dashboard UI edits.
|
||||
- `core`
|
||||
- For general source changes where `typecheck` + `test:fast` is the best cheap signal.
|
||||
- `launcher-plugin`
|
||||
- For `launcher/`, `plugin/subminer/`, plugin gating scripts, and wrapper/mpv routing work.
|
||||
- `runtime-compat`
|
||||
- For `src/main*`, runtime/composer wiring, mpv/overlay services, window trackers, and dist-sensitive behavior.
|
||||
- `real-runtime`
|
||||
- Only after deliberate escalation.
|
||||
|
||||
## Real Runtime Escalation
|
||||
|
||||
Escalate only when the change claim depends on actual runtime behavior, for example:
|
||||
|
||||
- overlay appears, hides, or tracks a real mpv window
|
||||
- mpv launch flags or pause-until-ready behavior
|
||||
- plugin/socket/auto-start handshake under a real player
|
||||
- macOS/window-tracker/focus-sensitive behavior
|
||||
|
||||
If the environment cannot support authoritative runtime verification, report the blocker explicitly. Do not silently downgrade a runtime-required claim to a pass.
|
||||
|
||||
## Artifact contract
|
||||
|
||||
The verifier writes under `.tmp/skill-verification/<timestamp>/`:
|
||||
|
||||
- `summary.json`
|
||||
- `summary.txt`
|
||||
- `classification.txt`
|
||||
- `env.txt`
|
||||
- `lanes.txt`
|
||||
- `steps.tsv`
|
||||
- `steps/*.stdout.log`
|
||||
- `steps/*.stderr.log`
|
||||
|
||||
On failure, quote the exact failing command and point at the artifact directory.
|
||||
-171
@@ -1,171 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: classify_subminer_diff.sh [path ...]
|
||||
|
||||
Emit suggested verification lanes for explicit paths or current local git changes.
|
||||
|
||||
Output format:
|
||||
lane:<name>
|
||||
flag:<name>
|
||||
reason:<text>
|
||||
EOF
|
||||
}
|
||||
|
||||
has_item() {
|
||||
local needle=$1
|
||||
shift || true
|
||||
local item
|
||||
for item in "$@"; do
|
||||
if [[ "$item" == "$needle" ]]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
add_lane() {
|
||||
local lane=$1
|
||||
if ! has_item "$lane" "${LANES[@]:-}"; then
|
||||
LANES+=("$lane")
|
||||
fi
|
||||
}
|
||||
|
||||
add_flag() {
|
||||
local flag=$1
|
||||
if ! has_item "$flag" "${FLAGS[@]:-}"; then
|
||||
FLAGS+=("$flag")
|
||||
fi
|
||||
}
|
||||
|
||||
add_reason() {
|
||||
REASONS+=("$1")
|
||||
}
|
||||
|
||||
collect_git_paths() {
|
||||
local top_level
|
||||
if ! top_level=$(git rev-parse --show-toplevel 2>/dev/null); then
|
||||
return 0
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$top_level"
|
||||
if git rev-parse --verify HEAD >/dev/null 2>&1; then
|
||||
git diff --name-only --relative HEAD --
|
||||
git diff --name-only --relative --cached --
|
||||
else
|
||||
git diff --name-only --relative --
|
||||
git diff --name-only --relative --cached --
|
||||
fi
|
||||
git ls-files --others --exclude-standard
|
||||
) | awk 'NF' | sort -u
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
declare -a PATHS=()
|
||||
declare -a LANES=()
|
||||
declare -a FLAGS=()
|
||||
declare -a REASONS=()
|
||||
|
||||
if [[ $# -gt 0 ]]; then
|
||||
while [[ $# -gt 0 ]]; do
|
||||
PATHS+=("$1")
|
||||
shift
|
||||
done
|
||||
else
|
||||
while IFS= read -r line; do
|
||||
[[ -n "$line" ]] && PATHS+=("$line")
|
||||
done < <(collect_git_paths)
|
||||
fi
|
||||
|
||||
if [[ ${#PATHS[@]} -eq 0 ]]; then
|
||||
add_lane "core"
|
||||
add_reason "no changed paths detected -> default to core"
|
||||
fi
|
||||
|
||||
for path in "${PATHS[@]}"; do
|
||||
specialized=0
|
||||
|
||||
case "$path" in
|
||||
docs-site/*|docs/*|changes/*|README.md)
|
||||
add_lane "docs"
|
||||
add_reason "$path -> docs"
|
||||
specialized=1
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$path" in
|
||||
src/config/*|src/generate-config-example.ts|src/verify-config-example.ts|docs-site/public/config.example.jsonc|config.example.jsonc)
|
||||
add_lane "config"
|
||||
add_reason "$path -> config"
|
||||
specialized=1
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$path" in
|
||||
stats/*)
|
||||
add_lane "stats"
|
||||
add_reason "$path -> stats"
|
||||
specialized=1
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$path" in
|
||||
launcher/*|plugin/subminer/*|plugin/subminer.conf|scripts/test-plugin-*|scripts/get-mpv-window-*|scripts/configure-plugin-binary-path.mjs)
|
||||
add_lane "launcher-plugin"
|
||||
add_reason "$path -> launcher-plugin"
|
||||
add_flag "real-runtime-candidate"
|
||||
add_reason "$path -> real-runtime-candidate"
|
||||
specialized=1
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$path" in
|
||||
src/main.ts|src/main-entry.ts|src/preload.ts|src/main/*|src/core/services/mpv*|src/core/services/overlay*|src/renderer/*|src/window-trackers/*|scripts/prepare-build-assets.mjs)
|
||||
add_lane "runtime-compat"
|
||||
add_reason "$path -> runtime-compat"
|
||||
add_flag "real-runtime-candidate"
|
||||
add_reason "$path -> real-runtime-candidate"
|
||||
specialized=1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$specialized" == "0" ]]; then
|
||||
case "$path" in
|
||||
src/*|package.json|tsconfig*.json|scripts/*|Makefile)
|
||||
add_lane "core"
|
||||
add_reason "$path -> core"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
case "$path" in
|
||||
package.json|src/main.ts|src/main-entry.ts|src/preload.ts)
|
||||
add_flag "broad-impact"
|
||||
add_reason "$path -> broad-impact"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ ${#LANES[@]} -eq 0 ]]; then
|
||||
add_lane "core"
|
||||
add_reason "no lane-specific matches -> default to core"
|
||||
fi
|
||||
|
||||
for lane in "${LANES[@]}"; do
|
||||
printf 'lane:%s\n' "$lane"
|
||||
done
|
||||
|
||||
for flag in "${FLAGS[@]}"; do
|
||||
printf 'flag:%s\n' "$flag"
|
||||
done
|
||||
|
||||
for reason in "${REASONS[@]}"; do
|
||||
printf 'reason:%s\n' "$reason"
|
||||
done
|
||||
-537
@@ -1,537 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: verify_subminer_change.sh [options] [path ...]
|
||||
|
||||
Options:
|
||||
--lane <name> Force a verification lane. Repeatable.
|
||||
--artifact-dir <dir> Use an explicit artifact directory.
|
||||
--allow-real-runtime Allow explicit real-runtime execution.
|
||||
--allow-real-gui Deprecated alias for --allow-real-runtime.
|
||||
--dry-run Record planned steps without executing commands.
|
||||
--help Show this help text.
|
||||
|
||||
If no lanes are supplied, the script classifies the provided paths. If no paths are
|
||||
provided, it classifies the current local git changes.
|
||||
|
||||
Authoritative real-runtime verification should be requested with explicit path
|
||||
arguments instead of relying on inferred local git changes.
|
||||
EOF
|
||||
}
|
||||
|
||||
timestamp() {
|
||||
date +%Y%m%d-%H%M%S
|
||||
}
|
||||
|
||||
timestamp_iso() {
|
||||
date -u +%Y-%m-%dT%H:%M:%SZ
|
||||
}
|
||||
|
||||
generate_session_id() {
|
||||
local tmp_dir
|
||||
tmp_dir=$(mktemp -d "${TMPDIR:-/tmp}/subminer-verify-$(timestamp)-XXXXXX")
|
||||
basename "$tmp_dir"
|
||||
rmdir "$tmp_dir"
|
||||
}
|
||||
|
||||
has_item() {
|
||||
local needle=$1
|
||||
shift || true
|
||||
local item
|
||||
for item in "$@"; do
|
||||
if [[ "$item" == "$needle" ]]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
normalize_lane_name() {
|
||||
case "$1" in
|
||||
real-gui)
|
||||
printf '%s' "real-runtime"
|
||||
;;
|
||||
*)
|
||||
printf '%s' "$1"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
add_lane() {
|
||||
local lane
|
||||
lane=$(normalize_lane_name "$1")
|
||||
if ! has_item "$lane" "${SELECTED_LANES[@]:-}"; then
|
||||
SELECTED_LANES+=("$lane")
|
||||
fi
|
||||
}
|
||||
|
||||
add_blocker() {
|
||||
BLOCKERS+=("$1")
|
||||
BLOCKED=1
|
||||
}
|
||||
|
||||
validate_artifact_dir() {
|
||||
local candidate=$1
|
||||
if [[ ! "$candidate" =~ ^[A-Za-z0-9._/@:+-]+$ ]]; then
|
||||
echo "Invalid characters in --artifact-dir path" >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
append_step_record() {
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" >>"$STEPS_TSV"
|
||||
}
|
||||
|
||||
record_env() {
|
||||
{
|
||||
printf 'repo_root=%s\n' "$REPO_ROOT"
|
||||
printf 'session_id=%s\n' "$SESSION_ID"
|
||||
printf 'artifact_dir=%s\n' "$ARTIFACT_DIR"
|
||||
printf 'path_selection_mode=%s\n' "$PATH_SELECTION_MODE"
|
||||
printf 'dry_run=%s\n' "$DRY_RUN"
|
||||
printf 'allow_real_runtime=%s\n' "$ALLOW_REAL_RUNTIME"
|
||||
printf 'session_home=%s\n' "$SESSION_HOME"
|
||||
printf 'session_xdg_config_home=%s\n' "$SESSION_XDG_CONFIG_HOME"
|
||||
printf 'session_mpv_dir=%s\n' "$SESSION_MPV_DIR"
|
||||
printf 'session_logs_dir=%s\n' "$SESSION_LOGS_DIR"
|
||||
printf 'session_mpv_log=%s\n' "$SESSION_MPV_LOG"
|
||||
printf 'pwd=%s\n' "$(pwd)"
|
||||
git rev-parse --short HEAD 2>/dev/null | sed 's/^/git_head=/' || true
|
||||
git status --short 2>/dev/null || true
|
||||
if [[ ${#PATH_ARGS[@]} -gt 0 ]]; then
|
||||
printf 'requested_paths=\n'
|
||||
printf ' %s\n' "${PATH_ARGS[@]}"
|
||||
fi
|
||||
} >"$ARTIFACT_DIR/env.txt"
|
||||
}
|
||||
|
||||
run_step() {
|
||||
local lane=$1
|
||||
local name=$2
|
||||
local command=$3
|
||||
local note=${4:-}
|
||||
local lane_slug=${lane//[^a-zA-Z0-9_-]/-}
|
||||
local slug=${name//[^a-zA-Z0-9_-]/-}
|
||||
local step_slug="${lane_slug}--${slug}"
|
||||
local stdout_rel="steps/${step_slug}.stdout.log"
|
||||
local stderr_rel="steps/${step_slug}.stderr.log"
|
||||
local stdout_path="$ARTIFACT_DIR/$stdout_rel"
|
||||
local stderr_path="$ARTIFACT_DIR/$stderr_rel"
|
||||
local status exit_code
|
||||
|
||||
COMMANDS_RUN+=("$command")
|
||||
printf '%s\n' "$command" >"$ARTIFACT_DIR/steps/${step_slug}.command.txt"
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
printf '[dry-run] %s\n' "$command" >"$stdout_path"
|
||||
: >"$stderr_path"
|
||||
status="dry-run"
|
||||
exit_code=0
|
||||
else
|
||||
if HOME="$SESSION_HOME" \
|
||||
XDG_CONFIG_HOME="$SESSION_XDG_CONFIG_HOME" \
|
||||
SUBMINER_SESSION_LOGS_DIR="$SESSION_LOGS_DIR" \
|
||||
SUBMINER_SESSION_MPV_LOG="$SESSION_MPV_LOG" \
|
||||
bash -c "cd \"$REPO_ROOT\" && $command" >"$stdout_path" 2>"$stderr_path"; then
|
||||
status="passed"
|
||||
exit_code=0
|
||||
EXECUTED_REAL_STEPS=1
|
||||
else
|
||||
exit_code=$?
|
||||
status="failed"
|
||||
FAILED=1
|
||||
fi
|
||||
fi
|
||||
|
||||
append_step_record "$lane" "$name" "$status" "$exit_code" "$command" "$stdout_rel" "$stderr_rel" "$note"
|
||||
printf '%s\t%s\t%s\n' "$lane" "$name" "$status"
|
||||
|
||||
if [[ "$status" == "failed" ]]; then
|
||||
FAILURE_STEP="$name"
|
||||
FAILURE_COMMAND="$command"
|
||||
FAILURE_STDOUT="$stdout_rel"
|
||||
FAILURE_STDERR="$stderr_rel"
|
||||
return "$exit_code"
|
||||
fi
|
||||
}
|
||||
|
||||
record_nonpassing_step() {
|
||||
local lane=$1
|
||||
local name=$2
|
||||
local status=$3
|
||||
local note=$4
|
||||
local lane_slug=${lane//[^a-zA-Z0-9_-]/-}
|
||||
local slug=${name//[^a-zA-Z0-9_-]/-}
|
||||
local step_slug="${lane_slug}--${slug}"
|
||||
local stdout_rel="steps/${step_slug}.stdout.log"
|
||||
local stderr_rel="steps/${step_slug}.stderr.log"
|
||||
printf '%s\n' "$note" >"$ARTIFACT_DIR/$stdout_rel"
|
||||
: >"$ARTIFACT_DIR/$stderr_rel"
|
||||
append_step_record "$lane" "$name" "$status" "0" "" "$stdout_rel" "$stderr_rel" "$note"
|
||||
printf '%s\t%s\t%s\n' "$lane" "$name" "$status"
|
||||
}
|
||||
|
||||
record_skipped_step() {
|
||||
record_nonpassing_step "$1" "$2" "skipped" "$3"
|
||||
}
|
||||
|
||||
record_blocked_step() {
|
||||
add_blocker "$3"
|
||||
record_nonpassing_step "$1" "$2" "blocked" "$3"
|
||||
}
|
||||
|
||||
record_failed_step() {
|
||||
FAILED=1
|
||||
FAILURE_STEP=$2
|
||||
FAILURE_COMMAND=${FAILURE_COMMAND:-"(validation)"}
|
||||
local lane_slug=${1//[^a-zA-Z0-9_-]/-}
|
||||
local step_slug=${2//[^a-zA-Z0-9_-]/-}
|
||||
FAILURE_STDOUT="steps/${lane_slug}--${step_slug}.stdout.log"
|
||||
FAILURE_STDERR="steps/${lane_slug}--${step_slug}.stderr.log"
|
||||
add_blocker "$3"
|
||||
record_nonpassing_step "$1" "$2" "failed" "$3"
|
||||
}
|
||||
|
||||
find_real_runtime_helper() {
|
||||
local candidate
|
||||
for candidate in \
|
||||
"$SCRIPT_DIR/run_real_runtime_smoke.sh" \
|
||||
"$SCRIPT_DIR/run_real_mpv_smoke.sh"; do
|
||||
if [[ -x "$candidate" ]]; then
|
||||
printf '%s' "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
acquire_real_runtime_lease() {
|
||||
local lease_root="$REPO_ROOT/.tmp/skill-verification/locks"
|
||||
local lease_dir="$lease_root/exclusive-real-runtime"
|
||||
mkdir -p "$lease_root"
|
||||
if mkdir "$lease_dir" 2>/dev/null; then
|
||||
REAL_RUNTIME_LEASE_DIR="$lease_dir"
|
||||
printf '%s\n' "$SESSION_ID" >"$lease_dir/session_id"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local owner=""
|
||||
if [[ -f "$lease_dir/session_id" ]]; then
|
||||
owner=$(cat "$lease_dir/session_id")
|
||||
fi
|
||||
REAL_RUNTIME_LEASE_ERROR="real-runtime lease already held${owner:+ by $owner}"
|
||||
return 1
|
||||
}
|
||||
|
||||
release_real_runtime_lease() {
|
||||
if [[ -n "$REAL_RUNTIME_LEASE_DIR" && -d "$REAL_RUNTIME_LEASE_DIR" ]]; then
|
||||
if [[ -f "$REAL_RUNTIME_LEASE_DIR/session_id" ]]; then
|
||||
local owner
|
||||
owner=$(cat "$REAL_RUNTIME_LEASE_DIR/session_id")
|
||||
if [[ "$owner" != "$SESSION_ID" ]]; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
rm -rf "$REAL_RUNTIME_LEASE_DIR"
|
||||
fi
|
||||
}
|
||||
|
||||
compute_final_status() {
|
||||
if [[ "$FAILED" == "1" ]]; then
|
||||
FINAL_STATUS="failed"
|
||||
elif [[ "$BLOCKED" == "1" ]]; then
|
||||
FINAL_STATUS="blocked"
|
||||
elif [[ "$EXECUTED_REAL_STEPS" == "1" ]]; then
|
||||
FINAL_STATUS="passed"
|
||||
else
|
||||
FINAL_STATUS="skipped"
|
||||
fi
|
||||
}
|
||||
|
||||
write_summary_files() {
|
||||
local lane_lines
|
||||
lane_lines=$(printf '%s\n' "${SELECTED_LANES[@]}")
|
||||
printf '%s\n' "$lane_lines" >"$ARTIFACT_DIR/lanes.txt"
|
||||
# bash 3.2 raises "unbound variable" under set -u when expanding an empty
|
||||
# array, so guard on length (matching the idiom used elsewhere here).
|
||||
if [[ ${#BLOCKERS[@]} -gt 0 ]]; then
|
||||
printf '%s\n' "${BLOCKERS[@]}" >"$ARTIFACT_DIR/blockers.txt"
|
||||
else
|
||||
: >"$ARTIFACT_DIR/blockers.txt"
|
||||
fi
|
||||
if [[ ${#PATH_ARGS[@]} -gt 0 ]]; then
|
||||
printf '%s\n' "${PATH_ARGS[@]}" >"$ARTIFACT_DIR/requested-paths.txt"
|
||||
else
|
||||
: >"$ARTIFACT_DIR/requested-paths.txt"
|
||||
fi
|
||||
|
||||
ARTIFACT_DIR_ENV="$ARTIFACT_DIR" \
|
||||
SESSION_ID_ENV="$SESSION_ID" \
|
||||
FINAL_STATUS_ENV="$FINAL_STATUS" \
|
||||
PATH_SELECTION_MODE_ENV="$PATH_SELECTION_MODE" \
|
||||
ALLOW_REAL_RUNTIME_ENV="$ALLOW_REAL_RUNTIME" \
|
||||
SESSION_HOME_ENV="$SESSION_HOME" \
|
||||
SESSION_XDG_CONFIG_HOME_ENV="$SESSION_XDG_CONFIG_HOME" \
|
||||
SESSION_MPV_DIR_ENV="$SESSION_MPV_DIR" \
|
||||
SESSION_LOGS_DIR_ENV="$SESSION_LOGS_DIR" \
|
||||
SESSION_MPV_LOG_ENV="$SESSION_MPV_LOG" \
|
||||
STARTED_AT_ENV="$STARTED_AT" \
|
||||
FINISHED_AT_ENV="$FINISHED_AT" \
|
||||
FAILED_ENV="$FAILED" \
|
||||
FAILURE_COMMAND_ENV="${FAILURE_COMMAND:-}" \
|
||||
FAILURE_STDOUT_ENV="${FAILURE_STDOUT:-}" \
|
||||
FAILURE_STDERR_ENV="${FAILURE_STDERR:-}" \
|
||||
bun -e '
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const lines = fs
|
||||
.readFileSync(path.join(process.env.ARTIFACT_DIR_ENV, "steps.tsv"), "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.slice(1)
|
||||
.map((line) => {
|
||||
const [lane, name, status, exitCode, command, stdout, stderr, note] = line.split("\t");
|
||||
return { lane, name, status, exitCode: Number(exitCode), command, stdout, stderr, note };
|
||||
});
|
||||
|
||||
const payload = {
|
||||
sessionId: process.env.SESSION_ID_ENV,
|
||||
startedAt: process.env.STARTED_AT_ENV,
|
||||
finishedAt: process.env.FINISHED_AT_ENV,
|
||||
status: process.env.FINAL_STATUS_ENV,
|
||||
pathSelectionMode: process.env.PATH_SELECTION_MODE_ENV,
|
||||
allowRealRuntime: process.env.ALLOW_REAL_RUNTIME_ENV === "1",
|
||||
sessionHome: process.env.SESSION_HOME_ENV,
|
||||
sessionXdgConfigHome: process.env.SESSION_XDG_CONFIG_HOME_ENV,
|
||||
sessionMpvDir: process.env.SESSION_MPV_DIR_ENV,
|
||||
sessionLogsDir: process.env.SESSION_LOGS_DIR_ENV,
|
||||
sessionMpvLog: process.env.SESSION_MPV_LOG_ENV,
|
||||
failed: process.env.FAILED_ENV === "1",
|
||||
failure: process.env.FAILURE_COMMAND_ENV
|
||||
? {
|
||||
command: process.env.FAILURE_COMMAND_ENV,
|
||||
stdout: process.env.FAILURE_STDOUT_ENV,
|
||||
stderr: process.env.FAILURE_STDERR_ENV,
|
||||
}
|
||||
: null,
|
||||
blockers: fs
|
||||
.readFileSync(path.join(process.env.ARTIFACT_DIR_ENV, "blockers.txt"), "utf8")
|
||||
.split("\n")
|
||||
.filter(Boolean),
|
||||
lanes: fs
|
||||
.readFileSync(path.join(process.env.ARTIFACT_DIR_ENV, "lanes.txt"), "utf8")
|
||||
.split("\n")
|
||||
.filter(Boolean),
|
||||
requestedPaths: fs
|
||||
.readFileSync(path.join(process.env.ARTIFACT_DIR_ENV, "requested-paths.txt"), "utf8")
|
||||
.split("\n")
|
||||
.filter(Boolean),
|
||||
steps: lines,
|
||||
};
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(process.env.ARTIFACT_DIR_ENV, "summary.json"),
|
||||
JSON.stringify(payload, null, 2) + "\n",
|
||||
);
|
||||
|
||||
const summaryLines = [
|
||||
`status: ${payload.status}`,
|
||||
`session: ${payload.sessionId}`,
|
||||
`artifacts: ${process.env.ARTIFACT_DIR_ENV}`,
|
||||
`lanes: ${payload.lanes.join(", ") || "(none)"}`,
|
||||
];
|
||||
|
||||
if (payload.requestedPaths.length > 0) {
|
||||
summaryLines.push("requested paths:");
|
||||
for (const entry of payload.requestedPaths) {
|
||||
summaryLines.push(`- ${entry}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (payload.failure) {
|
||||
summaryLines.push(`failure command: ${payload.failure.command}`);
|
||||
summaryLines.push(`failure stdout: ${payload.failure.stdout}`);
|
||||
summaryLines.push(`failure stderr: ${payload.failure.stderr}`);
|
||||
}
|
||||
|
||||
if (payload.blockers.length > 0) {
|
||||
summaryLines.push("blockers:");
|
||||
for (const blocker of payload.blockers) {
|
||||
summaryLines.push(`- ${blocker}`);
|
||||
}
|
||||
}
|
||||
|
||||
summaryLines.push("steps:");
|
||||
for (const step of payload.steps) {
|
||||
summaryLines.push(`- ${step.lane}/${step.name}: ${step.status}`);
|
||||
}
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(process.env.ARTIFACT_DIR_ENV, "summary.txt"),
|
||||
summaryLines.join("\n") + "\n",
|
||||
);
|
||||
'
|
||||
}
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
SKILL_DIR=$(cd "$SCRIPT_DIR/.." && pwd)
|
||||
REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
|
||||
|
||||
declare -a PATH_ARGS=()
|
||||
declare -a SELECTED_LANES=()
|
||||
declare -a COMMANDS_RUN=()
|
||||
declare -a BLOCKERS=()
|
||||
|
||||
ALLOW_REAL_RUNTIME=0
|
||||
DRY_RUN=0
|
||||
FAILED=0
|
||||
BLOCKED=0
|
||||
EXECUTED_REAL_STEPS=0
|
||||
FAILURE_STEP=""
|
||||
FAILURE_COMMAND=""
|
||||
FAILURE_STDOUT=""
|
||||
FAILURE_STDERR=""
|
||||
REAL_RUNTIME_LEASE_DIR=""
|
||||
REAL_RUNTIME_LEASE_ERROR=""
|
||||
PATH_SELECTION_MODE="auto"
|
||||
|
||||
trap 'release_real_runtime_lease' EXIT
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--lane)
|
||||
shift
|
||||
[[ $# -gt 0 ]] || {
|
||||
echo "Missing value for --lane" >&2
|
||||
exit 2
|
||||
}
|
||||
add_lane "$1"
|
||||
PATH_SELECTION_MODE="explicit-lanes"
|
||||
;;
|
||||
--artifact-dir)
|
||||
shift
|
||||
[[ $# -gt 0 ]] || {
|
||||
echo "Missing value for --artifact-dir" >&2
|
||||
exit 2
|
||||
}
|
||||
ARTIFACT_DIR=$1
|
||||
;;
|
||||
--allow-real-runtime|--allow-real-gui)
|
||||
ALLOW_REAL_RUNTIME=1
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=1
|
||||
;;
|
||||
--help|-h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
PATH_ARGS+=("$1")
|
||||
;;
|
||||
esac
|
||||
shift || true
|
||||
done
|
||||
|
||||
if [[ -z "${ARTIFACT_DIR:-}" ]]; then
|
||||
SESSION_ID=$(generate_session_id)
|
||||
ARTIFACT_DIR="$REPO_ROOT/.tmp/skill-verification/$SESSION_ID"
|
||||
else
|
||||
validate_artifact_dir "$ARTIFACT_DIR"
|
||||
SESSION_ID=$(basename "$ARTIFACT_DIR")
|
||||
fi
|
||||
|
||||
mkdir -p "$ARTIFACT_DIR/steps"
|
||||
STEPS_TSV="$ARTIFACT_DIR/steps.tsv"
|
||||
printf 'lane\tstep\tstatus\texit_code\tcommand\tstdout\tstderr\tnote\n' >"$STEPS_TSV"
|
||||
|
||||
STARTED_AT=$(timestamp_iso)
|
||||
SESSION_HOME="$REPO_ROOT/.tmp/skill-verification/runtime/$SESSION_ID/home"
|
||||
SESSION_XDG_CONFIG_HOME="$REPO_ROOT/.tmp/skill-verification/runtime/$SESSION_ID/xdg-config"
|
||||
SESSION_MPV_DIR="$SESSION_XDG_CONFIG_HOME/mpv"
|
||||
SESSION_LOGS_DIR="$REPO_ROOT/.tmp/skill-verification/runtime/$SESSION_ID/logs"
|
||||
SESSION_MPV_LOG="$SESSION_LOGS_DIR/mpv.log"
|
||||
mkdir -p "$SESSION_HOME" "$SESSION_MPV_DIR" "$SESSION_LOGS_DIR"
|
||||
|
||||
CLASSIFIER_OUTPUT="$ARTIFACT_DIR/classification.txt"
|
||||
if [[ ${#SELECTED_LANES[@]} -eq 0 ]]; then
|
||||
if [[ ${#PATH_ARGS[@]} -gt 0 ]]; then
|
||||
PATH_SELECTION_MODE="explicit-paths"
|
||||
fi
|
||||
if "$SCRIPT_DIR/classify_subminer_diff.sh" "${PATH_ARGS[@]}" >"$CLASSIFIER_OUTPUT"; then
|
||||
while IFS= read -r line; do
|
||||
case "$line" in
|
||||
lane:*)
|
||||
add_lane "${line#lane:}"
|
||||
;;
|
||||
esac
|
||||
done <"$CLASSIFIER_OUTPUT"
|
||||
else
|
||||
record_failed_step "meta" "classify" "classification failed"
|
||||
fi
|
||||
else
|
||||
: >"$CLASSIFIER_OUTPUT"
|
||||
fi
|
||||
|
||||
record_env
|
||||
|
||||
if [[ ${#SELECTED_LANES[@]} -eq 0 ]]; then
|
||||
add_lane "core"
|
||||
fi
|
||||
|
||||
for lane in "${SELECTED_LANES[@]}"; do
|
||||
case "$lane" in
|
||||
docs)
|
||||
run_step "$lane" "docs-kb" "bun run test:docs:kb" || break
|
||||
;;
|
||||
config)
|
||||
run_step "$lane" "config" "bun run test:config" || break
|
||||
;;
|
||||
stats)
|
||||
run_step "$lane" "stats" "bun run test:stats" || break
|
||||
;;
|
||||
core)
|
||||
run_step "$lane" "typecheck" "bun run typecheck" || break
|
||||
run_step "$lane" "fast-tests" "bun run test:fast" || break
|
||||
;;
|
||||
launcher-plugin)
|
||||
run_step "$lane" "launcher" "bun run test:launcher" || break
|
||||
run_step "$lane" "plugin-src" "bun run test:plugin:src" || break
|
||||
;;
|
||||
runtime-compat)
|
||||
run_step "$lane" "runtime-compat" "bun run test:runtime:compat" || break
|
||||
;;
|
||||
real-runtime)
|
||||
if [[ "$ALLOW_REAL_RUNTIME" != "1" ]]; then
|
||||
record_blocked_step "$lane" "real-runtime" "real-runtime requested without --allow-real-runtime"
|
||||
continue
|
||||
fi
|
||||
if ! acquire_real_runtime_lease; then
|
||||
record_blocked_step "$lane" "real-runtime-lease" "$REAL_RUNTIME_LEASE_ERROR"
|
||||
continue
|
||||
fi
|
||||
helper=$(find_real_runtime_helper || true)
|
||||
if [[ -z "${helper:-}" ]]; then
|
||||
record_blocked_step "$lane" "real-runtime-helper" "no real-runtime helper script available in $SCRIPT_DIR"
|
||||
continue
|
||||
fi
|
||||
run_step "$lane" "real-runtime" "\"$helper\" \"$SESSION_ID\" \"$ARTIFACT_DIR\"" || break
|
||||
;;
|
||||
*)
|
||||
record_blocked_step "$lane" "unknown-lane" "unknown lane: $lane"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
release_real_runtime_lease
|
||||
FINISHED_AT=$(timestamp_iso)
|
||||
compute_final_status
|
||||
write_summary_files
|
||||
|
||||
printf 'summary:%s\n' "$ARTIFACT_DIR/summary.txt"
|
||||
cat "$ARTIFACT_DIR/summary.txt"
|
||||
@@ -1,118 +0,0 @@
|
||||
---
|
||||
name: 'subminer-scrum-master'
|
||||
description: 'Use in the SubMiner repo when a request should be turned into planned work and driven through execution. Records a plan, dispatches one or more subagents when useful, and requires verification before handoff.'
|
||||
---
|
||||
|
||||
# SubMiner Scrum Master
|
||||
|
||||
Canonical source: this plugin path.
|
||||
|
||||
Own workflow, not code by default.
|
||||
|
||||
Use this skill when the user gives a feature request, bug report, issue, refactor, or implementation ask and the agent should manage intake, planning, worker dispatch, and verification through completion.
|
||||
|
||||
## Core Rules
|
||||
|
||||
1. Keep the process light for questions, obvious mechanical edits, and tiny isolated changes.
|
||||
2. Record a plan before dispatching coding work.
|
||||
3. Split multi-part work into clear phases and ownership areas.
|
||||
4. Dispatch conservatively. Parallelize only disjoint write scopes.
|
||||
5. Require verification before handoff, typically via `subminer-change-verification`.
|
||||
6. Report dispatched workers, verification, blockers, and remaining risks.
|
||||
|
||||
## Intake Workflow
|
||||
|
||||
1. Parse the request.
|
||||
Classify it as question, mechanical edit, bugfix, feature, refactor, investigation, or follow-up.
|
||||
2. Write a short working plan in-thread when the work is nontrivial.
|
||||
3. Choose execution mode:
|
||||
- no subagents for trivial work
|
||||
- one worker for focused work
|
||||
- parallel workers only for disjoint scopes
|
||||
4. Run verification before handoff.
|
||||
|
||||
## Dispatch Rules
|
||||
|
||||
The scrum master orchestrates. Workers implement.
|
||||
|
||||
- Do not become the default implementer unless delegation is unnecessary.
|
||||
- Do not parallelize overlapping files or tightly coupled runtime work.
|
||||
- Give every worker explicit ownership of files/modules.
|
||||
- Tell every worker other agents may be active and they must not revert unrelated edits.
|
||||
- Require each worker to report:
|
||||
- changed files
|
||||
- tests run
|
||||
- blockers
|
||||
|
||||
Use worker agents for implementation and explorer agents only for bounded codebase questions.
|
||||
|
||||
## Verification
|
||||
|
||||
Every nontrivial code task gets verification.
|
||||
|
||||
Preferred flow:
|
||||
|
||||
1. use `subminer-change-verification`
|
||||
2. start with the cheapest sufficient lane
|
||||
3. escalate only when needed
|
||||
4. if worker verification is sufficient, accept it or run one final consolidating pass
|
||||
|
||||
Never hand off nontrivial work without stating what was verified and what was skipped.
|
||||
|
||||
## Pre-Handoff Policy Checks
|
||||
|
||||
Before handoff, always ask and answer both questions explicitly:
|
||||
|
||||
1. Docs update required?
|
||||
2. Changelog fragment required?
|
||||
|
||||
Rules:
|
||||
|
||||
- Do not assume silence implies "no."
|
||||
- If the answer is yes, complete the update or report the blocker.
|
||||
- Include final yes/no answers in the handoff summary even when both answers are "no."
|
||||
|
||||
## Failure / Scope Handling
|
||||
|
||||
- If a worker hits ambiguity, pause and ask the user.
|
||||
- If verification fails, either:
|
||||
- send the worker back with exact failure context, or
|
||||
- fix it directly if it is tiny and clearly in scope
|
||||
- If new scope appears, pause and re-plan before silently expanding work.
|
||||
|
||||
## Representative Flows
|
||||
|
||||
### Trivial work
|
||||
|
||||
- keep a short plan
|
||||
- implement directly or with one worker if helpful
|
||||
- run targeted verification
|
||||
- report outcome concisely
|
||||
|
||||
### Focused implementation
|
||||
|
||||
- record plan
|
||||
- dispatch one worker
|
||||
- integrate
|
||||
- verify
|
||||
- report outcome
|
||||
|
||||
### Multi-part execution
|
||||
|
||||
- define distinct deliverables/phases
|
||||
- record sequencing in the plan
|
||||
- dispatch workers only where scopes are disjoint
|
||||
- integrate
|
||||
- run consolidated verification
|
||||
- report outcome
|
||||
|
||||
## Output Expectations
|
||||
|
||||
At the end, report:
|
||||
|
||||
- which workers were dispatched and what they owned
|
||||
- what verification ran
|
||||
- explicit answers to:
|
||||
- docs update required?
|
||||
- changelog fragment required?
|
||||
- blockers, skips, and risks
|
||||
@@ -1,67 +0,0 @@
|
||||
## Highlights
|
||||
### Added
|
||||
- **Anki Maturity Known-Word Highlighting**
|
||||
- Subtitle words you already know can now be color-coded by their Anki card maturity (new, learning, young, mature), like asbplayer's known-word coloring.
|
||||
- Enable it with `ankiConnect.knownWords.maturityEnabled` (or toggle it mid-session); tier colors and the "mature" day threshold are configurable, and the in-session help legend shows the active colors.
|
||||
- **Cross-Machine Sync for Stats & Watch History**
|
||||
- Sync immersion stats and watch history between machines over SSH from a new Sync window (tray menu → Sync Stats & History, or `subminer sync --ui`) or the CLI (`subminer sync <host>`).
|
||||
- Save multiple devices with per-host sync direction, run one-click syncs with live progress, and take manual database snapshots for backup or transfer.
|
||||
- Windows remotes are supported over OpenSSH, and hosts can auto-sync in the background on a schedule, even during playback.
|
||||
- **History Menu After Playback**
|
||||
- After a watch-history episode ends (or mpv closes), the fzf/rofi launcher now offers to play the previous or next episode, rewatch, pick another episode, or quit, right from where you left off. Previous/Next continue across season folders.
|
||||
- **Delete Entire Library Titles from Stats**
|
||||
- The stats Library detail view now has a "Delete Entry" action that removes a whole title in one step, episodes, sessions, subtitle lines, rollups, cover art, and vocabulary counts, instead of clearing it episode by episode.
|
||||
- Delete progress (sessions, episodes, or whole titles) now shows app-wide with a progress bar and status toast visible from any tab or window.
|
||||
- **TsukiHime English Subtitle Downloads**
|
||||
- Download subtitles for the currently playing video directly from TsukiHime, with Japanese loaded as the primary track and your configured secondary language alongside it.
|
||||
|
||||
### Changed
|
||||
- **Configurable Clipboard-Video Shortcut**
|
||||
- The "append clipboard video to queue" shortcut is now configurable via `shortcuts.appendClipboardVideoToQueue` instead of fixed.
|
||||
|
||||
### Fixed
|
||||
- **AniList Season Matching**
|
||||
- Season 2+ episodes now resolve to the correct AniList entry instead of silently falling back to season 1. SubMiner follows AniList's sequel relations to find the right season, and cover art and watch progress now use the same season-aware match.
|
||||
- If a season still can't be found, SubMiner no longer force-writes progress or a cover to the season 1 entry, it skips the update and points you to a manual AniList override, which now fixes the character dictionary and watch progress together.
|
||||
- Manual overrides now stay applied consistently across every episode in a season folder, even when filenames guess differently episode to episode.
|
||||
- **Subtitle Highlighting Accuracy**
|
||||
- Fixed several known-word/annotation edge cases: part-of-speech exclusions now apply consistently to merged quote-particle tokens, annotations for rarer kanji are preserved, katakana punctuation is no longer mistaken for plain kana, and a specific noun-tagging case no longer loses its known+1 highlight.
|
||||
- **AnkiConnect Proxy Port Conflicts**
|
||||
- Fixed a crash on video startup when another process already held the configured AnkiConnect proxy port; you'll now get a notification explaining how to resolve it instead.
|
||||
- **AppImage Crash Notification on Quit**
|
||||
- Fixed a spurious "Service Crash" desktop notification appearing after closing a video when running the Linux AppImage.
|
||||
- **Startup Playback Pause Timing**
|
||||
- Fixed playback occasionally resuming a couple seconds before subtitle tokenization actually finished warming up, most noticeable when resuming mid-episode or when a subtitle appears in the first two seconds.
|
||||
- **Stats Library Cover After Relinking**
|
||||
- Fixed the stats Library grid showing a stale cover image after relinking a title to a different AniList entry.
|
||||
- **Faster Stats Deletes and Vocabulary Tab**
|
||||
- Deleting sessions, episodes, and titles from stats is now dramatically faster and no longer stalls playback while it runs; the Vocabulary tab also loads much faster.
|
||||
- The first launch after updating runs a one-time database migration (a few seconds, database grows about 20%); no action needed.
|
||||
- **Settings Validation and Stats Server Hardening**
|
||||
- Invalid AnkiConnect settings now fall back safely with a warning instead of silently breaking, and the stats server is hardened against malformed requests, stalled AniList searches, and other edge cases that could previously crash it.
|
||||
- **Rofi Prompt Spacing**
|
||||
- Fixed rofi menu prompts running into the search placeholder text with no space between them.
|
||||
|
||||
## What's Changed
|
||||
|
||||
- feat(shortcuts): make clipboard-video-append shortcut configurable by @ksyasuda in #158
|
||||
- refactor(tokenizer): extract subtitle annotation filter into rule table by @ksyasuda in #162
|
||||
- refactor(tsukihime): swap Animetosho backend for TsukiHime API by @ksyasuda in #165
|
||||
- refactor: split anki-connect and stats-server resolvers into modules by @ksyasuda in #169
|
||||
- feat(launcher): add post-playback history menu with previous episode by @ksyasuda in #170
|
||||
- Anki maturity-based known-word highlighting by @ksyasuda in #172
|
||||
- fix(anilist): resolve later seasons via sequel relations, not title guessing by @ksyasuda in #173
|
||||
- feat(stats): add library entry deletion and app-wide delete progress by @ksyasuda in #174
|
||||
|
||||
## Installation
|
||||
|
||||
See the README and docs/installation guide for full setup steps.
|
||||
|
||||
## Assets
|
||||
|
||||
- Linux: `SubMiner.AppImage`
|
||||
- macOS: `SubMiner-*.dmg` and `SubMiner-*.zip`
|
||||
- Windows: `SubMiner-*.exe` and `SubMiner-*-win.zip`
|
||||
- Optional extras: `subminer-assets.tar.gz` and the `subminer` launcher
|
||||
|
||||
Note: the `subminer` wrapper script uses Bun (`#!/usr/bin/env bun`), so `bun` must be installed and on `PATH`.
|
||||
@@ -19,6 +19,7 @@ const requiredDocs = [
|
||||
'docs/knowledge-base/catalog.md',
|
||||
'docs/knowledge-base/quality.md',
|
||||
'docs/workflow/README.md',
|
||||
'docs/workflow/agent-skills.md',
|
||||
'docs/workflow/planning.md',
|
||||
'docs/workflow/verification.md',
|
||||
] as const;
|
||||
|
||||
@@ -1,202 +0,0 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import test from 'node:test';
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const classifyScript = path.join(
|
||||
repoRoot,
|
||||
'.agents/skills/subminer-change-verification/scripts/classify_subminer_diff.sh',
|
||||
);
|
||||
const verifyScript = path.join(
|
||||
repoRoot,
|
||||
'.agents/skills/subminer-change-verification/scripts/verify_subminer_change.sh',
|
||||
);
|
||||
|
||||
function withTempDir<T>(fn: (dir: string) => T): T {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-change-verification-test-'));
|
||||
try {
|
||||
return fn(dir);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function runBash(args: string[]) {
|
||||
return spawnSync('bash', args, {
|
||||
cwd: repoRoot,
|
||||
env: process.env,
|
||||
encoding: 'utf8',
|
||||
});
|
||||
}
|
||||
|
||||
function parseArtifactDir(stdout: string): string {
|
||||
const match = stdout.match(/^artifacts: (.+)$/m);
|
||||
assert.ok(match, `expected artifact_dir in stdout, got:\n${stdout}`);
|
||||
return match[1] ?? '';
|
||||
}
|
||||
|
||||
function readSummaryJson(artifactDir: string) {
|
||||
return JSON.parse(fs.readFileSync(path.join(artifactDir, 'summary.json'), 'utf8')) as {
|
||||
sessionId: string;
|
||||
status: string;
|
||||
lanes: string[];
|
||||
blockers?: string[];
|
||||
artifactDir: string;
|
||||
pathSelectionMode?: string;
|
||||
steps: Array<{
|
||||
lane: string;
|
||||
name: string;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
note: string;
|
||||
}>;
|
||||
};
|
||||
}
|
||||
|
||||
test('classifier marks launcher and plugin paths as real-runtime candidates', () => {
|
||||
const result = runBash([classifyScript, 'launcher/mpv.ts', 'plugin/subminer/process.lua']);
|
||||
|
||||
assert.equal(result.status, 0, result.stderr || result.stdout);
|
||||
assert.match(result.stdout, /^lane:launcher-plugin$/m);
|
||||
assert.match(result.stdout, /^flag:real-runtime-candidate$/m);
|
||||
assert.doesNotMatch(result.stdout, /real-gui-candidate/);
|
||||
});
|
||||
|
||||
test('verifier blocks requested real-runtime lane when runtime execution is not allowed', () => {
|
||||
withTempDir((root) => {
|
||||
const artifactDir = path.join(root, 'artifacts');
|
||||
const result = runBash([
|
||||
verifyScript,
|
||||
'--dry-run',
|
||||
'--artifact-dir',
|
||||
artifactDir,
|
||||
'--lane',
|
||||
'real-runtime',
|
||||
'launcher/mpv.ts',
|
||||
]);
|
||||
|
||||
assert.equal(result.status, 0, result.stdout);
|
||||
|
||||
const summary = readSummaryJson(artifactDir);
|
||||
assert.equal(summary.status, 'blocked');
|
||||
assert.deepEqual(summary.lanes, ['real-runtime']);
|
||||
assert.ok(summary.sessionId.length > 0);
|
||||
assert.ok(summary.blockers?.some((entry) => entry.includes('--allow-real-runtime')));
|
||||
assert.equal(fs.existsSync(path.join(artifactDir, 'summary.json')), true);
|
||||
});
|
||||
});
|
||||
|
||||
test('verifier fails closed for unknown lanes', () => {
|
||||
withTempDir((root) => {
|
||||
const artifactDir = path.join(root, 'artifacts');
|
||||
const result = runBash([
|
||||
verifyScript,
|
||||
'--dry-run',
|
||||
'--artifact-dir',
|
||||
artifactDir,
|
||||
'--lane',
|
||||
'not-a-lane',
|
||||
'src/main.ts',
|
||||
]);
|
||||
|
||||
assert.equal(result.status, 0, result.stdout);
|
||||
|
||||
const summary = readSummaryJson(artifactDir);
|
||||
assert.equal(summary.status, 'blocked');
|
||||
assert.deepEqual(summary.lanes, ['not-a-lane']);
|
||||
assert.ok(summary.blockers?.some((entry) => entry.includes('unknown lane')));
|
||||
});
|
||||
});
|
||||
|
||||
test('verifier keeps non-passing step artifacts distinct across lanes', () => {
|
||||
withTempDir((root) => {
|
||||
const artifactDir = path.join(root, 'artifacts');
|
||||
const result = runBash([
|
||||
verifyScript,
|
||||
'--dry-run',
|
||||
'--artifact-dir',
|
||||
artifactDir,
|
||||
'--lane',
|
||||
'docs',
|
||||
'--lane',
|
||||
'not-a-lane',
|
||||
'src/main.ts',
|
||||
]);
|
||||
|
||||
assert.equal(result.status, 0, result.stdout);
|
||||
|
||||
const summary = readSummaryJson(artifactDir);
|
||||
const docsStep = summary.steps.find((step) => step.lane === 'docs' && step.name === 'docs-kb');
|
||||
const unknownStep = summary.steps.find(
|
||||
(step) => step.lane === 'not-a-lane' && step.name === 'unknown-lane',
|
||||
);
|
||||
|
||||
assert.ok(docsStep);
|
||||
assert.ok(unknownStep);
|
||||
assert.notEqual(docsStep?.stdout, unknownStep?.stdout);
|
||||
assert.equal(fs.existsSync(path.join(artifactDir, docsStep!.stdout)), true);
|
||||
assert.equal(fs.existsSync(path.join(artifactDir, unknownStep!.stdout)), true);
|
||||
});
|
||||
});
|
||||
|
||||
test('verifier records the real-runtime lease blocker once', () => {
|
||||
withTempDir((root) => {
|
||||
const artifactDir = path.join(root, 'artifacts');
|
||||
const leaseDir = path.join(
|
||||
repoRoot,
|
||||
'.tmp',
|
||||
'skill-verification',
|
||||
'locks',
|
||||
'exclusive-real-runtime',
|
||||
);
|
||||
fs.mkdirSync(leaseDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(leaseDir, 'session_id'), 'other-session');
|
||||
|
||||
try {
|
||||
const result = runBash([
|
||||
verifyScript,
|
||||
'--dry-run',
|
||||
'--artifact-dir',
|
||||
artifactDir,
|
||||
'--allow-real-runtime',
|
||||
'--lane',
|
||||
'real-runtime',
|
||||
'launcher/mpv.ts',
|
||||
]);
|
||||
|
||||
assert.equal(result.status, 0, result.stdout);
|
||||
|
||||
const summary = readSummaryJson(artifactDir);
|
||||
assert.deepEqual(summary.blockers, ['real-runtime lease already held by other-session']);
|
||||
} finally {
|
||||
fs.rmSync(leaseDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test('verifier allocates unique session ids and artifact roots by default', () => {
|
||||
const first = runBash([verifyScript, '--dry-run', '--lane', 'core', 'src/main.ts']);
|
||||
const second = runBash([verifyScript, '--dry-run', '--lane', 'core', 'src/main.ts']);
|
||||
|
||||
assert.equal(first.status, 0, first.stderr || first.stdout);
|
||||
assert.equal(second.status, 0, second.stderr || second.stdout);
|
||||
|
||||
const firstArtifactDir = parseArtifactDir(first.stdout);
|
||||
const secondArtifactDir = parseArtifactDir(second.stdout);
|
||||
|
||||
try {
|
||||
const firstSummary = readSummaryJson(firstArtifactDir);
|
||||
const secondSummary = readSummaryJson(secondArtifactDir);
|
||||
|
||||
assert.notEqual(firstSummary.sessionId, secondSummary.sessionId);
|
||||
assert.notEqual(firstArtifactDir, secondArtifactDir);
|
||||
assert.equal(firstSummary.pathSelectionMode, 'explicit-lanes');
|
||||
assert.equal(secondSummary.pathSelectionMode, 'explicit-lanes');
|
||||
} finally {
|
||||
fs.rmSync(firstArtifactDir, { recursive: true, force: true });
|
||||
fs.rmSync(secondArtifactDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
+7
-74
@@ -22,10 +22,12 @@ import { MediaGenerator } from './media-generator';
|
||||
import path from 'path';
|
||||
import {
|
||||
AnkiConnectConfig,
|
||||
type CardKind,
|
||||
KikuDuplicateCardInfo,
|
||||
KikuFieldGroupingChoice,
|
||||
KikuMergePreviewResponse,
|
||||
NotificationOptions,
|
||||
type WordCardKind,
|
||||
} from './types/anki';
|
||||
import { AiConfig } from './types/integrations';
|
||||
import type { KnownWordMaturityTier } from './types/subtitle';
|
||||
@@ -50,6 +52,7 @@ import {
|
||||
withUpdateProgress,
|
||||
UiFeedbackState,
|
||||
} from './anki-integration/ui-feedback';
|
||||
import { applyCardKindFlagFields, resolveWordCardKindSetting } from './anki-integration/card-kinds';
|
||||
import { KnownWordCacheManager } from './anki-integration/known-word-cache';
|
||||
import { PollingRunner } from './anki-integration/polling';
|
||||
import type { AnkiConnectProxyServer } from './anki-integration/anki-connect-proxy';
|
||||
@@ -83,8 +86,6 @@ interface NoteInfo {
|
||||
fields: Record<string, { value: string }>;
|
||||
}
|
||||
|
||||
type CardKind = 'sentence' | 'audio' | 'word-and-sentence';
|
||||
|
||||
function trimToNonEmptyString(value: unknown): string | null {
|
||||
if (typeof value !== 'string') return null;
|
||||
const trimmed = value.trim();
|
||||
@@ -840,6 +841,7 @@ export class AnkiIntegration {
|
||||
kikuEnabled: boolean;
|
||||
kikuFieldGrouping: 'auto' | 'manual' | 'disabled';
|
||||
kikuDeleteDuplicateInAuto: boolean;
|
||||
wordCardKind: WordCardKind;
|
||||
} {
|
||||
const lapis = this.getLapisConfig();
|
||||
const kiku = this.getKikuConfig();
|
||||
@@ -852,6 +854,7 @@ export class AnkiIntegration {
|
||||
kikuEnabled: kiku.enabled,
|
||||
kikuFieldGrouping: (kiku.fieldGrouping || 'disabled') as 'auto' | 'manual' | 'disabled',
|
||||
kikuDeleteDuplicateInAuto: kiku.deleteDuplicateInAuto !== false,
|
||||
wordCardKind: resolveWordCardKindSetting(this.config.lapisKiku?.wordCardKind),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1315,79 +1318,9 @@ export class AnkiIntegration {
|
||||
availableFieldNames: string[],
|
||||
cardKind: CardKind,
|
||||
): void {
|
||||
const audioFlagNames = ['IsAudioCard'];
|
||||
|
||||
if (cardKind === 'word-and-sentence') {
|
||||
const wordAndSentenceFlag = this.resolveFieldName(
|
||||
availableFieldNames,
|
||||
'IsWordAndSentenceCard',
|
||||
);
|
||||
if (!wordAndSentenceFlag) {
|
||||
return;
|
||||
}
|
||||
updatedFields[wordAndSentenceFlag] = 'x';
|
||||
|
||||
const sentenceFlag = this.resolveFieldName(availableFieldNames, 'IsSentenceCard');
|
||||
if (sentenceFlag && sentenceFlag !== wordAndSentenceFlag) {
|
||||
updatedFields[sentenceFlag] = '';
|
||||
}
|
||||
|
||||
for (const audioFlagName of audioFlagNames) {
|
||||
const resolved = this.resolveFieldName(availableFieldNames, audioFlagName);
|
||||
if (resolved && resolved !== wordAndSentenceFlag) {
|
||||
updatedFields[resolved] = '';
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (cardKind === 'sentence') {
|
||||
const sentenceFlag = this.resolveFieldName(availableFieldNames, 'IsSentenceCard');
|
||||
if (sentenceFlag) {
|
||||
updatedFields[sentenceFlag] = 'x';
|
||||
}
|
||||
|
||||
for (const audioFlagName of audioFlagNames) {
|
||||
const resolved = this.resolveFieldName(availableFieldNames, audioFlagName);
|
||||
if (resolved && resolved !== sentenceFlag) {
|
||||
updatedFields[resolved] = '';
|
||||
}
|
||||
}
|
||||
|
||||
const wordAndSentenceFlag = this.resolveFieldName(
|
||||
availableFieldNames,
|
||||
'IsWordAndSentenceCard',
|
||||
);
|
||||
if (wordAndSentenceFlag && wordAndSentenceFlag !== sentenceFlag) {
|
||||
updatedFields[wordAndSentenceFlag] = '';
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvedAudioFlags = Array.from(
|
||||
new Set(
|
||||
audioFlagNames
|
||||
.map((name) => this.resolveFieldName(availableFieldNames, name))
|
||||
.filter((name): name is string => Boolean(name)),
|
||||
),
|
||||
applyCardKindFlagFields(updatedFields, cardKind, (preferredName) =>
|
||||
this.resolveFieldName(availableFieldNames, preferredName),
|
||||
);
|
||||
const audioFlagName = resolvedAudioFlags[0] || null;
|
||||
if (audioFlagName) {
|
||||
updatedFields[audioFlagName] = 'x';
|
||||
}
|
||||
for (const extraAudioFlag of resolvedAudioFlags.slice(1)) {
|
||||
updatedFields[extraAudioFlag] = '';
|
||||
}
|
||||
|
||||
const sentenceFlag = this.resolveFieldName(availableFieldNames, 'IsSentenceCard');
|
||||
if (sentenceFlag && sentenceFlag !== audioFlagName) {
|
||||
updatedFields[sentenceFlag] = '';
|
||||
}
|
||||
|
||||
const wordAndSentenceFlag = this.resolveFieldName(availableFieldNames, 'IsWordAndSentenceCard');
|
||||
if (wordAndSentenceFlag && wordAndSentenceFlag !== audioFlagName) {
|
||||
updatedFields[wordAndSentenceFlag] = '';
|
||||
}
|
||||
}
|
||||
|
||||
private async showNotification(
|
||||
|
||||
@@ -4,29 +4,23 @@ import test from 'node:test';
|
||||
import { CardCreationService } from './card-creation';
|
||||
import { toMpvEdlValue } from './mpv-edl-test-utils';
|
||||
import type { MediaInput } from '../media-generator';
|
||||
import type { AnkiConnectConfig } from '../types/anki';
|
||||
import type { AnkiConnectConfig, CardKind } from '../types/anki';
|
||||
import { applyCardKindFlagFields } from './card-kinds';
|
||||
|
||||
type CardCreationDeps = ConstructorParameters<typeof CardCreationService>[0];
|
||||
|
||||
function setWordAndSentenceCardTypeFields(
|
||||
function setCardTypeFields(
|
||||
updatedFields: Record<string, string>,
|
||||
availableFieldNames: string[],
|
||||
cardKind: 'sentence' | 'audio' | 'word-and-sentence',
|
||||
cardKind: CardKind,
|
||||
): void {
|
||||
if (cardKind !== 'word-and-sentence') return;
|
||||
|
||||
const resolveFieldName = (preferredName: string): string | null =>
|
||||
availableFieldNames.find((name) => name.toLowerCase() === preferredName.toLowerCase()) ?? null;
|
||||
const wordAndSentenceFlag = resolveFieldName('IsWordAndSentenceCard');
|
||||
if (!wordAndSentenceFlag) return;
|
||||
|
||||
updatedFields[wordAndSentenceFlag] = 'x';
|
||||
for (const flagName of ['IsSentenceCard', 'IsAudioCard']) {
|
||||
const resolved = resolveFieldName(flagName);
|
||||
if (resolved && resolved !== wordAndSentenceFlag) {
|
||||
updatedFields[resolved] = '';
|
||||
}
|
||||
}
|
||||
applyCardKindFlagFields(
|
||||
updatedFields,
|
||||
cardKind,
|
||||
(preferredName) =>
|
||||
availableFieldNames.find((name) => name.toLowerCase() === preferredName.toLowerCase()) ??
|
||||
null,
|
||||
);
|
||||
}
|
||||
|
||||
function createManualUpdateService(overrides: Partial<CardCreationDeps> = {}): {
|
||||
@@ -217,7 +211,7 @@ test('manual clipboard subtitle update marks Kiku word cards as word-and-sentenc
|
||||
kikuFieldGrouping: 'disabled',
|
||||
kikuDeleteDuplicateInAuto: false,
|
||||
}),
|
||||
setCardTypeFields: setWordAndSentenceCardTypeFields,
|
||||
setCardTypeFields,
|
||||
});
|
||||
|
||||
await service.updateLastAddedFromClipboard('字幕');
|
||||
|
||||
@@ -3,7 +3,7 @@ import {
|
||||
getConfiguredWordFieldName,
|
||||
getPreferredWordValueFromExtractedFields,
|
||||
} from '../anki-field-config';
|
||||
import { AnkiConnectConfig } from '../types/anki';
|
||||
import { AnkiConnectConfig, type CardKind, type WordCardKind } from '../types/anki';
|
||||
import { createLogger } from '../logger';
|
||||
import type { MediaInput } from '../media-input';
|
||||
import { SubtitleTimingTracker } from '../subtitle-timing-tracker';
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
resolveAudioStreamIndexForMediaGeneration,
|
||||
type MediaGenerationInputResolverOptions,
|
||||
} from './media-source';
|
||||
import { shouldMarkWordAndSentenceCard } from './note-field-utils';
|
||||
import { resolveWordCardKind } from './note-field-utils';
|
||||
import type { PendingYoutubeMediaUpdate } from './pending-youtube-media';
|
||||
import { resolveMpvVolumeScale } from './mpv-volume';
|
||||
|
||||
@@ -42,8 +42,6 @@ export interface CardCreationNoteInfo {
|
||||
fields: Record<string, { value: string }>;
|
||||
}
|
||||
|
||||
type CardKind = 'sentence' | 'audio' | 'word-and-sentence';
|
||||
|
||||
interface CardCreationClient {
|
||||
addNote(
|
||||
deck: string,
|
||||
@@ -136,6 +134,7 @@ interface CardCreationDeps {
|
||||
kikuEnabled: boolean;
|
||||
kikuFieldGrouping: 'auto' | 'manual' | 'disabled';
|
||||
kikuDeleteDuplicateInAuto: boolean;
|
||||
wordCardKind?: WordCardKind;
|
||||
};
|
||||
getFallbackDurationSeconds: () => number;
|
||||
appendKnownWordsFromNoteInfo: (noteInfo: CardCreationNoteInfo) => void;
|
||||
@@ -274,12 +273,9 @@ export class CardCreationService {
|
||||
if (sentenceField) {
|
||||
const processedSentence = this.deps.processSentence(sentence, fields);
|
||||
updatedFields[sentenceField] = processedSentence;
|
||||
if (shouldMarkWordAndSentenceCard(noteInfo, sentenceCardConfig)) {
|
||||
this.deps.setCardTypeFields(
|
||||
updatedFields,
|
||||
Object.keys(noteInfo.fields),
|
||||
'word-and-sentence',
|
||||
);
|
||||
const wordCardKind = resolveWordCardKind(noteInfo, sentenceCardConfig);
|
||||
if (wordCardKind) {
|
||||
this.deps.setCardTypeFields(updatedFields, Object.keys(noteInfo.fields), wordCardKind);
|
||||
}
|
||||
updatePerformed = true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
|
||||
import { applyCardKindFlagFields } from './card-kinds';
|
||||
|
||||
function resolverFor(availableFieldNames: string[]) {
|
||||
return (preferredName: string): string | null =>
|
||||
availableFieldNames.find((name) => name.toLowerCase() === preferredName.toLowerCase()) ?? null;
|
||||
}
|
||||
|
||||
const KIKU_FLAG_FIELDS = ['IsWordAndSentenceCard', 'IsClickCard', 'IsSentenceCard', 'IsAudioCard'];
|
||||
|
||||
test('flags the requested card kind and clears the others', () => {
|
||||
const fields: Record<string, string> = {};
|
||||
|
||||
applyCardKindFlagFields(fields, 'click', resolverFor(KIKU_FLAG_FIELDS));
|
||||
|
||||
assert.deepEqual(fields, {
|
||||
IsClickCard: 'x',
|
||||
IsWordAndSentenceCard: '',
|
||||
IsSentenceCard: '',
|
||||
IsAudioCard: '',
|
||||
});
|
||||
});
|
||||
|
||||
test('matches flag fields case-insensitively', () => {
|
||||
const fields: Record<string, string> = {};
|
||||
|
||||
applyCardKindFlagFields(fields, 'word-and-sentence', resolverFor(['iswordandsentencecard']));
|
||||
|
||||
assert.deepEqual(fields, { iswordandsentencecard: 'x' });
|
||||
});
|
||||
|
||||
test('leaves flags untouched when the note type has no flag for a word card kind', () => {
|
||||
const fields: Record<string, string> = {};
|
||||
|
||||
applyCardKindFlagFields(
|
||||
fields,
|
||||
'click',
|
||||
resolverFor(['IsWordAndSentenceCard', 'IsSentenceCard']),
|
||||
);
|
||||
|
||||
assert.deepEqual(fields, {});
|
||||
});
|
||||
|
||||
test('clears stale flags for explicit mine actions even without the target flag', () => {
|
||||
const fields: Record<string, string> = {};
|
||||
|
||||
applyCardKindFlagFields(
|
||||
fields,
|
||||
'audio',
|
||||
resolverFor(['IsWordAndSentenceCard', 'IsSentenceCard']),
|
||||
);
|
||||
|
||||
assert.deepEqual(fields, { IsWordAndSentenceCard: '', IsSentenceCard: '' });
|
||||
});
|
||||
|
||||
test('does not blank the target flag it just set', () => {
|
||||
const fields: Record<string, string> = {};
|
||||
|
||||
applyCardKindFlagFields(fields, 'sentence', resolverFor(['IsSentenceCard']));
|
||||
|
||||
assert.deepEqual(fields, { IsSentenceCard: 'x' });
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
import type { CardKind, WordCardKind } from '../types/anki';
|
||||
|
||||
/**
|
||||
* Kiku/Lapis note types decide which card a note generates from mutually exclusive
|
||||
* `Is...Card` flag fields. Setting one always means clearing the others.
|
||||
*/
|
||||
export const CARD_KIND_FLAG_FIELDS: Record<CardKind, string> = {
|
||||
'word-and-sentence': 'IsWordAndSentenceCard',
|
||||
click: 'IsClickCard',
|
||||
sentence: 'IsSentenceCard',
|
||||
audio: 'IsAudioCard',
|
||||
};
|
||||
|
||||
export const WORD_CARD_KINDS: readonly WordCardKind[] = [
|
||||
'word-and-sentence',
|
||||
'click',
|
||||
'sentence',
|
||||
'audio',
|
||||
'none',
|
||||
];
|
||||
|
||||
export const DEFAULT_WORD_CARD_KIND: WordCardKind = 'word-and-sentence';
|
||||
|
||||
/**
|
||||
* Card kinds SubMiner marks on its own initiative (word cards). They are only applied
|
||||
* when the note type actually carries the matching flag field, so plain note types keep
|
||||
* their fields untouched.
|
||||
*/
|
||||
const IMPLICIT_CARD_KINDS = new Set<CardKind>(['word-and-sentence', 'click']);
|
||||
|
||||
export function isWordCardKind(value: unknown): value is WordCardKind {
|
||||
return typeof value === 'string' && WORD_CARD_KINDS.includes(value as WordCardKind);
|
||||
}
|
||||
|
||||
export function resolveWordCardKindSetting(value: unknown): WordCardKind {
|
||||
return isWordCardKind(value) ? value : DEFAULT_WORD_CARD_KIND;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flags `cardKind` on the note and clears every other card-kind flag it has, so the note
|
||||
* never ends up claiming to be two kinds of card at once.
|
||||
*/
|
||||
export function applyCardKindFlagFields(
|
||||
updatedFields: Record<string, string>,
|
||||
cardKind: CardKind,
|
||||
resolveFieldName: (preferredName: string) => string | null,
|
||||
): void {
|
||||
const targetFlag = resolveFieldName(CARD_KIND_FLAG_FIELDS[cardKind]);
|
||||
if (!targetFlag && IMPLICIT_CARD_KINDS.has(cardKind)) {
|
||||
return;
|
||||
}
|
||||
if (targetFlag) {
|
||||
updatedFields[targetFlag] = 'x';
|
||||
}
|
||||
|
||||
for (const [kind, flagName] of Object.entries(CARD_KIND_FLAG_FIELDS)) {
|
||||
if (kind === cardKind) continue;
|
||||
const resolved = resolveFieldName(flagName);
|
||||
if (resolved && resolved !== targetFlag) {
|
||||
updatedFields[resolved] = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
|
||||
import { resolveWordCardKind, type NoteFieldValueInfo } from './note-field-utils';
|
||||
|
||||
function kikuNote(values: Record<string, string> = {}): NoteFieldValueInfo {
|
||||
const defaults: Record<string, string> = {
|
||||
Expression: '単語',
|
||||
Sentence: '',
|
||||
IsWordAndSentenceCard: '',
|
||||
IsClickCard: '',
|
||||
IsSentenceCard: '',
|
||||
IsAudioCard: '',
|
||||
};
|
||||
return {
|
||||
fields: Object.fromEntries(
|
||||
Object.entries({ ...defaults, ...values }).map(([name, value]) => [name, { value }]),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
test('marks word-and-sentence cards by default when Kiku is enabled', () => {
|
||||
assert.equal(
|
||||
resolveWordCardKind(kikuNote(), { lapisEnabled: false, kikuEnabled: true }),
|
||||
'word-and-sentence',
|
||||
);
|
||||
});
|
||||
|
||||
test('honors the configured word card kind', () => {
|
||||
assert.equal(
|
||||
resolveWordCardKind(kikuNote(), {
|
||||
lapisEnabled: false,
|
||||
kikuEnabled: true,
|
||||
wordCardKind: 'click',
|
||||
}),
|
||||
'click',
|
||||
);
|
||||
});
|
||||
|
||||
test('marks nothing when neither Kiku nor Lapis is enabled', () => {
|
||||
assert.equal(
|
||||
resolveWordCardKind(kikuNote(), {
|
||||
lapisEnabled: false,
|
||||
kikuEnabled: false,
|
||||
wordCardKind: 'click',
|
||||
}),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test('marks nothing when the word card kind is "none"', () => {
|
||||
assert.equal(
|
||||
resolveWordCardKind(kikuNote(), {
|
||||
lapisEnabled: true,
|
||||
kikuEnabled: false,
|
||||
wordCardKind: 'none',
|
||||
}),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test('falls back to the default kind for an unrecognized setting', () => {
|
||||
assert.equal(
|
||||
resolveWordCardKind(kikuNote(), {
|
||||
lapisEnabled: false,
|
||||
kikuEnabled: true,
|
||||
wordCardKind: 'bogus' as never,
|
||||
}),
|
||||
'word-and-sentence',
|
||||
);
|
||||
});
|
||||
|
||||
test('marks nothing when the note type lacks the configured flag field', () => {
|
||||
const note: NoteFieldValueInfo = {
|
||||
fields: { Expression: { value: '単語' }, Sentence: { value: '' } },
|
||||
};
|
||||
|
||||
assert.equal(
|
||||
resolveWordCardKind(note, { lapisEnabled: false, kikuEnabled: true, wordCardKind: 'click' }),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test('leaves cards already mined as sentence or audio cards alone', () => {
|
||||
for (const flagField of ['IsSentenceCard', 'IsAudioCard']) {
|
||||
assert.equal(
|
||||
resolveWordCardKind(kikuNote({ [flagField]: 'x' }), {
|
||||
lapisEnabled: false,
|
||||
kikuEnabled: true,
|
||||
wordCardKind: 'click',
|
||||
}),
|
||||
null,
|
||||
flagField,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('re-affirms the configured kind when the note already carries its flag', () => {
|
||||
assert.equal(
|
||||
resolveWordCardKind(kikuNote({ IsSentenceCard: 'x' }), {
|
||||
lapisEnabled: false,
|
||||
kikuEnabled: true,
|
||||
wordCardKind: 'sentence',
|
||||
}),
|
||||
'sentence',
|
||||
);
|
||||
});
|
||||
|
||||
test('overrides a differently flagged word card', () => {
|
||||
assert.equal(
|
||||
resolveWordCardKind(kikuNote({ IsWordAndSentenceCard: 'x' }), {
|
||||
lapisEnabled: false,
|
||||
kikuEnabled: true,
|
||||
wordCardKind: 'click',
|
||||
}),
|
||||
'click',
|
||||
);
|
||||
});
|
||||
@@ -1,3 +1,13 @@
|
||||
import type { CardKind, WordCardKind } from '../types/anki';
|
||||
import { createLogger } from '../logger';
|
||||
import {
|
||||
CARD_KIND_FLAG_FIELDS,
|
||||
DEFAULT_WORD_CARD_KIND,
|
||||
resolveWordCardKindSetting,
|
||||
} from './card-kinds';
|
||||
|
||||
const log = createLogger('anki').child('integration.note-fields');
|
||||
|
||||
export interface NoteFieldValueInfo {
|
||||
fields: Record<string, { value: string }>;
|
||||
}
|
||||
@@ -16,22 +26,57 @@ export function hasNoteFieldValue(noteInfo: NoteFieldValueInfo, preferredName: s
|
||||
return (getNoteFieldValue(noteInfo, preferredName) ?? '').trim().length > 0;
|
||||
}
|
||||
|
||||
export function shouldMarkWordAndSentenceCard(
|
||||
noteInfo: NoteFieldValueInfo,
|
||||
sentenceCardConfig: { lapisEnabled: boolean; kikuEnabled: boolean },
|
||||
): boolean {
|
||||
if (!sentenceCardConfig.lapisEnabled && !sentenceCardConfig.kikuEnabled) {
|
||||
return false;
|
||||
}
|
||||
/** Flags set only by an explicit mine action; a note carrying one is not a word card. */
|
||||
const EXPLICIT_CARD_FLAG_FIELDS = [CARD_KIND_FLAG_FIELDS.sentence, CARD_KIND_FLAG_FIELDS.audio];
|
||||
|
||||
const wordAndSentenceValue = getNoteFieldValue(noteInfo, 'IsWordAndSentenceCard');
|
||||
if (wordAndSentenceValue === null) {
|
||||
return false;
|
||||
const warnedMissingFlagFields = new Set<CardKind>();
|
||||
|
||||
function warnMissingFlagFieldOnce(wordCardKind: CardKind, flagField: string): void {
|
||||
if (wordCardKind === DEFAULT_WORD_CARD_KIND || warnedMissingFlagFields.has(wordCardKind)) {
|
||||
// The default kind is also the fallback for plain note types, so its absence is expected.
|
||||
return;
|
||||
}
|
||||
if (wordAndSentenceValue.trim().length > 0) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
!hasNoteFieldValue(noteInfo, 'IsSentenceCard') && !hasNoteFieldValue(noteInfo, 'IsAudioCard')
|
||||
warnedMissingFlagFields.add(wordCardKind);
|
||||
log.warn(
|
||||
`Word card type "${wordCardKind}" is configured but the note has no ${flagField} field; leaving card type flags unchanged.`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Card kind to flag when SubMiner fills a word card's sentence, or null to leave the
|
||||
* card-kind flags alone. Kiku/Lapis only: other note types have no such fields.
|
||||
*/
|
||||
export function resolveWordCardKind(
|
||||
noteInfo: NoteFieldValueInfo,
|
||||
sentenceCardConfig: {
|
||||
lapisEnabled: boolean;
|
||||
kikuEnabled: boolean;
|
||||
wordCardKind?: WordCardKind;
|
||||
},
|
||||
): CardKind | null {
|
||||
if (!sentenceCardConfig.lapisEnabled && !sentenceCardConfig.kikuEnabled) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const wordCardKind = resolveWordCardKindSetting(sentenceCardConfig.wordCardKind);
|
||||
if (wordCardKind === 'none') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const flagField = CARD_KIND_FLAG_FIELDS[wordCardKind];
|
||||
const flagValue = getNoteFieldValue(noteInfo, flagField);
|
||||
if (flagValue === null) {
|
||||
// Note type has no flag field for the configured kind.
|
||||
warnMissingFlagFieldOnce(wordCardKind, flagField);
|
||||
return null;
|
||||
}
|
||||
if (flagValue.trim().length > 0) {
|
||||
return wordCardKind;
|
||||
}
|
||||
|
||||
const alreadyExplicitCard = EXPLICIT_CARD_FLAG_FIELDS.some(
|
||||
(fieldName) =>
|
||||
fieldName.toLowerCase() !== flagField.toLowerCase() && hasNoteFieldValue(noteInfo, fieldName),
|
||||
);
|
||||
return alreadyExplicitCard ? null : wordCardKind;
|
||||
}
|
||||
|
||||
@@ -6,26 +6,21 @@ import {
|
||||
type NoteUpdateWorkflowNoteInfo,
|
||||
} from './note-update-workflow';
|
||||
import type { SubtitleMiningContext } from '../types/subtitle';
|
||||
import type { CardKind } from '../types/anki';
|
||||
import { applyCardKindFlagFields } from './card-kinds';
|
||||
|
||||
function setWordAndSentenceCardTypeFields(
|
||||
function setCardTypeFields(
|
||||
updatedFields: Record<string, string>,
|
||||
availableFieldNames: string[],
|
||||
cardKind: 'word-and-sentence',
|
||||
cardKind: CardKind,
|
||||
): void {
|
||||
assert.equal(cardKind, 'word-and-sentence');
|
||||
const resolveFieldName = (preferredName: string): string | null =>
|
||||
availableFieldNames.find((name) => name.toLowerCase() === preferredName.toLowerCase()) ?? null;
|
||||
|
||||
const wordAndSentenceFlag = resolveFieldName('IsWordAndSentenceCard');
|
||||
if (!wordAndSentenceFlag) return;
|
||||
|
||||
updatedFields[wordAndSentenceFlag] = 'x';
|
||||
for (const flagName of ['IsSentenceCard', 'IsAudioCard']) {
|
||||
const resolved = resolveFieldName(flagName);
|
||||
if (resolved && resolved !== wordAndSentenceFlag) {
|
||||
updatedFields[resolved] = '';
|
||||
}
|
||||
}
|
||||
applyCardKindFlagFields(
|
||||
updatedFields,
|
||||
cardKind,
|
||||
(preferredName) =>
|
||||
availableFieldNames.find((name) => name.toLowerCase() === preferredName.toLowerCase()) ??
|
||||
null,
|
||||
);
|
||||
}
|
||||
|
||||
function createWorkflowHarness() {
|
||||
@@ -79,7 +74,7 @@ function createWorkflowHarness() {
|
||||
handleFieldGroupingManual: async (_originalNoteId, _newNoteId, _newNoteInfo, _expression) =>
|
||||
false,
|
||||
processSentence: (text: string, _noteFields: Record<string, string>) => text,
|
||||
setCardTypeFields: setWordAndSentenceCardTypeFields,
|
||||
setCardTypeFields,
|
||||
resolveConfiguredFieldName: (noteInfo: NoteUpdateWorkflowNoteInfo, preferred?: string) => {
|
||||
if (!preferred) return null;
|
||||
const names = Object.keys(noteInfo.fields);
|
||||
@@ -183,6 +178,73 @@ test('NoteUpdateWorkflow marks enriched Kiku word cards as word-and-sentence car
|
||||
});
|
||||
});
|
||||
|
||||
test('NoteUpdateWorkflow marks the configured word card kind instead of word-and-sentence', async () => {
|
||||
const harness = createWorkflowHarness();
|
||||
harness.deps.getEffectiveSentenceCardConfig = () => ({
|
||||
sentenceField: 'Sentence',
|
||||
lapisEnabled: false,
|
||||
kikuEnabled: true,
|
||||
kikuFieldGrouping: 'manual',
|
||||
wordCardKind: 'click',
|
||||
});
|
||||
harness.deps.client.notesInfo = async () =>
|
||||
[
|
||||
{
|
||||
noteId: 42,
|
||||
fields: {
|
||||
Expression: { value: 'taberu' },
|
||||
Sentence: { value: '' },
|
||||
IsWordAndSentenceCard: { value: 'x' },
|
||||
IsClickCard: { value: '' },
|
||||
IsSentenceCard: { value: '' },
|
||||
IsAudioCard: { value: '' },
|
||||
},
|
||||
},
|
||||
] satisfies NoteUpdateWorkflowNoteInfo[];
|
||||
|
||||
await harness.workflow.execute(42);
|
||||
|
||||
assert.equal(harness.updates.length, 1);
|
||||
assert.deepEqual(harness.updates[0]?.fields, {
|
||||
Sentence: 'subtitle-text',
|
||||
IsClickCard: 'x',
|
||||
IsWordAndSentenceCard: '',
|
||||
IsSentenceCard: '',
|
||||
IsAudioCard: '',
|
||||
});
|
||||
});
|
||||
|
||||
test('NoteUpdateWorkflow leaves card type flags alone when the word card kind is none', async () => {
|
||||
const harness = createWorkflowHarness();
|
||||
harness.deps.getEffectiveSentenceCardConfig = () => ({
|
||||
sentenceField: 'Sentence',
|
||||
lapisEnabled: false,
|
||||
kikuEnabled: true,
|
||||
kikuFieldGrouping: 'manual',
|
||||
wordCardKind: 'none',
|
||||
});
|
||||
harness.deps.client.notesInfo = async () =>
|
||||
[
|
||||
{
|
||||
noteId: 42,
|
||||
fields: {
|
||||
Expression: { value: 'taberu' },
|
||||
Sentence: { value: '' },
|
||||
IsWordAndSentenceCard: { value: '' },
|
||||
IsSentenceCard: { value: '' },
|
||||
IsAudioCard: { value: '' },
|
||||
},
|
||||
},
|
||||
] satisfies NoteUpdateWorkflowNoteInfo[];
|
||||
|
||||
await harness.workflow.execute(42);
|
||||
|
||||
assert.equal(harness.updates.length, 1);
|
||||
assert.deepEqual(harness.updates[0]?.fields, {
|
||||
Sentence: 'subtitle-text',
|
||||
});
|
||||
});
|
||||
|
||||
test('NoteUpdateWorkflow does not set Kiku card flags when Lapis and Kiku are disabled', async () => {
|
||||
const harness = createWorkflowHarness();
|
||||
harness.deps.client.notesInfo = async () =>
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { DEFAULT_ANKI_CONNECT_CONFIG } from '../config';
|
||||
import { getPreferredWordValueFromExtractedFields } from '../anki-field-config';
|
||||
import type { SubtitleMiningContext } from '../types/subtitle';
|
||||
import { shouldMarkWordAndSentenceCard } from './note-field-utils';
|
||||
import type { CardKind, WordCardKind } from '../types/anki';
|
||||
import { resolveWordCardKind } from './note-field-utils';
|
||||
|
||||
export interface NoteUpdateWorkflowNoteInfo {
|
||||
noteId: number;
|
||||
@@ -39,6 +40,7 @@ export interface NoteUpdateWorkflowDeps {
|
||||
lapisEnabled: boolean;
|
||||
kikuEnabled: boolean;
|
||||
kikuFieldGrouping: 'auto' | 'manual' | 'disabled';
|
||||
wordCardKind?: WordCardKind;
|
||||
};
|
||||
appendKnownWordsFromNoteInfo: (noteInfo: NoteUpdateWorkflowNoteInfo) => void;
|
||||
extractFields: (fields: Record<string, { value: string }>) => Record<string, string>;
|
||||
@@ -67,7 +69,7 @@ export interface NoteUpdateWorkflowDeps {
|
||||
setCardTypeFields: (
|
||||
updatedFields: Record<string, string>,
|
||||
availableFieldNames: string[],
|
||||
cardKind: 'word-and-sentence',
|
||||
cardKind: CardKind,
|
||||
) => void;
|
||||
resolveConfiguredFieldName: (
|
||||
noteInfo: NoteUpdateWorkflowNoteInfo,
|
||||
@@ -207,12 +209,9 @@ export class NoteUpdateWorkflow {
|
||||
if (sentenceField && currentSubtitleText) {
|
||||
const processedSentence = this.deps.processSentence(currentSubtitleText, fields);
|
||||
updatedFields[sentenceField] = processedSentence;
|
||||
if (shouldMarkWordAndSentenceCard(noteInfo, sentenceCardConfig)) {
|
||||
this.deps.setCardTypeFields(
|
||||
updatedFields,
|
||||
Object.keys(noteInfo.fields),
|
||||
'word-and-sentence',
|
||||
);
|
||||
const wordCardKind = resolveWordCardKind(noteInfo, sentenceCardConfig);
|
||||
if (wordCardKind) {
|
||||
this.deps.setCardTypeFields(updatedFields, Object.keys(noteInfo.fields), wordCardKind);
|
||||
}
|
||||
updatePerformed = true;
|
||||
}
|
||||
|
||||
@@ -116,6 +116,10 @@ export function normalizeAnkiIntegrationConfig(config: AnkiConnectConfig): AnkiC
|
||||
...DEFAULT_ANKI_CONNECT_CONFIG.isKiku,
|
||||
...(config.isKiku ?? {}),
|
||||
},
|
||||
lapisKiku: {
|
||||
...DEFAULT_ANKI_CONNECT_CONFIG.lapisKiku,
|
||||
...(config.lapisKiku ?? {}),
|
||||
},
|
||||
} as AnkiConnectConfig;
|
||||
}
|
||||
|
||||
@@ -205,6 +209,10 @@ export class AnkiIntegrationRuntime {
|
||||
patch.isKiku !== undefined
|
||||
? { ...this.config.isKiku, ...patch.isKiku }
|
||||
: this.config.isKiku,
|
||||
lapisKiku:
|
||||
patch.lapisKiku !== undefined
|
||||
? { ...this.config.lapisKiku, ...patch.lapisKiku }
|
||||
: this.config.lapisKiku,
|
||||
};
|
||||
this.config = normalizeAnkiIntegrationConfig(mergedConfig);
|
||||
this.deps.onConfigChanged?.(this.config);
|
||||
|
||||
@@ -399,6 +399,30 @@ test('hasExplicitCommand and shouldStartApp preserve command intent', () => {
|
||||
assert.equal(statsLifetimeRebuild.statsCleanupLifetime, true);
|
||||
assert.equal(statsLifetimeRebuild.statsCleanupVocab, false);
|
||||
|
||||
assert.throws(
|
||||
() =>
|
||||
parseArgs([
|
||||
'--stats',
|
||||
'--stats-cleanup',
|
||||
'--stats-cleanup-duplicate-lines',
|
||||
'--stats-cleanup-lookback-days',
|
||||
'0.5',
|
||||
]),
|
||||
/at least one day/,
|
||||
);
|
||||
assert.equal(
|
||||
parseArgs([
|
||||
'--stats',
|
||||
'--stats-cleanup',
|
||||
'--stats-cleanup-duplicate-lines',
|
||||
'--stats-cleanup-lookback-days',
|
||||
'1.5',
|
||||
]).statsCleanupLookbackDays,
|
||||
1,
|
||||
);
|
||||
assert.equal(parseArgs(['--stats-cleanup-lookback-days=30']).statsCleanupLookbackDays, 30);
|
||||
assert.throws(() => parseArgs(['--stats-cleanup-lookback-days=30=oops']), /at least one day/);
|
||||
|
||||
const jellyfinLibraries = parseArgs(['--jellyfin-libraries']);
|
||||
assert.equal(jellyfinLibraries.jellyfinLibraries, true);
|
||||
assert.equal(hasExplicitCommand(jellyfinLibraries), true);
|
||||
|
||||
+22
-1
@@ -64,6 +64,9 @@ export interface CliArgs {
|
||||
statsCleanup?: boolean;
|
||||
statsCleanupVocab?: boolean;
|
||||
statsCleanupLifetime?: boolean;
|
||||
statsCleanupDuplicateLines?: boolean;
|
||||
statsCleanupDryRun?: boolean;
|
||||
statsCleanupLookbackDays?: number;
|
||||
statsResponsePath?: string;
|
||||
jellyfin: boolean;
|
||||
jellyfinLogin: boolean;
|
||||
@@ -109,6 +112,14 @@ export interface CliArgs {
|
||||
|
||||
export type CliCommandSource = 'initial' | 'second-instance';
|
||||
|
||||
function parseStatsCleanupLookbackDays(value: string | undefined): number {
|
||||
const days = Number(value);
|
||||
if (!Number.isFinite(days) || days < 1) {
|
||||
throw new Error('Stats --lookback-days must be at least one day.');
|
||||
}
|
||||
return Math.floor(days);
|
||||
}
|
||||
|
||||
export function parseArgs(argv: string[]): CliArgs {
|
||||
const args: CliArgs = {
|
||||
background: false,
|
||||
@@ -167,6 +178,8 @@ export function parseArgs(argv: string[]): CliArgs {
|
||||
statsCleanup: false,
|
||||
statsCleanupVocab: false,
|
||||
statsCleanupLifetime: false,
|
||||
statsCleanupDuplicateLines: false,
|
||||
statsCleanupDryRun: false,
|
||||
jellyfin: false,
|
||||
jellyfinLogin: false,
|
||||
jellyfinLogout: false,
|
||||
@@ -368,7 +381,15 @@ export function parseArgs(argv: string[]): CliArgs {
|
||||
} else if (arg === '--stats-cleanup') args.statsCleanup = true;
|
||||
else if (arg === '--stats-cleanup-vocab') args.statsCleanupVocab = true;
|
||||
else if (arg === '--stats-cleanup-lifetime') args.statsCleanupLifetime = true;
|
||||
else if (arg.startsWith('--stats-response-path=')) {
|
||||
else if (arg === '--stats-cleanup-duplicate-lines') args.statsCleanupDuplicateLines = true;
|
||||
else if (arg === '--stats-cleanup-dry-run') args.statsCleanupDryRun = true;
|
||||
else if (arg.startsWith('--stats-cleanup-lookback-days=')) {
|
||||
args.statsCleanupLookbackDays = parseStatsCleanupLookbackDays(
|
||||
arg.slice('--stats-cleanup-lookback-days='.length),
|
||||
);
|
||||
} else if (arg === '--stats-cleanup-lookback-days') {
|
||||
args.statsCleanupLookbackDays = parseStatsCleanupLookbackDays(readValue(argv[i + 1]));
|
||||
} else if (arg.startsWith('--stats-response-path=')) {
|
||||
const value = arg.split('=', 2)[1];
|
||||
if (value) args.statsResponsePath = value;
|
||||
} else if (arg === '--stats-response-path') {
|
||||
|
||||
@@ -2738,6 +2738,43 @@ test('ignores deprecated isLapis sentence-card field overrides', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts a Kiku/Lapis word card kind and warns on an unknown one', () => {
|
||||
const dir = makeTempDir();
|
||||
fs.writeFileSync(
|
||||
path.join(dir, 'config.jsonc'),
|
||||
`{
|
||||
"ankiConnect": {
|
||||
"isKiku": { "enabled": true },
|
||||
"lapisKiku": { "wordCardKind": "click" }
|
||||
}
|
||||
}`,
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const service = new ConfigService(dir);
|
||||
assert.equal(service.getConfig().ankiConnect.lapisKiku.wordCardKind, 'click');
|
||||
assert.equal(service.getWarnings().length, 0);
|
||||
|
||||
const invalidDir = makeTempDir();
|
||||
fs.writeFileSync(
|
||||
path.join(invalidDir, 'config.jsonc'),
|
||||
`{
|
||||
"ankiConnect": {
|
||||
"lapisKiku": { "wordCardKind": "isClickCard" }
|
||||
}
|
||||
}`,
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const invalidService = new ConfigService(invalidDir);
|
||||
assert.equal(invalidService.getConfig().ankiConnect.lapisKiku.wordCardKind, 'word-and-sentence');
|
||||
assert.ok(
|
||||
invalidService
|
||||
.getWarnings()
|
||||
.some((warning) => warning.path === 'ankiConnect.lapisKiku.wordCardKind'),
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts valid ankiConnect knownWords deck object', () => {
|
||||
const dir = makeTempDir();
|
||||
fs.writeFileSync(
|
||||
|
||||
@@ -91,6 +91,9 @@ export const INTEGRATIONS_DEFAULT_CONFIG: Pick<
|
||||
fieldGrouping: 'disabled',
|
||||
deleteDuplicateInAuto: true,
|
||||
},
|
||||
lapisKiku: {
|
||||
wordCardKind: 'word-and-sentence',
|
||||
},
|
||||
},
|
||||
jimaku: {
|
||||
apiBaseUrl: 'https://jimaku.cc',
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { ResolvedConfig } from '../../types/config';
|
||||
import { WORD_CARD_KINDS } from '../../anki-integration/card-kinds';
|
||||
import { MPV_LAUNCH_MODE_VALUES } from '../../shared/mpv-launch-mode';
|
||||
import {
|
||||
NOTIFICATION_TYPE_VALUES,
|
||||
@@ -374,6 +375,21 @@ export function buildIntegrationConfigOptionRegistry(
|
||||
defaultValue: defaultConfig.ankiConnect.isLapis.sentenceCardModel,
|
||||
description: 'Note type name used by Lapis sentence cards.',
|
||||
},
|
||||
{
|
||||
path: 'ankiConnect.lapisKiku.wordCardKind',
|
||||
kind: 'enum',
|
||||
enumValues: WORD_CARD_KINDS,
|
||||
enumLabels: {
|
||||
'word-and-sentence': 'Word and sentence card (IsWordAndSentenceCard)',
|
||||
click: 'Click card (IsClickCard)',
|
||||
sentence: 'Sentence card (IsSentenceCard)',
|
||||
audio: 'Audio card (IsAudioCard)',
|
||||
none: 'Leave card type flags untouched',
|
||||
},
|
||||
defaultValue: defaultConfig.ankiConnect.lapisKiku.wordCardKind,
|
||||
description:
|
||||
'Card-type flag SubMiner marks on Kiku/Lapis word cards. Only one flag is set at a time; the others are cleared. Requires isKiku.enabled or isLapis.enabled.',
|
||||
},
|
||||
{
|
||||
path: 'ankiConnect.metadata.pattern',
|
||||
kind: 'string',
|
||||
|
||||
@@ -135,7 +135,7 @@ const INTEGRATION_TEMPLATE_SECTIONS: ConfigTemplateSection[] = [
|
||||
title: 'AnkiConnect Integration',
|
||||
description: ['Automatic Anki updates and media generation options.'],
|
||||
notes: [
|
||||
'Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, and isKiku.fieldGrouping update live while SubMiner is running.',
|
||||
'Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running.',
|
||||
'Shared AI provider transport settings are read from top-level ai and typically require restart.',
|
||||
'Most other AnkiConnect settings still require restart.',
|
||||
],
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { ResolveContext } from './context';
|
||||
import { initializeAnkiConnectResolution } from './anki-connect/initialize';
|
||||
import { applyAnkiKikuResolution } from './anki-connect/kiku';
|
||||
import { applyAnkiLapisKikuResolution } from './anki-connect/lapis-kiku';
|
||||
import { applyAnkiKnownWordsResolution } from './anki-connect/known-words';
|
||||
import { applyAnkiLegacyResolution } from './anki-connect/legacy';
|
||||
import { applyAnkiModernResolution } from './anki-connect/modern';
|
||||
@@ -22,4 +23,5 @@ export function applyAnkiConnectResolution(context: ResolveContext): void {
|
||||
applyAnkiLegacyResolution(context, ankiConnect, behavior, fields, media, metadata);
|
||||
applyAnkiKnownWordsResolution(context, ankiConnect, behavior);
|
||||
applyAnkiKikuResolution(context);
|
||||
applyAnkiLapisKikuResolution(context, ankiConnect);
|
||||
}
|
||||
|
||||
@@ -77,5 +77,8 @@ export function initializeAnkiConnectResolution(
|
||||
? (ankiConnect.isKiku as (typeof context.resolved)['ankiConnect']['isKiku'])
|
||||
: {}),
|
||||
},
|
||||
lapisKiku: {
|
||||
...context.resolved.ankiConnect.lapisKiku,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { isWordCardKind, WORD_CARD_KINDS } from '../../../anki-integration/card-kinds';
|
||||
import { DEFAULT_CONFIG } from '../../definitions';
|
||||
import type { ResolveContext } from '../context';
|
||||
import { isObject } from '../shared';
|
||||
|
||||
export function applyAnkiLapisKikuResolution(
|
||||
context: ResolveContext,
|
||||
ankiConnect: Record<string, unknown>,
|
||||
): void {
|
||||
if (!isObject(ankiConnect.lapisKiku)) {
|
||||
if (ankiConnect.lapisKiku !== undefined) {
|
||||
context.warn(
|
||||
'ankiConnect.lapisKiku',
|
||||
ankiConnect.lapisKiku,
|
||||
context.resolved.ankiConnect.lapisKiku,
|
||||
'Expected object.',
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const wordCardKind = ankiConnect.lapisKiku.wordCardKind;
|
||||
if (wordCardKind === undefined) {
|
||||
return;
|
||||
}
|
||||
if (isWordCardKind(wordCardKind)) {
|
||||
context.resolved.ankiConnect.lapisKiku.wordCardKind = wordCardKind;
|
||||
return;
|
||||
}
|
||||
|
||||
context.warn(
|
||||
'ankiConnect.lapisKiku.wordCardKind',
|
||||
wordCardKind,
|
||||
DEFAULT_CONFIG.ankiConnect.lapisKiku.wordCardKind,
|
||||
`Expected one of ${WORD_CARD_KINDS.join(', ')}.`,
|
||||
);
|
||||
context.resolved.ankiConnect.lapisKiku.wordCardKind =
|
||||
DEFAULT_CONFIG.ankiConnect.lapisKiku.wordCardKind;
|
||||
}
|
||||
@@ -221,6 +221,7 @@ const LABEL_OVERRIDES: Record<string, string> = {
|
||||
'ankiConnect.nPlusOne.enabled': 'Enabled',
|
||||
'ankiConnect.isLapis.enabled': 'Enable Lapis Features',
|
||||
'ankiConnect.isKiku.enabled': 'Enable Kiku Features',
|
||||
'ankiConnect.lapisKiku.wordCardKind': 'Word Card Type',
|
||||
'stats.toggleKey': 'Toggle Stats Overlay',
|
||||
'shortcuts.openCharacterDictionaryManager': 'Open Character Dictionary Manager',
|
||||
'subtitleSidebar.pauseVideoOnHover': 'Pause Video On Hover - Sidebar',
|
||||
@@ -255,6 +256,8 @@ const DESCRIPTION_OVERRIDES: Record<string, string> = {
|
||||
'Enable Lapis-specific mining behavior and sentence-card model targeting. When Kiku is enabled, Lapis features still work and Kiku-specific features are added on top.',
|
||||
'ankiConnect.isLapis.sentenceCardModel':
|
||||
'Anki note type used for Lapis sentence cards. Select from note types reported by AnkiConnect.',
|
||||
'ankiConnect.lapisKiku.wordCardKind':
|
||||
'Card-type flag marked on mined word cards. Only one flag is set at a time; the others are cleared. Requires Kiku or Lapis to be enabled.',
|
||||
'subtitleStyle.css':
|
||||
'CSS declarations applied to primary subtitles. Includes color, background-color, and all font properties.',
|
||||
'subtitleStyle.secondary.css':
|
||||
@@ -401,7 +404,11 @@ function categoryAndSection(path: string): { category: ConfigSettingsCategory; s
|
||||
if (path.startsWith('ankiConnect.media.')) {
|
||||
return { category: 'mining-anki', section: 'Media Capture' };
|
||||
}
|
||||
if (path.startsWith('ankiConnect.isKiku.') || path.startsWith('ankiConnect.isLapis.')) {
|
||||
if (
|
||||
path.startsWith('ankiConnect.isKiku.') ||
|
||||
path.startsWith('ankiConnect.isLapis.') ||
|
||||
path.startsWith('ankiConnect.lapisKiku.')
|
||||
) {
|
||||
return { category: 'mining-anki', section: 'Kiku/Lapis Features' };
|
||||
}
|
||||
if (path.startsWith('ankiConnect.ai.')) {
|
||||
@@ -702,6 +709,7 @@ function restartBehaviorForPath(path: string): ConfigSettingsRestartBehavior {
|
||||
path === 'ankiConnect.fields.miscInfo' ||
|
||||
path === 'ankiConnect.isLapis.sentenceCardModel' ||
|
||||
path === 'ankiConnect.isKiku.fieldGrouping' ||
|
||||
path === 'ankiConnect.lapisKiku.wordCardKind' ||
|
||||
path === 'mpv.aniskipEnabled' ||
|
||||
path === 'mpv.aniskipButtonKey' ||
|
||||
path === 'stats.toggleKey' ||
|
||||
|
||||
@@ -1032,6 +1032,180 @@ describe('stats server API routes', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('POST /api/stats/maintenance/duplicate-lines forwards the window and dry-run flag', async () => {
|
||||
let seenOptions: unknown = null;
|
||||
const summary = {
|
||||
dryRun: true,
|
||||
lookbackDays: 30,
|
||||
scannedLines: 900,
|
||||
burstGroups: 2,
|
||||
removedLines: 180,
|
||||
removedWordOccurrences: 540,
|
||||
removedKanjiOccurrences: 120,
|
||||
samples: [],
|
||||
};
|
||||
const app = createStatsApp(
|
||||
createMockTracker({
|
||||
cleanupDuplicateSubtitleLines: async (options: unknown) => {
|
||||
seenOptions = options;
|
||||
return summary;
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const res = await app.request('/api/stats/maintenance/duplicate-lines', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ dryRun: true, lookbackDays: 30 }),
|
||||
});
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(await res.json(), summary);
|
||||
assert.deepEqual(seenOptions, { dryRun: true, lookbackDays: 30 });
|
||||
});
|
||||
|
||||
it('POST /api/stats/maintenance/duplicate-lines rejects cross-origin simple requests', async () => {
|
||||
let cleanupCalls = 0;
|
||||
const app = createStatsApp(
|
||||
createMockTracker({
|
||||
cleanupDuplicateSubtitleLines: async () => {
|
||||
cleanupCalls += 1;
|
||||
throw new Error('cleanup must not run');
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const res = await app.request('/api/stats/maintenance/duplicate-lines', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'text/plain',
|
||||
Origin: 'https://attacker.example',
|
||||
},
|
||||
body: JSON.stringify({ dryRun: false, lookbackDays: null }),
|
||||
});
|
||||
|
||||
assert.equal(res.status, 415);
|
||||
assert.equal(cleanupCalls, 0);
|
||||
});
|
||||
|
||||
it('POST /api/stats/maintenance/duplicate-lines rejects a window shorter than a day', async () => {
|
||||
let cleanupCalls = 0;
|
||||
const app = createStatsApp(
|
||||
createMockTracker({
|
||||
cleanupDuplicateSubtitleLines: async () => {
|
||||
cleanupCalls += 1;
|
||||
return {
|
||||
dryRun: true,
|
||||
lookbackDays: null,
|
||||
scannedLines: 0,
|
||||
burstGroups: 0,
|
||||
removedLines: 0,
|
||||
removedWordOccurrences: 0,
|
||||
removedKanjiOccurrences: 0,
|
||||
samples: [],
|
||||
};
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const res = await app.request('/api/stats/maintenance/duplicate-lines', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ dryRun: true, lookbackDays: 0.5 }),
|
||||
});
|
||||
|
||||
assert.equal(res.status, 400);
|
||||
assert.equal(cleanupCalls, 0);
|
||||
});
|
||||
|
||||
it('POST /api/stats/maintenance/duplicate-lines floors a fractional multi-day window', async () => {
|
||||
let seenOptions: unknown = null;
|
||||
const app = createStatsApp(
|
||||
createMockTracker({
|
||||
cleanupDuplicateSubtitleLines: async (options: unknown) => {
|
||||
seenOptions = options;
|
||||
return {
|
||||
dryRun: true,
|
||||
lookbackDays: 1,
|
||||
scannedLines: 0,
|
||||
burstGroups: 0,
|
||||
removedLines: 0,
|
||||
removedWordOccurrences: 0,
|
||||
removedKanjiOccurrences: 0,
|
||||
samples: [],
|
||||
};
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const res = await app.request('/api/stats/maintenance/duplicate-lines', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ dryRun: true, lookbackDays: 1.5 }),
|
||||
});
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(seenOptions, { dryRun: true, lookbackDays: 1 });
|
||||
});
|
||||
|
||||
it('POST /api/stats/maintenance/duplicate-lines accepts an explicit empty object for all history', async () => {
|
||||
let seenOptions: unknown = null;
|
||||
const app = createStatsApp(
|
||||
createMockTracker({
|
||||
cleanupDuplicateSubtitleLines: async (options: unknown) => {
|
||||
seenOptions = options;
|
||||
return {
|
||||
dryRun: false,
|
||||
lookbackDays: null,
|
||||
scannedLines: 0,
|
||||
burstGroups: 0,
|
||||
removedLines: 0,
|
||||
removedWordOccurrences: 0,
|
||||
removedKanjiOccurrences: 0,
|
||||
samples: [],
|
||||
};
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const res = await app.request('/api/stats/maintenance/duplicate-lines', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: '{}',
|
||||
});
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(seenOptions, { dryRun: false, lookbackDays: null });
|
||||
});
|
||||
|
||||
for (const malformed of [
|
||||
{ name: 'a missing body', body: undefined },
|
||||
{ name: 'malformed JSON', body: '{' },
|
||||
{ name: 'JSON null', body: 'null' },
|
||||
{ name: 'a JSON array', body: '[]' },
|
||||
]) {
|
||||
it(`POST /api/stats/maintenance/duplicate-lines rejects ${malformed.name}`, async () => {
|
||||
let cleanupCalls = 0;
|
||||
const app = createStatsApp(
|
||||
createMockTracker({
|
||||
cleanupDuplicateSubtitleLines: async () => {
|
||||
cleanupCalls += 1;
|
||||
throw new Error('cleanup must not run');
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const res = await app.request('/api/stats/maintenance/duplicate-lines', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: malformed.body,
|
||||
});
|
||||
|
||||
assert.equal(res.status, 400);
|
||||
assert.equal(cleanupCalls, 0);
|
||||
});
|
||||
}
|
||||
|
||||
it('PUT /api/stats/excluded-words rejects malformed rows', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
|
||||
@@ -2454,6 +2628,80 @@ Aligned English subtitle
|
||||
});
|
||||
});
|
||||
|
||||
it('POST /api/stats/mine-card marks the configured Kiku word card kind', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
fs.writeFileSync(sourcePath, 'fake media');
|
||||
|
||||
await withFakeAnkiConnect(
|
||||
async (requests, url) => {
|
||||
const app = createStatsApp(createMockTracker(), {
|
||||
addYomitanNote: async () => 777,
|
||||
createMediaGenerator: () => ({
|
||||
generateAudio: async () => null,
|
||||
generateScreenshot: async () => null,
|
||||
generateAnimatedImage: async () => null,
|
||||
}),
|
||||
ankiConnectConfig: {
|
||||
url,
|
||||
deck: 'Mining',
|
||||
fields: {
|
||||
image: 'Picture',
|
||||
sentence: 'Sentence',
|
||||
},
|
||||
media: {
|
||||
generateAudio: false,
|
||||
generateImage: false,
|
||||
},
|
||||
isKiku: {
|
||||
enabled: true,
|
||||
fieldGrouping: 'disabled',
|
||||
deleteDuplicateInAuto: true,
|
||||
},
|
||||
lapisKiku: {
|
||||
wordCardKind: 'click',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const res = await app.request('/api/stats/mine-card?mode=word', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
sourcePath,
|
||||
startMs: 1_000,
|
||||
endMs: 2_000,
|
||||
sentence: '猫を見た',
|
||||
word: '猫',
|
||||
videoTitle: 'Episode 1',
|
||||
}),
|
||||
});
|
||||
|
||||
const body = await res.json();
|
||||
assert.equal(res.status, 200, JSON.stringify(body));
|
||||
|
||||
const updateRequest = requests.find((request) => request.action === 'updateNoteFields');
|
||||
const fields = updateRequest?.params?.note?.fields ?? {};
|
||||
assert.equal(fields.IsClickCard, 'x');
|
||||
assert.equal(fields.IsWordAndSentenceCard, '');
|
||||
assert.equal(fields.IsSentenceCard, '');
|
||||
assert.equal(fields.IsAudioCard, '');
|
||||
},
|
||||
{
|
||||
notesInfoFields: {
|
||||
Expression: { value: '猫' },
|
||||
Sentence: { value: '' },
|
||||
Picture: { value: '' },
|
||||
IsWordAndSentenceCard: { value: '' },
|
||||
IsClickCard: { value: '' },
|
||||
IsSentenceCard: { value: '' },
|
||||
IsAudioCard: { value: '' },
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('POST /api/stats/mine-card writes word mining sentence audio and image together', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
assOverrideSignature,
|
||||
assToPlainText,
|
||||
collectAssOverrideCommands,
|
||||
extractAssOverrideBlocks,
|
||||
hasAssTemporalOverride,
|
||||
isAnimatedAssEffectKind,
|
||||
isAssTemporalCommand,
|
||||
normalizePlainSubtitleText,
|
||||
parseAssEffectField,
|
||||
} from './ass-text';
|
||||
|
||||
test('assToPlainText drops vector drawing runs', () => {
|
||||
assert.equal(
|
||||
assToPlainText(
|
||||
'{\\an5\\pos(730,1042)\\p1\\blur1}m 20 0 b 10 0 0 10 0 20 b 0 31 10 40 20 40 {\\p0}',
|
||||
),
|
||||
'',
|
||||
);
|
||||
});
|
||||
|
||||
test('assToPlainText keeps text around drawing runs on the same event', () => {
|
||||
assert.equal(
|
||||
assToPlainText('{\\p1}m 0 0 l 10 10{\\p0}本文{\\p1}m 5 5 l 6 6{\\p0}続き'),
|
||||
'本文続き',
|
||||
);
|
||||
});
|
||||
|
||||
test('assToPlainText leaves \\pos alone when no drawing mode is active', () => {
|
||||
assert.equal(assToPlainText('{\\pos(960,1068)\\bord3}位置指定'), '位置指定');
|
||||
});
|
||||
|
||||
test('assToPlainText does not read \\pos as a drawing tag', () => {
|
||||
assert.equal(assToPlainText('{\\p1\\pos(1,2)}m 0 0 l 5 5'), '');
|
||||
});
|
||||
|
||||
test('assToPlainText resolves line-break and space escapes', () => {
|
||||
assert.equal(assToPlainText('一行目\\N二行目'), '一行目\n二行目');
|
||||
assert.equal(assToPlainText('一行目\\n二行目'), '一行目\n二行目');
|
||||
assert.equal(assToPlainText('一行目\\N二行目', ' '), '一行目 二行目');
|
||||
assert.equal(assToPlainText('間\\h隔'), '間 隔');
|
||||
});
|
||||
|
||||
test('assToPlainText matches mpv on brace and backslash sequences', () => {
|
||||
// mpv has no `\{` / `\}` / `\\` escapes: the backslashes are literal text and the
|
||||
// braces still open and close an override block.
|
||||
assert.equal(assToPlainText('\\{注\\}'), '\\');
|
||||
assert.equal(assToPlainText('\\\\N'), '\\\n');
|
||||
});
|
||||
|
||||
test('assToPlainText renders an unclosed override block verbatim', () => {
|
||||
// mpv shows the stray brace; guessing where the block ended can eat a whole line.
|
||||
assert.equal(assToPlainText('本文{\\pos(1,2)'), '本文{\\pos(1,2)');
|
||||
});
|
||||
|
||||
test('assToPlainText is idempotent', () => {
|
||||
const samples = [
|
||||
'{\\an5\\p1}m 0 0 l 5 5{\\p0}本文',
|
||||
'\\{注\\}',
|
||||
'\\\\N',
|
||||
'本文{\\pos(1,2)',
|
||||
'一行目\\N二行目\\h終わり',
|
||||
];
|
||||
|
||||
for (const sample of samples) {
|
||||
const once = assToPlainText(sample);
|
||||
assert.equal(assToPlainText(once), once, sample);
|
||||
}
|
||||
});
|
||||
|
||||
test('assToPlainText normalizes CRLF before converting', () => {
|
||||
assert.equal(assToPlainText('一行目\r\n二行目'), '一行目\n二行目');
|
||||
});
|
||||
|
||||
test('normalizePlainSubtitleText settles whitespace without decoding ASS', () => {
|
||||
// A brace reaching this layer is literal text mpv chose to show, not markup.
|
||||
assert.equal(normalizePlainSubtitleText('本文{\\pos(1,2)'), '本文{\\pos(1,2)');
|
||||
assert.equal(normalizePlainSubtitleText('一行目\\N二行目'), '一行目\n二行目');
|
||||
assert.equal(
|
||||
normalizePlainSubtitleText('一行目\\N二行目', { collapseLineBreaks: true }),
|
||||
'一行目 二行目',
|
||||
);
|
||||
assert.equal(normalizePlainSubtitleText(' 余白 ', { trim: false }), ' 余白 ');
|
||||
});
|
||||
|
||||
test('normalizePlainSubtitleText is idempotent', () => {
|
||||
for (const sample of ['一行目\\N二行目', '間\\h隔', '本文{\\pos(1,2)', ' 余白 ']) {
|
||||
const once = normalizePlainSubtitleText(sample);
|
||||
assert.equal(normalizePlainSubtitleText(once), once, sample);
|
||||
}
|
||||
});
|
||||
|
||||
test('extractAssOverrideBlocks returns block contents', () => {
|
||||
assert.deepEqual(extractAssOverrideBlocks('{\\an8}上{\\fad(200,200)}下'), [
|
||||
'\\an8',
|
||||
'\\fad(200,200)',
|
||||
]);
|
||||
assert.deepEqual(extractAssOverrideBlocks('括弧なし'), []);
|
||||
});
|
||||
|
||||
test('collectAssOverrideCommands captures names and arguments from blocks only', () => {
|
||||
const commands = collectAssOverrideCommands('{\\pos(1,2)\\1c&HFFFFFF&\\kf30}歌詞');
|
||||
|
||||
assert.deepEqual(commands, [
|
||||
{ name: 'pos', args: '1,2', animated: false },
|
||||
{ name: '1c', args: '&HFFFFFF&', animated: false },
|
||||
{ name: 'kf', args: '30', animated: false },
|
||||
]);
|
||||
|
||||
// A `\pos(...)` sitting in visible text is not typesetting markup.
|
||||
assert.deepEqual(collectAssOverrideCommands('\\pos(730,1042) と書いてある'), []);
|
||||
});
|
||||
|
||||
test('collectAssOverrideCommands marks tags animated by a wrapping \\t', () => {
|
||||
const commands = collectAssOverrideCommands('{\\clip(0,0,10,10)\\t(0,500,\\frz30)}文字');
|
||||
|
||||
assert.deepEqual(
|
||||
commands.map((command) => [command.name, command.animated]),
|
||||
[
|
||||
['clip', false],
|
||||
['t', false],
|
||||
['frz', true],
|
||||
],
|
||||
);
|
||||
assert.equal(hasAssTemporalOverride(commands), true);
|
||||
});
|
||||
|
||||
test('collectAssOverrideCommands stops descending into deeply nested \\t tags', () => {
|
||||
// Nested far past the recursion cap. Uncapped, this recurses once per level, and a
|
||||
// pathological line (real files reach one or two levels) overflows the stack.
|
||||
const nesting = 32;
|
||||
const block = `{${'\\t(0,500,'.repeat(nesting)}\\frz30${')'.repeat(nesting)}}文字`;
|
||||
|
||||
const commands = collectAssOverrideCommands(block);
|
||||
|
||||
// The outer `\t` plus one per allowed recursion level, and nothing from below the cap.
|
||||
assert.equal(commands.length, 9);
|
||||
assert.deepEqual(new Set(commands.map((command) => command.name)), new Set(['t']));
|
||||
assert.equal(hasAssTemporalOverride(commands), true);
|
||||
});
|
||||
|
||||
test('hasAssTemporalOverride ignores static placement and shape tags', () => {
|
||||
assert.equal(
|
||||
hasAssTemporalOverride(collectAssOverrideCommands('{\\pos(1,2)\\clip(m 1 1)\\blur2}文字')),
|
||||
false,
|
||||
);
|
||||
assert.equal(hasAssTemporalOverride(collectAssOverrideCommands('{\\move(1,2,3,4)}文字')), true);
|
||||
});
|
||||
|
||||
test('isAssTemporalCommand covers only intrinsically animated tags', () => {
|
||||
for (const command of ['t', 'move', 'k', 'kf', 'ko', 'K']) {
|
||||
assert.equal(isAssTemporalCommand(command), true, command);
|
||||
}
|
||||
for (const command of ['clip', 'iclip', 'frz', 'fscx', 'blur', 'be', 'pos', 'fad']) {
|
||||
assert.equal(isAssTemporalCommand(command), false, command);
|
||||
}
|
||||
});
|
||||
|
||||
test('assOverrideSignature distinguishes events by their override values', () => {
|
||||
const first = assOverrideSignature(collectAssOverrideCommands('{\\clip(m 1 1)}歌詞'));
|
||||
const second = assOverrideSignature(collectAssOverrideCommands('{\\clip(m 2 2)}歌詞'));
|
||||
const repeat = assOverrideSignature(collectAssOverrideCommands('{\\clip(m 1 1)}別の行'));
|
||||
|
||||
assert.notEqual(first, second);
|
||||
assert.equal(first, repeat);
|
||||
});
|
||||
|
||||
test('parseAssEffectField classifies the event-level Effect column', () => {
|
||||
assert.equal(parseAssEffectField(''), 'none');
|
||||
assert.equal(parseAssEffectField(' '), 'none');
|
||||
assert.equal(parseAssEffectField('Banner;20;1;0'), 'banner');
|
||||
assert.equal(parseAssEffectField('Scroll up;0;0;30;10'), 'scroll');
|
||||
assert.equal(parseAssEffectField('Scroll down;0;0;30;10'), 'scroll');
|
||||
assert.equal(parseAssEffectField('Karaoke'), 'karaoke');
|
||||
assert.equal(parseAssEffectField('fx-template'), 'other');
|
||||
});
|
||||
|
||||
test('parseAssEffectField matches stock effect names exactly', () => {
|
||||
// Custom effect names that merely start with a stock name are not stock effects.
|
||||
assert.equal(parseAssEffectField('scrolling-credit'), 'other');
|
||||
assert.equal(parseAssEffectField('bannerfx;1'), 'other');
|
||||
assert.equal(parseAssEffectField('karaoke-template'), 'other');
|
||||
assert.equal(parseAssEffectField('Scroll'), 'other');
|
||||
});
|
||||
|
||||
test('isAnimatedAssEffectKind covers the stock animated effects only', () => {
|
||||
assert.equal(isAnimatedAssEffectKind('karaoke'), true);
|
||||
assert.equal(isAnimatedAssEffectKind('banner'), true);
|
||||
assert.equal(isAnimatedAssEffectKind('scroll'), true);
|
||||
// Typesetting groups put static template names in this column too.
|
||||
assert.equal(isAnimatedAssEffectKind('other'), false);
|
||||
assert.equal(isAnimatedAssEffectKind('none'), false);
|
||||
});
|
||||
@@ -0,0 +1,280 @@
|
||||
/*
|
||||
* ASS/SSA text handling, split into two deliberately distinct contracts:
|
||||
*
|
||||
* assToPlainText() raw ASS event text -> plain text. Ingestion only.
|
||||
* normalizePlainSubtitleText() already-decoded text -> display/lookup form.
|
||||
*
|
||||
* Subtitle text is decoded from ASS exactly once, at the point it enters the app: the
|
||||
* file cue parser does it for sidecar/embedded scripts, and mpv does it for live text
|
||||
* (`sub-text` is already run through mpv's own `ass_to_plaintext`). Everything
|
||||
* downstream -- renderer, timing tracker, tokenizer, tokenization cache keys -- gets
|
||||
* plain text and only normalizes whitespace, so no layer decodes the same string twice.
|
||||
*
|
||||
* assToPlainText mirrors mpv's `ass_to_plaintext` rather than inventing its own rules,
|
||||
* so a cue parsed from a file reads the same as the same line arriving live:
|
||||
* - `{...}` override blocks are markup
|
||||
* - `\pN ... \p0` runs are vector paths, not dialogue
|
||||
* - `\N`, `\n` and `\h` are the only escapes; `\{`, `\}` and `\\` are NOT escapes,
|
||||
* so `\{注\}` decodes to a lone backslash exactly as mpv renders it
|
||||
* - an unclosed `{` is rendered verbatim instead of swallowing the rest of the line
|
||||
* Because the decoder never emits an escape or a closed brace, running it twice is a
|
||||
* no-op -- but downstream code should still use normalizePlainSubtitleText.
|
||||
*/
|
||||
|
||||
/** What `\N` and `\n` become. */
|
||||
export type AssLineBreak = '\n' | ' ';
|
||||
|
||||
// `\p<n>` with n > 0 switches libass into vector-drawing mode: everything until the
|
||||
// next `\p0` is a path (`m 20 0 b 10 0 ...`), not dialogue. The negative lookahead keeps
|
||||
// `\pos(...)` from being read as a drawing tag.
|
||||
const ASS_DRAWING_SCALE_PATTERN = /\\p(?![a-zA-Z])(\d*)/g;
|
||||
|
||||
function readDrawingScale(block: string): number | null {
|
||||
ASS_DRAWING_SCALE_PATTERN.lastIndex = 0;
|
||||
let scale: number | null = null;
|
||||
let match: RegExpExecArray | null;
|
||||
// Drawing mode is whatever the last `\p` tag in this block set it to.
|
||||
while ((match = ASS_DRAWING_SCALE_PATTERN.exec(block)) !== null) {
|
||||
scale = match[1] ? Number(match[1]) : 0;
|
||||
}
|
||||
return scale;
|
||||
}
|
||||
|
||||
/** Resolve `\N`, `\n` and `\h`. The only text-level escapes libass recognises. */
|
||||
function resolveWhitespaceEscapes(text: string, lineBreak: AssLineBreak): string {
|
||||
return text.replace(/\\([Nnh])/g, (_match, escaped: string) =>
|
||||
escaped === 'h' ? ' ' : lineBreak,
|
||||
);
|
||||
}
|
||||
|
||||
/** Strip `{...}` override blocks and the drawing runs they enable. */
|
||||
function stripAssMarkup(raw: string): string {
|
||||
let out = '';
|
||||
let cursor = 0;
|
||||
let drawing = false;
|
||||
|
||||
while (cursor < raw.length) {
|
||||
if (raw[cursor] !== '{') {
|
||||
if (!drawing) {
|
||||
out += raw[cursor];
|
||||
}
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const close = raw.indexOf('}', cursor + 1);
|
||||
if (close === -1) {
|
||||
// mpv shows an unclosed `{` and everything after it. Guessing where the block was
|
||||
// meant to end can eat a whole line of dialogue.
|
||||
if (!drawing) {
|
||||
out += raw.slice(cursor);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
const scale = readDrawingScale(raw.slice(cursor, close + 1));
|
||||
if (scale !== null) {
|
||||
drawing = scale > 0;
|
||||
}
|
||||
cursor = close + 1;
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a raw ASS/SSA event text field. Call this once, where the text enters the app;
|
||||
* downstream layers take the result as plain text.
|
||||
*/
|
||||
export function assToPlainText(text: string, lineBreak: AssLineBreak = '\n'): string {
|
||||
if (!text) return '';
|
||||
return resolveWhitespaceEscapes(stripAssMarkup(text.replace(/\r\n/g, '\n')), lineBreak);
|
||||
}
|
||||
|
||||
export interface NormalizePlainSubtitleTextOptions {
|
||||
/** Fold every line break into a single space. */
|
||||
collapseLineBreaks?: boolean;
|
||||
trim?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whitespace normalization for text that has already been decoded -- by mpv for live
|
||||
* subtitles, by the cue parser for files. Override blocks and drawing runs are none of
|
||||
* this function's business; a `{` that reaches here is literal text mpv chose to show.
|
||||
*
|
||||
* `\N`/`\n`/`\h` are still folded, because subtitle sources outside the ASS path (asbplayer
|
||||
* and other websocket clients) forward them raw and the display layer has to cope.
|
||||
*/
|
||||
export function normalizePlainSubtitleText(
|
||||
text: string,
|
||||
options: NormalizePlainSubtitleTextOptions = {},
|
||||
): string {
|
||||
if (!text) return '';
|
||||
const { collapseLineBreaks = false, trim = true } = options;
|
||||
|
||||
let normalized = resolveWhitespaceEscapes(
|
||||
text.replace(/\r\n/g, '\n'),
|
||||
collapseLineBreaks ? ' ' : '\n',
|
||||
);
|
||||
if (collapseLineBreaks) {
|
||||
normalized = normalized.replace(/\n/g, ' ').replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
return trim ? normalized.trim() : normalized;
|
||||
}
|
||||
|
||||
/** The contents of each `{...}` block, without the braces. */
|
||||
export function extractAssOverrideBlocks(text: string): string[] {
|
||||
const blocks: string[] = [];
|
||||
let cursor = 0;
|
||||
|
||||
while (cursor < text.length) {
|
||||
const open = text.indexOf('{', cursor);
|
||||
if (open === -1) {
|
||||
break;
|
||||
}
|
||||
const close = text.indexOf('}', open + 1);
|
||||
if (close === -1) {
|
||||
break;
|
||||
}
|
||||
blocks.push(text.slice(open + 1, close));
|
||||
cursor = close + 1;
|
||||
}
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
export interface AssOverrideCommand {
|
||||
/** Tag name without the backslash, e.g. `pos`, `kf`, `1c`. */
|
||||
name: string;
|
||||
/** Everything the tag was given, e.g. `960,1068` for `\pos(960,1068)`. */
|
||||
args: string;
|
||||
/** Nested inside a `\t(...)` argument, so its value is animated over the event. */
|
||||
animated: boolean;
|
||||
}
|
||||
|
||||
const ASS_OVERRIDE_NAME_PATTERN = /[1-4]?[a-zA-Z]+/y;
|
||||
|
||||
function readCommandArgs(block: string, start: number): { args: string; next: number } {
|
||||
if (block[start] === '(') {
|
||||
let depth = 0;
|
||||
for (let i = start; i < block.length; i += 1) {
|
||||
if (block[i] === '(') depth += 1;
|
||||
else if (block[i] === ')') {
|
||||
depth -= 1;
|
||||
if (depth === 0) {
|
||||
return { args: block.slice(start + 1, i), next: i + 1 };
|
||||
}
|
||||
}
|
||||
}
|
||||
return { args: block.slice(start + 1), next: block.length };
|
||||
}
|
||||
|
||||
const nextTag = block.indexOf('\\', start);
|
||||
const end = nextTag === -1 ? block.length : nextTag;
|
||||
return { args: block.slice(start, end), next: end };
|
||||
}
|
||||
|
||||
// `\t(...)` can wrap another `\t(...)`, and nothing in the format stops an author (or a
|
||||
// malformed file) from nesting them thousands deep. Real typesetting never goes past one
|
||||
// or two levels, so stop recursing well before the call stack is at risk.
|
||||
const MAX_ANIMATION_NESTING_DEPTH = 8;
|
||||
|
||||
function parseOverrideBlock(
|
||||
block: string,
|
||||
animated: boolean,
|
||||
into: AssOverrideCommand[],
|
||||
depth = 0,
|
||||
): void {
|
||||
let cursor = 0;
|
||||
|
||||
while (cursor < block.length) {
|
||||
if (block[cursor] !== '\\') {
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
ASS_OVERRIDE_NAME_PATTERN.lastIndex = cursor + 1;
|
||||
const nameMatch = ASS_OVERRIDE_NAME_PATTERN.exec(block);
|
||||
if (!nameMatch) {
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const name = nameMatch[0];
|
||||
const { args, next } = readCommandArgs(block, cursor + 1 + name.length);
|
||||
into.push({ name, args: args.trim(), animated });
|
||||
// `\t(0,500,\frz30)` animates whatever it wraps, so record the inner tags too.
|
||||
if (name === 't' && args.includes('\\') && depth < MAX_ANIMATION_NESTING_DEPTH) {
|
||||
parseOverrideBlock(args, true, into, depth + 1);
|
||||
}
|
||||
cursor = next;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Override commands with their arguments, in source order. Only `{...}` blocks are
|
||||
* inspected, so a `\pos(...)` sitting in visible text is never mistaken for markup.
|
||||
*/
|
||||
export function collectAssOverrideCommands(text: string): AssOverrideCommand[] {
|
||||
const commands: AssOverrideCommand[] = [];
|
||||
for (const block of extractAssOverrideBlocks(text)) {
|
||||
parseOverrideBlock(block, false, commands);
|
||||
}
|
||||
return commands;
|
||||
}
|
||||
|
||||
// Tags that are animated by definition: `\t` interpolates, `\move` travels, and the
|
||||
// karaoke tags advance a highlight across the event's own duration. Everything else --
|
||||
// `\pos`, `\clip`, `\frz`, `\blur`, `\fad` -- is a static value for the event, so its
|
||||
// presence says nothing about whether neighbouring events form one animation.
|
||||
const ASS_TEMPORAL_COMMANDS = new Set(['t', 'move', 'k', 'kf', 'ko', 'K']);
|
||||
|
||||
export function isAssTemporalCommand(name: string): boolean {
|
||||
return ASS_TEMPORAL_COMMANDS.has(name);
|
||||
}
|
||||
|
||||
/** True when the event animates on its own, or animates a static tag through `\t(...)`. */
|
||||
export function hasAssTemporalOverride(commands: readonly AssOverrideCommand[]): boolean {
|
||||
return commands.some((command) => command.animated || isAssTemporalCommand(command.name));
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical form of an event's override values, for comparing consecutive events. Two
|
||||
* events with the same signature were typeset identically, so neither is a frame of an
|
||||
* animation the other belongs to.
|
||||
*/
|
||||
export function assOverrideSignature(commands: readonly AssOverrideCommand[]): string {
|
||||
return commands.map((command) => `${command.name}(${command.args})`).join('|');
|
||||
}
|
||||
|
||||
export type AssEffectKind = 'none' | 'banner' | 'scroll' | 'karaoke' | 'other';
|
||||
|
||||
// The stock effects, matched exactly. Typesetting groups put their own template names in
|
||||
// this column -- `scrolling-credit` is a static sign, not libass's `Scroll up` -- so a
|
||||
// prefix match would hand out animation evidence to arbitrary custom effects.
|
||||
const STOCK_ASS_EFFECTS = new Map<string, AssEffectKind>([
|
||||
['banner', 'banner'],
|
||||
['scroll up', 'scroll'],
|
||||
['scroll down', 'scroll'],
|
||||
['karaoke', 'karaoke'],
|
||||
]);
|
||||
|
||||
/**
|
||||
* The event-level `Effect` column. The stock values (`Banner;...`, `Scroll up;...`,
|
||||
* `Scroll down;...`, `Karaoke`) all animate; anything else is a custom name and lands in
|
||||
* `other`.
|
||||
*/
|
||||
export function parseAssEffectField(raw: string): AssEffectKind {
|
||||
const value = raw.trim().toLowerCase();
|
||||
if (!value) return 'none';
|
||||
|
||||
const name = value.split(';', 1)[0]!.trim();
|
||||
return STOCK_ASS_EFFECTS.get(name) ?? 'other';
|
||||
}
|
||||
|
||||
const ANIMATED_ASS_EFFECT_KINDS = new Set<AssEffectKind>(['banner', 'scroll', 'karaoke']);
|
||||
|
||||
export function isAnimatedAssEffectKind(kind: AssEffectKind): boolean {
|
||||
return ANIMATED_ASS_EFFECT_KINDS.has(kind);
|
||||
}
|
||||
@@ -85,6 +85,7 @@ const HOT_RELOAD_EXACT_OR_PREFIX_PATHS = [
|
||||
'ankiConnect.fields.miscInfo',
|
||||
'ankiConnect.isLapis.sentenceCardModel',
|
||||
'ankiConnect.isKiku.fieldGrouping',
|
||||
'ankiConnect.lapisKiku.wordCardKind',
|
||||
] as const;
|
||||
|
||||
function hotReloadFieldForChangedPath(path: string): string | null {
|
||||
|
||||
@@ -1414,6 +1414,353 @@ test('deleteSession ignores the currently active session and keeps new writes fl
|
||||
}
|
||||
});
|
||||
|
||||
test('deleteSession yields the main event loop while delete maintenance is pending', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
const deleteGate: { release?: () => void } = {};
|
||||
let deleteRunnerCalled = false;
|
||||
let bufferedWritesAtDeleteStart = -1;
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
const createdTracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runDeleteMaintenanceTask: async () => {
|
||||
deleteRunnerCalled = true;
|
||||
bufferedWritesAtDeleteStart = (tracker as unknown as { queue: unknown[] }).queue.length;
|
||||
await new Promise<void>((resolve) => {
|
||||
deleteGate.release = resolve;
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
tracker = createdTracker;
|
||||
createdTracker.handleMediaChange('/tmp/delete-yield-first.mkv', 'Delete Yield First');
|
||||
createdTracker.handleMediaChange('/tmp/delete-yield-active.mkv', 'Delete Yield Active');
|
||||
|
||||
const privateApi = createdTracker as unknown as {
|
||||
db: DatabaseSync;
|
||||
queue: unknown[];
|
||||
flushNow: () => void;
|
||||
};
|
||||
const sessionId = (
|
||||
privateApi.db
|
||||
.prepare(
|
||||
`SELECT session_id AS sessionId
|
||||
FROM imm_sessions
|
||||
WHERE ended_at_ms IS NOT NULL
|
||||
ORDER BY session_id
|
||||
LIMIT 1`,
|
||||
)
|
||||
.get() as { sessionId: number } | null
|
||||
)?.sessionId;
|
||||
assert.ok(sessionId);
|
||||
|
||||
const deletePromise = createdTracker.deleteSession(sessionId);
|
||||
let timerAdvanced = false;
|
||||
setTimeout(() => {
|
||||
timerAdvanced = true;
|
||||
}, 0);
|
||||
|
||||
await waitForCondition(() => deleteRunnerCalled);
|
||||
assert.equal(deleteRunnerCalled, true, 'delete should be dispatched to the maintenance runner');
|
||||
assert.equal(
|
||||
bufferedWritesAtDeleteStart,
|
||||
0,
|
||||
'writes buffered before delete should flush first',
|
||||
);
|
||||
await waitForCondition(() => timerAdvanced);
|
||||
|
||||
createdTracker.recordSubtitleLine('queued during delete', 0, 1);
|
||||
privateApi.flushNow();
|
||||
assert.ok(privateApi.queue.length > 0, 'tracking writes should wait for delete maintenance');
|
||||
|
||||
assert.ok(deleteGate.release);
|
||||
deleteGate.release();
|
||||
await deletePromise;
|
||||
} finally {
|
||||
deleteGate.release?.();
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('delete maintenance flushes the entire write queue before locking writes', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
const deleteGate: { release?: () => void } = {};
|
||||
let queuedWritesAtDeleteStart = -1;
|
||||
let writeLockedAtDeleteStart = false;
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runDeleteMaintenanceTask: async () => {
|
||||
const privateApi = tracker as unknown as {
|
||||
queue: unknown[];
|
||||
writeLock: { locked: boolean };
|
||||
};
|
||||
queuedWritesAtDeleteStart = privateApi.queue.length;
|
||||
writeLockedAtDeleteStart = privateApi.writeLock.locked;
|
||||
await new Promise<void>((resolve) => {
|
||||
deleteGate.release = resolve;
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const privateApi = tracker as unknown as {
|
||||
batchSize: number;
|
||||
flushNow: () => void;
|
||||
queue: unknown[];
|
||||
};
|
||||
privateApi.batchSize = 1;
|
||||
privateApi.queue.push({}, {}, {});
|
||||
privateApi.flushNow = () => {
|
||||
privateApi.queue.shift();
|
||||
};
|
||||
|
||||
const deletePromise = tracker.deleteSession(101);
|
||||
await waitForCondition(() => deleteGate.release !== undefined);
|
||||
|
||||
assert.equal(queuedWritesAtDeleteStart, 0);
|
||||
assert.equal(writeLockedAtDeleteStart, true);
|
||||
|
||||
deleteGate.release?.();
|
||||
await deletePromise;
|
||||
} finally {
|
||||
deleteGate.release?.();
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('delete maintenance tasks stay serialized under concurrent requests', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
const releases: Array<() => void> = [];
|
||||
let activeTasks = 0;
|
||||
let maxActiveTasks = 0;
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runDeleteMaintenanceTask: async () => {
|
||||
activeTasks += 1;
|
||||
maxActiveTasks = Math.max(maxActiveTasks, activeTasks);
|
||||
await new Promise<void>((resolve) => {
|
||||
releases.push(resolve);
|
||||
});
|
||||
activeTasks -= 1;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const firstDelete = tracker.deleteSession(101);
|
||||
await waitForCondition(() => releases.length === 1);
|
||||
assert.equal(maxActiveTasks, 1);
|
||||
|
||||
const secondDelete = tracker.deleteSession(102);
|
||||
|
||||
releases[0]?.();
|
||||
await waitForCondition(() => releases.length === 2);
|
||||
assert.equal(maxActiveTasks, 1);
|
||||
|
||||
releases[1]?.();
|
||||
await Promise.all([firstDelete, secondDelete]);
|
||||
} finally {
|
||||
for (const release of releases) release();
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('concurrent delete requests share one maintenance worker batch', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
const tasks: unknown[] = [];
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runDeleteMaintenanceTask: async (_path, task) => {
|
||||
tasks.push(task);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const firstDelete = tracker.deleteSession(201);
|
||||
const secondDelete = tracker.deleteSessions([202, 203]);
|
||||
const thirdDelete = tracker.deleteVideo(204);
|
||||
await Promise.all([firstDelete, secondDelete, thirdDelete]);
|
||||
|
||||
assert.equal(tasks.length, 1, 'concurrent deletes should use one maintenance pass');
|
||||
assert.deepEqual(tasks[0], {
|
||||
kind: 'batch',
|
||||
tasks: [
|
||||
{ kind: 'session', sessionId: 201 },
|
||||
{ kind: 'sessions', sessionIds: [202, 203] },
|
||||
{ kind: 'video', videoId: 204 },
|
||||
],
|
||||
});
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('destroy rejects delete requests waiting behind active maintenance', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let releaseFirstTask: () => void = () => {};
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
let markFirstTaskStarted: () => void = () => {};
|
||||
const firstTaskStarted = new Promise<void>((resolve) => {
|
||||
markFirstTaskStarted = resolve;
|
||||
});
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runDeleteMaintenanceTask: async () => {
|
||||
markFirstTaskStarted();
|
||||
await new Promise<void>((resolve) => {
|
||||
releaseFirstTask = resolve;
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const firstDelete = tracker.deleteSession(301);
|
||||
await firstTaskStarted;
|
||||
const queuedDelete = tracker.deleteSession(302);
|
||||
tracker.destroy();
|
||||
|
||||
const queuedOutcome = await Promise.race([
|
||||
queuedDelete.then(
|
||||
() => 'resolved',
|
||||
(error: unknown) =>
|
||||
error instanceof Error && /shutting down/.test(error.message)
|
||||
? 'rejected'
|
||||
: 'wrong-error',
|
||||
),
|
||||
new Promise<'pending'>((resolve) => setTimeout(() => resolve('pending'), 25)),
|
||||
]);
|
||||
assert.equal(queuedOutcome, 'rejected');
|
||||
releaseFirstTask();
|
||||
await firstDelete;
|
||||
} finally {
|
||||
releaseFirstTask();
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('delete requested after destroy rejects without running maintenance', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let maintenanceCalls = 0;
|
||||
const Ctor = await loadTrackerCtor();
|
||||
const tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runDeleteMaintenanceTask: async () => {
|
||||
maintenanceCalls += 1;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
tracker.destroy();
|
||||
|
||||
await assert.rejects(tracker.deleteSession(303), /shutting down/);
|
||||
assert.equal(maintenanceCalls, 0);
|
||||
cleanupDbPath(dbPath);
|
||||
});
|
||||
|
||||
test('deleteSessions skips maintenance when no sessions are deletable', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
const tasks: unknown[] = [];
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runDeleteMaintenanceTask: async (_path, task) => {
|
||||
tasks.push(task);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
await tracker.deleteSessions([]);
|
||||
|
||||
assert.deepEqual(tasks, []);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('queued video delete is skipped when that video becomes active before dispatch', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
const tasks: Array<{ kind: string }> = [];
|
||||
let releaseFirstTask: () => void = () => {};
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
const createdTracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runDeleteMaintenanceTask: async (_path, task) => {
|
||||
tasks.push(task);
|
||||
if (tasks.length === 1) {
|
||||
await new Promise<void>((resolve) => {
|
||||
releaseFirstTask = resolve;
|
||||
});
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
tracker = createdTracker;
|
||||
createdTracker.handleMediaChange('/tmp/delete-race-target.mkv', 'Delete Race Target');
|
||||
createdTracker.handleMediaChange('/tmp/delete-race-other.mkv', 'Delete Race Other');
|
||||
|
||||
const privateApi = createdTracker as unknown as { db: DatabaseSync };
|
||||
const targetVideoId = (
|
||||
privateApi.db
|
||||
.prepare(`SELECT video_id AS videoId FROM imm_videos WHERE video_key LIKE '%target.mkv'`)
|
||||
.get() as { videoId: number } | null
|
||||
)?.videoId;
|
||||
assert.ok(targetVideoId);
|
||||
|
||||
const firstDelete = createdTracker.deleteSession(999_001);
|
||||
await waitForCondition(() => tasks.length === 1);
|
||||
|
||||
const queuedVideoDelete = createdTracker.deleteVideo(targetVideoId);
|
||||
createdTracker.handleMediaChange('/tmp/delete-race-target.mkv', 'Delete Race Target');
|
||||
releaseFirstTask();
|
||||
await Promise.all([firstDelete, queuedVideoDelete]);
|
||||
|
||||
assert.deepEqual(
|
||||
tasks.map((task) => task.kind),
|
||||
['session'],
|
||||
);
|
||||
} finally {
|
||||
releaseFirstTask();
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('deleteVideo ignores the currently active video and keeps new writes flushable', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
@@ -3459,6 +3806,8 @@ test('reassignAnimeAnilist redistributes conflicting legacy combined row before
|
||||
(3, 6000, 3000, 3000, 3, 30, 0, 0, 0, 0, 0, 0, 0, 0);
|
||||
`);
|
||||
|
||||
await tracker.rebuildLifetimeSummaries();
|
||||
|
||||
await tracker.reassignAnimeAnilist(2, {
|
||||
anilistId: 21202,
|
||||
titleRomaji: 'Kono Subarashii Sekai ni Shukufuku wo!',
|
||||
|
||||
@@ -30,6 +30,9 @@ import {
|
||||
applySessionLifetimeSummary,
|
||||
reconcileStaleActiveSessions,
|
||||
rebuildLifetimeSummaries as rebuildLifetimeSummaryTables,
|
||||
recomputeLifetimeAnimeFromMedia,
|
||||
recomputeLifetimeGlobalFromSummaries,
|
||||
repairLifetimeSummariesFromMedia,
|
||||
shouldBackfillLifetimeSummaries,
|
||||
} from './immersion-tracker/lifetime';
|
||||
import {
|
||||
@@ -83,14 +86,20 @@ import {
|
||||
} from './immersion-tracker/query-library';
|
||||
import {
|
||||
cleanupVocabularyStats,
|
||||
deleteAnime as deleteAnimeQuery,
|
||||
deleteSession as deleteSessionQuery,
|
||||
deleteSessions as deleteSessionsQuery,
|
||||
deleteVideo as deleteVideoQuery,
|
||||
getVideoDurationMs,
|
||||
markVideoWatched,
|
||||
upsertCoverArt,
|
||||
} from './immersion-tracker/query-maintenance';
|
||||
import {
|
||||
DeleteMaintenanceWorkerRuntime,
|
||||
type RunDeleteMaintenanceTask,
|
||||
} from './immersion-tracker/delete-maintenance-worker-runtime';
|
||||
import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
|
||||
import {
|
||||
cleanupDuplicateSubtitleLines,
|
||||
type DuplicateSubtitleLineCleanupOptions,
|
||||
type DuplicateSubtitleLineCleanupSummary,
|
||||
} from './immersion-tracker/duplicate-line-cleanup';
|
||||
import { repairJellyfinStreamVideoLinks } from './immersion-tracker/jellyfin-link-repair';
|
||||
import {
|
||||
repairLegacySeasonlessAnimeRows,
|
||||
@@ -182,6 +191,7 @@ const YOUTUBE_SCREENSHOT_MAX_SECONDS = 120;
|
||||
const YOUTUBE_OEMBED_ENDPOINT = 'https://www.youtube.com/oembed';
|
||||
const YOUTUBE_ID_PATTERN = /^[A-Za-z0-9_-]{6,}$/;
|
||||
const YOUTUBE_METADATA_REFRESH_MS = 24 * 60 * 60 * 1000;
|
||||
const DELETE_MAINTENANCE_BATCH_WINDOW_MS = 10;
|
||||
|
||||
function isValidYouTubeVideoId(value: string | null): boolean {
|
||||
return Boolean(value && YOUTUBE_ID_PATTERN.test(value));
|
||||
@@ -385,6 +395,8 @@ export class ImmersionTrackerService {
|
||||
private readonly vacuumIntervalMs: number;
|
||||
private readonly dbPath: string;
|
||||
private readonly writeLock = { locked: false };
|
||||
private readonly destroyDeleteMaintenanceRunner: () => void;
|
||||
private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
|
||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private flushScheduled = false;
|
||||
@@ -406,9 +418,38 @@ export class ImmersionTrackerService {
|
||||
| ((row: LegacyVocabularyPosRow) => Promise<LegacyVocabularyPosResolution | null>)
|
||||
| undefined;
|
||||
|
||||
constructor(options: ImmersionTrackerOptions) {
|
||||
constructor(
|
||||
options: ImmersionTrackerOptions,
|
||||
dependencies: {
|
||||
runDeleteMaintenanceTask?: RunDeleteMaintenanceTask;
|
||||
destroyDeleteMaintenanceRunner?: () => void;
|
||||
} = {},
|
||||
) {
|
||||
this.dbPath = options.dbPath;
|
||||
this.resolveLegacyVocabularyPos = options.resolveLegacyVocabularyPos;
|
||||
let runDeleteMaintenanceTask: RunDeleteMaintenanceTask;
|
||||
if (dependencies.runDeleteMaintenanceTask) {
|
||||
runDeleteMaintenanceTask = dependencies.runDeleteMaintenanceTask;
|
||||
this.destroyDeleteMaintenanceRunner =
|
||||
dependencies.destroyDeleteMaintenanceRunner ?? (() => {});
|
||||
} else {
|
||||
const deleteMaintenanceRuntime = new DeleteMaintenanceWorkerRuntime();
|
||||
runDeleteMaintenanceTask = (dbPath, task) => deleteMaintenanceRuntime.run(dbPath, task);
|
||||
this.destroyDeleteMaintenanceRunner = () => deleteMaintenanceRuntime.destroy();
|
||||
}
|
||||
this.deleteMaintenanceScheduler = new DeleteMaintenanceScheduler({
|
||||
batchWindowMs: DELETE_MAINTENANCE_BATCH_WINDOW_MS,
|
||||
runTask: (task) => runDeleteMaintenanceTask(this.dbPath, task),
|
||||
onBusy: () => {
|
||||
this.flushTelemetry(true);
|
||||
while (this.queue.length > 0) this.flushNow();
|
||||
this.writeLock.locked = true;
|
||||
},
|
||||
onIdle: () => {
|
||||
this.writeLock.locked = false;
|
||||
if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
|
||||
},
|
||||
});
|
||||
const parentDir = path.dirname(this.dbPath);
|
||||
if (!fs.existsSync(parentDir)) {
|
||||
fs.mkdirSync(parentDir, { recursive: true });
|
||||
@@ -485,7 +526,7 @@ export class ImmersionTrackerService {
|
||||
this.logger.info(
|
||||
`Repaired season-scoped stats links on startup: scanned=${seasonRepair.scanned} movedVideos=${seasonRepair.movedVideos} deletedAnimeRows=${seasonRepair.deletedAnimeRows}`,
|
||||
);
|
||||
rebuildLifetimeSummaryTables(this.db);
|
||||
repairLifetimeSummariesFromMedia(this.db);
|
||||
}
|
||||
if (shouldBackfillLifetimeSummaries(this.db)) {
|
||||
const result = rebuildLifetimeSummaryTables(this.db);
|
||||
@@ -512,6 +553,8 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
this.finalizeActiveSession();
|
||||
this.isDestroyed = true;
|
||||
this.deleteMaintenanceScheduler.destroy();
|
||||
this.destroyDeleteMaintenanceRunner();
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
@@ -595,10 +638,32 @@ export class ImmersionTrackerService {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapse animation bursts that earlier versions recorded frame by frame. The whole
|
||||
* queue is drained first so a burst still waiting to be written is scanned as stored
|
||||
* rows rather than surviving the cleanup and landing a moment after it.
|
||||
*/
|
||||
async cleanupDuplicateSubtitleLines(
|
||||
options: DuplicateSubtitleLineCleanupOptions = {},
|
||||
): Promise<DuplicateSubtitleLineCleanupSummary> {
|
||||
this.drainQueue();
|
||||
return cleanupDuplicateSubtitleLines(this.db, options);
|
||||
}
|
||||
|
||||
async rebuildLifetimeSummaries(): Promise<LifetimeRebuildSummary> {
|
||||
this.flushTelemetry(true);
|
||||
this.flushNow();
|
||||
return rebuildLifetimeSummaryTables(this.db);
|
||||
// Non-destructive: recomputes from the media ledger (or bootstraps empty
|
||||
// lifetime tables), so history older than session retention is never reset.
|
||||
const repaired = repairLifetimeSummariesFromMedia(this.db);
|
||||
// Sessions currently tracked in the applied-sessions ledger, not sessions
|
||||
// processed by this call — the repair recomputes summaries instead of
|
||||
// re-applying sessions. Retention prunes these rows (FK cascade), so on an
|
||||
// old database this reads lower than the history the totals still include.
|
||||
const appliedRow = this.db
|
||||
.prepare('SELECT COUNT(*) AS count FROM imm_lifetime_applied_sessions')
|
||||
.get() as { count: number };
|
||||
return { appliedSessions: Number(appliedRow.count), rebuiltAtMs: repaired.repairedAtMs };
|
||||
}
|
||||
|
||||
async getKanjiStats(limit = 100): Promise<KanjiStatsRow[]> {
|
||||
@@ -709,51 +774,66 @@ export class ImmersionTrackerService {
|
||||
this.logger.warn(`Ignoring delete request for active immersion session ${sessionId}`);
|
||||
return;
|
||||
}
|
||||
deleteSessionQuery(this.db, sessionId);
|
||||
await this.enqueueDeleteMaintenanceTask(() => ({ kind: 'session', sessionId }));
|
||||
}
|
||||
|
||||
async deleteSessions(sessionIds: number[]): Promise<void> {
|
||||
const activeSessionId = this.sessionState?.sessionId;
|
||||
const deletableSessionIds =
|
||||
activeSessionId === undefined
|
||||
? sessionIds
|
||||
: sessionIds.filter((sessionId) => sessionId !== activeSessionId);
|
||||
if (deletableSessionIds.length !== sessionIds.length) {
|
||||
this.logger.warn(
|
||||
`Ignoring bulk delete request for active immersion session ${activeSessionId}`,
|
||||
);
|
||||
}
|
||||
deleteSessionsQuery(this.db, deletableSessionIds);
|
||||
await this.enqueueDeleteMaintenanceTask(() => {
|
||||
const activeSessionId = this.sessionState?.sessionId;
|
||||
const deletableSessionIds =
|
||||
activeSessionId === undefined
|
||||
? sessionIds
|
||||
: sessionIds.filter((sessionId) => sessionId !== activeSessionId);
|
||||
if (deletableSessionIds.length !== sessionIds.length) {
|
||||
this.logger.warn(
|
||||
`Ignoring bulk delete request for active immersion session ${activeSessionId}`,
|
||||
);
|
||||
}
|
||||
if (deletableSessionIds.length === 0) return null;
|
||||
return { kind: 'sessions', sessionIds: deletableSessionIds };
|
||||
});
|
||||
}
|
||||
|
||||
async deleteVideo(videoId: number): Promise<void> {
|
||||
if (this.sessionState?.videoId === videoId) {
|
||||
this.logger.warn(`Ignoring delete request for active immersion video ${videoId}`);
|
||||
return;
|
||||
}
|
||||
deleteVideoQuery(this.db, videoId);
|
||||
await this.enqueueDeleteMaintenanceTask(() => {
|
||||
if (this.sessionState?.videoId === videoId) {
|
||||
this.logger.warn(`Ignoring delete request for active immersion video ${videoId}`);
|
||||
return null;
|
||||
}
|
||||
return { kind: 'video', videoId };
|
||||
});
|
||||
}
|
||||
|
||||
async deleteAnime(animeId: number): Promise<void> {
|
||||
// The active video's anime link is assigned asynchronously after the title
|
||||
// is parsed, so a guard reading imm_videos too early sees a null and lets
|
||||
// the delete through — then the late update recreates the anime row.
|
||||
const pendingVideoId = this.sessionState?.videoId;
|
||||
if (pendingVideoId !== undefined) {
|
||||
await this.pendingAnimeMetadataUpdates.get(pendingVideoId);
|
||||
}
|
||||
|
||||
const activeVideoId = this.sessionState?.videoId;
|
||||
if (activeVideoId !== undefined) {
|
||||
const activeAnime = this.db
|
||||
.prepare('SELECT anime_id FROM imm_videos WHERE video_id = ?')
|
||||
.get(activeVideoId) as { anime_id: number | null } | null;
|
||||
if (activeAnime?.anime_id === animeId) {
|
||||
this.logger.warn(`Ignoring delete request for active immersion anime ${animeId}`);
|
||||
return;
|
||||
await this.enqueueDeleteMaintenanceTask(async () => {
|
||||
// Resolve this at dispatch time because another queued delete can leave
|
||||
// enough time for playback to switch to an episode of this anime.
|
||||
const pendingVideoId = this.sessionState?.videoId;
|
||||
if (pendingVideoId !== undefined) {
|
||||
await this.pendingAnimeMetadataUpdates.get(pendingVideoId);
|
||||
}
|
||||
|
||||
const activeVideoId = this.sessionState?.videoId;
|
||||
if (activeVideoId !== undefined) {
|
||||
const activeAnime = this.db
|
||||
.prepare('SELECT anime_id FROM imm_videos WHERE video_id = ?')
|
||||
.get(activeVideoId) as { anime_id: number | null } | null;
|
||||
if (activeAnime?.anime_id === animeId) {
|
||||
this.logger.warn(`Ignoring delete request for active immersion anime ${animeId}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return { kind: 'anime', animeId };
|
||||
});
|
||||
}
|
||||
|
||||
private enqueueDeleteMaintenanceTask(
|
||||
resolveTask: Parameters<DeleteMaintenanceScheduler['enqueue']>[0],
|
||||
): Promise<void> {
|
||||
if (this.isDestroyed) {
|
||||
return Promise.reject(new Error('Immersion tracker is shutting down'));
|
||||
}
|
||||
deleteAnimeQuery(this.db, animeId);
|
||||
return this.deleteMaintenanceScheduler.enqueue(resolveTask);
|
||||
}
|
||||
|
||||
async reassignAnimeAnilist(
|
||||
@@ -768,7 +848,7 @@ export class ImmersionTrackerService {
|
||||
coverUrl?: string | null;
|
||||
},
|
||||
): Promise<void> {
|
||||
const repair = resolveAnimeAnilistConflict(this.db, animeId, info.anilistId);
|
||||
const conflictRepair = resolveAnimeAnilistConflict(this.db, animeId, info.anilistId);
|
||||
this.db
|
||||
.prepare(
|
||||
`
|
||||
@@ -794,8 +874,16 @@ export class ImmersionTrackerService {
|
||||
nowMs(),
|
||||
animeId,
|
||||
);
|
||||
if (repair.movedVideos > 0 || repair.deletedAnimeRows > 0) {
|
||||
rebuildLifetimeSummaryTables(this.db);
|
||||
// Empty lifetime tables still need the retained-session bootstrap. Once a
|
||||
// media ledger exists, only the redistributed and explicitly edited anime
|
||||
// can have changed.
|
||||
if (shouldBackfillLifetimeSummaries(this.db)) {
|
||||
repairLifetimeSummariesFromMedia(this.db);
|
||||
} else {
|
||||
const affectedAnimeIds = new Set(conflictRepair.affectedAnimeIds);
|
||||
affectedAnimeIds.add(animeId);
|
||||
recomputeLifetimeAnimeFromMedia(this.db, [...affectedAnimeIds]);
|
||||
recomputeLifetimeGlobalFromSummaries(this.db);
|
||||
}
|
||||
|
||||
// Update cover art for all videos in this anime
|
||||
@@ -1243,7 +1331,7 @@ export class ImmersionTrackerService {
|
||||
metadataJson: candidate.metadataJson,
|
||||
});
|
||||
}
|
||||
rebuildLifetimeSummaryTables(this.db);
|
||||
repairLifetimeSummariesFromMedia(this.db);
|
||||
}
|
||||
|
||||
recordJellyfinPlaybackMetadata(metadata: JellyfinPlaybackMetadataInput): void {
|
||||
@@ -1316,7 +1404,21 @@ export class ImmersionTrackerService {
|
||||
this.db.prepare('SELECT 1 FROM imm_lifetime_media WHERE video_id = ?').get(videoId),
|
||||
);
|
||||
if (hasLifetimeMedia || (previousLink && previousLink.animeId !== animeId)) {
|
||||
rebuildLifetimeSummaryTables(this.db);
|
||||
// Playback-time relink: only the old and new anime are affected, so
|
||||
// recompute just those from the media ledger instead of a full repair.
|
||||
const affectedAnimeIds = new Set<number>([animeId]);
|
||||
if (previousLink?.animeId) affectedAnimeIds.add(previousLink.animeId);
|
||||
let transactionStarted = false;
|
||||
try {
|
||||
this.db.exec('BEGIN IMMEDIATE');
|
||||
transactionStarted = true;
|
||||
recomputeLifetimeAnimeFromMedia(this.db, [...affectedAnimeIds]);
|
||||
recomputeLifetimeGlobalFromSummaries(this.db);
|
||||
this.db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
if (transactionStarted) this.db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1799,6 +1901,24 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write out everything queued, not just the next batch.
|
||||
*
|
||||
* `flushNow` writes at most `batchSize` entries and does nothing at all while the write
|
||||
* lock is held, so a maintenance pass that runs straight after it can still be reading
|
||||
* a database that is missing rows. Each pass has to shrink the queue to continue: a
|
||||
* failed flush puts its batch back, and looping on that would never finish.
|
||||
*/
|
||||
private drainQueue(): void {
|
||||
while (this.queue.length > 0) {
|
||||
const pendingBefore = this.queue.length;
|
||||
this.flushNow();
|
||||
if (this.queue.length >= pendingBefore) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private flushSingle(write: QueuedWrite): void {
|
||||
executeQueuedWrite(write, this.preparedStatements);
|
||||
}
|
||||
@@ -1811,7 +1931,7 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
|
||||
private runMaintenance(): void {
|
||||
if (this.isDestroyed) return;
|
||||
if (this.isDestroyed || this.writeLock.locked) return;
|
||||
try {
|
||||
this.flushTelemetry(true);
|
||||
this.flushNow();
|
||||
|
||||
@@ -0,0 +1,467 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { Database } from '../sqlite.js';
|
||||
import type { DatabaseSync } from '../sqlite.js';
|
||||
import { ensureSchema } from '../storage.js';
|
||||
import { cleanupDuplicateSubtitleLines } from '../duplicate-line-cleanup.js';
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
const BASE_MS = 1_700_000_000_000;
|
||||
const WORD_ID = 1;
|
||||
|
||||
interface SeedLine {
|
||||
session: number;
|
||||
text: string;
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
/** Recording wall-clock, i.e. what the lookback window filters on. */
|
||||
createdMs?: number;
|
||||
}
|
||||
|
||||
function makeDbPath(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-duplicate-line-test-'));
|
||||
return path.join(dir, 'immersion.sqlite');
|
||||
}
|
||||
|
||||
function cleanupDbPath(dbPath: string): void {
|
||||
const dir = path.dirname(dbPath);
|
||||
if (!fs.existsSync(dir)) return;
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
/** One episode, two sessions of it, and one word occurrence per seeded line. */
|
||||
function seed(db: DatabaseSync, lines: SeedLine[]): void {
|
||||
db.exec(`
|
||||
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, CREATED_DATE, LAST_UPDATE_DATE)
|
||||
VALUES (1, 'show', 'Show', ${BASE_MS}, ${BASE_MS});
|
||||
INSERT INTO imm_videos(video_id, video_key, anime_id, canonical_title, source_type, watched, duration_ms, CREATED_DATE, LAST_UPDATE_DATE)
|
||||
VALUES (1, 'v1', 1, 'Ep 1', 1, 1, 1440000, ${BASE_MS}, ${BASE_MS});
|
||||
INSERT INTO imm_sessions(session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status, CREATED_DATE, LAST_UPDATE_DATE)
|
||||
VALUES (1, 's1', 1, '${BASE_MS}', '${BASE_MS + 1000}', 2, ${BASE_MS}, ${BASE_MS}),
|
||||
(2, 's2', 1, '${BASE_MS + DAY_MS}', '${BASE_MS + DAY_MS + 1000}', 2, ${BASE_MS}, ${BASE_MS});
|
||||
INSERT INTO imm_words(id, headword, word, reading, part_of_speech, pos1, first_seen, last_seen, frequency)
|
||||
VALUES (${WORD_ID}, '飛び上がる', '飛び上がる', '', 'verb', '動詞', ${Math.floor(BASE_MS / 1000)}, ${Math.floor(BASE_MS / 1000)}, 0);
|
||||
`);
|
||||
|
||||
const insertLine = db.prepare(
|
||||
`INSERT INTO imm_subtitle_lines(
|
||||
line_id, session_id, video_id, anime_id, line_index,
|
||||
segment_start_ms, segment_end_ms, text, CREATED_DATE, LAST_UPDATE_DATE)
|
||||
VALUES (?, ?, 1, 1, ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
const insertOccurrence = db.prepare(
|
||||
`INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms)
|
||||
VALUES (?, ?, 1, ?)`,
|
||||
);
|
||||
|
||||
lines.forEach((line, index) => {
|
||||
const lineId = index + 1;
|
||||
const lineIndex = index + 1;
|
||||
const createdMs = line.createdMs ?? BASE_MS;
|
||||
insertLine.run(
|
||||
lineId,
|
||||
line.session,
|
||||
lineIndex,
|
||||
line.startMs,
|
||||
line.endMs,
|
||||
line.text,
|
||||
createdMs,
|
||||
createdMs,
|
||||
);
|
||||
insertOccurrence.run(lineId, WORD_ID, createdMs);
|
||||
});
|
||||
|
||||
db.exec(`
|
||||
UPDATE imm_words SET frequency = (
|
||||
SELECT COALESCE(SUM(o.occurrence_count), 0)
|
||||
FROM imm_word_line_occurrences o WHERE o.word_id = imm_words.id
|
||||
)
|
||||
`);
|
||||
}
|
||||
|
||||
function createDb(lines: SeedLine[]): { db: DatabaseSync; dbPath: string } {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
ensureSchema(db);
|
||||
seed(db, lines);
|
||||
return { db, dbPath };
|
||||
}
|
||||
|
||||
/** A typeset line mpv reported once per animation frame. */
|
||||
function karaokeFrames(
|
||||
session: number,
|
||||
text: string,
|
||||
startMs: number,
|
||||
frames: number,
|
||||
frameMs: number,
|
||||
): SeedLine[] {
|
||||
return Array.from({ length: frames }, (_, index) => ({
|
||||
session,
|
||||
text,
|
||||
startMs: startMs + index * frameMs,
|
||||
endMs: startMs + (index + 1) * frameMs,
|
||||
}));
|
||||
}
|
||||
|
||||
function countLines(db: DatabaseSync): number {
|
||||
return (db.prepare('SELECT COUNT(*) AS total FROM imm_subtitle_lines').get() as { total: number })
|
||||
.total;
|
||||
}
|
||||
|
||||
function wordFrequency(db: DatabaseSync): number {
|
||||
const row = db.prepare('SELECT frequency FROM imm_words WHERE id = ?').get(WORD_ID) as {
|
||||
frequency: number;
|
||||
} | null;
|
||||
return row?.frequency ?? 0;
|
||||
}
|
||||
|
||||
test('a karaoke burst collapses to one line and gives back its word counts', () => {
|
||||
const { db, dbPath } = createDb([
|
||||
...karaokeFrames(1, '飛び上がる', 10_000, 40, 40),
|
||||
{ session: 1, text: 'おはよう', startMs: 20_000, endMs: 22_000 },
|
||||
]);
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
|
||||
assert.equal(summary.burstGroups, 1);
|
||||
assert.equal(summary.removedLines, 39);
|
||||
assert.equal(summary.removedWordOccurrences, 39);
|
||||
assert.equal(countLines(db), 2);
|
||||
assert.equal(wordFrequency(db), 2);
|
||||
|
||||
// The surviving line covers the whole run, the way the parsed cue would.
|
||||
const kept = db
|
||||
.prepare(
|
||||
'SELECT segment_start_ms AS startMs, segment_end_ms AS endMs FROM imm_subtitle_lines WHERE line_id = 1',
|
||||
)
|
||||
.get() as { startMs: number; endMs: number };
|
||||
assert.equal(kept.startMs, 10_000);
|
||||
assert.equal(kept.endMs, 10_000 + 40 * 40);
|
||||
|
||||
assert.equal(summary.samples.length, 1);
|
||||
assert.equal(summary.samples[0]!.text, '飛び上がる');
|
||||
assert.equal(summary.samples[0]!.frames, 40);
|
||||
assert.equal(summary.samples[0]!.videoTitle, 'Ep 1');
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('ordinary repeated dialogue survives', () => {
|
||||
// Six contiguous `飛び上がる`, each held for a normal beat rather than a frame.
|
||||
const lines = Array.from({ length: 6 }, (_, index) => ({
|
||||
session: 1,
|
||||
text: '飛び上がる',
|
||||
startMs: 5_000 + index * 800,
|
||||
endMs: 5_000 + (index + 1) * 800,
|
||||
}));
|
||||
const { db, dbPath } = createDb(lines);
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
|
||||
assert.equal(summary.burstGroups, 0);
|
||||
assert.equal(summary.removedLines, 0);
|
||||
assert.equal(countLines(db), 6);
|
||||
assert.equal(wordFrequency(db), 6);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a long run of quarter-second frames is still a burst', () => {
|
||||
// Between the timing-only bound (0.1s) and the animation-frame bound (0.3s): heavier
|
||||
// typesetting lands here, and the run length is what makes it conclusive.
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
|
||||
assert.equal(summary.burstGroups, 1);
|
||||
assert.equal(summary.removedLines, 5);
|
||||
assert.equal(countLines(db), 1);
|
||||
assert.equal(wordFrequency(db), 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a qualifying short-frame burst may end with one long hold frame', () => {
|
||||
const { db, dbPath } = createDb([
|
||||
...karaokeFrames(1, '飛び上がる', 10_000, 8, 40),
|
||||
{ session: 1, text: '飛び上がる', startMs: 10_320, endMs: 12_320 },
|
||||
]);
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
|
||||
assert.equal(summary.burstGroups, 1);
|
||||
assert.equal(summary.removedLines, 8);
|
||||
assert.equal(countLines(db), 1);
|
||||
assert.equal(wordFrequency(db), 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a long event before the final frame prevents burst cleanup', () => {
|
||||
const { db, dbPath } = createDb([
|
||||
...karaokeFrames(1, '飛び上がる', 10_000, 5, 40),
|
||||
{ session: 1, text: '飛び上がる', startMs: 10_200, endMs: 12_200 },
|
||||
{ session: 1, text: '飛び上がる', startMs: 12_200, endMs: 12_240 },
|
||||
]);
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
|
||||
assert.equal(summary.burstGroups, 0);
|
||||
assert.equal(countLines(db), 7);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a run of frames longer than the animation bound survives', () => {
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 400));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
|
||||
assert.equal(summary.burstGroups, 0);
|
||||
assert.equal(countLines(db), 6);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('the four-frame residue the live gate stores is cleaned up', () => {
|
||||
// The streaming gate records the first four frames of a burst before the run is long
|
||||
// enough to recognise. Four contiguous identical events under the strict timing-only
|
||||
// bound are that residue, and no real dialogue.
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 40));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
|
||||
assert.equal(summary.burstGroups, 1);
|
||||
assert.equal(summary.removedLines, 3);
|
||||
assert.equal(countLines(db), 1);
|
||||
assert.equal(wordFrequency(db), 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a four-frame run above the strict frame bound survives', () => {
|
||||
// Long enough per event to be plausible dialogue; only a five-event run may use the
|
||||
// looser animation-frame bound.
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 250));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
|
||||
assert.equal(summary.burstGroups, 0);
|
||||
assert.equal(countLines(db), 4);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('an explicit minRunLength raises the bar', () => {
|
||||
// Five quarter-second frames qualify under the defaults; a cautious run asking for six
|
||||
// leaves them alone. Above the strict bound, so the residue rule stays out of it.
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 5, 250));
|
||||
|
||||
try {
|
||||
const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
|
||||
assert.equal(preview.burstGroups, 1);
|
||||
|
||||
const summary = cleanupDuplicateSubtitleLines(db, { minRunLength: 6 });
|
||||
assert.equal(summary.burstGroups, 0);
|
||||
assert.equal(countLines(db), 5);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('an explicit maxFrameSeconds tightens the frame bound', () => {
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db, { maxFrameSeconds: 0.2 });
|
||||
|
||||
assert.equal(summary.burstGroups, 0);
|
||||
assert.equal(countLines(db), 6);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a non-finite maxFrameSeconds falls back to the default bound', () => {
|
||||
// Six normal-beat lines: Infinity must not turn every event into a "short frame".
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 800));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db, { maxFrameSeconds: Infinity });
|
||||
|
||||
assert.equal(summary.burstGroups, 0);
|
||||
assert.equal(countLines(db), 6);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('sampleLimit zero removes bursts but reports no samples', () => {
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db, { sampleLimit: 0 });
|
||||
|
||||
assert.equal(summary.removedLines, 39);
|
||||
assert.deepEqual(summary.samples, []);
|
||||
assert.equal(countLines(db), 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a short run below every threshold survives', () => {
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 3, 40));
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
|
||||
assert.equal(summary.burstGroups, 0);
|
||||
assert.equal(countLines(db), 3);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('interleaved dual-line karaoke collapses each line to one row', () => {
|
||||
// Kanji and romaji lines frame-flipped together, the way fansub OPs are typeset. The
|
||||
// rows arrive interleaved in time order; each text must still chain into its own run.
|
||||
const kanji = karaokeFrames(1, '飛び上がる', 10_000, 20, 60);
|
||||
const romaji = karaokeFrames(1, 'tobiagaru', 10_001, 20, 60);
|
||||
const interleaved = [...kanji, ...romaji].sort((a, b) => a.startMs - b.startMs);
|
||||
const { db, dbPath } = createDb(interleaved);
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
|
||||
assert.equal(summary.burstGroups, 2);
|
||||
assert.equal(summary.removedLines, 38);
|
||||
assert.equal(countLines(db), 2);
|
||||
assert.equal(wordFrequency(db), 2);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('the same line in a rewatch session is never merged into the first watch', () => {
|
||||
const { db, dbPath } = createDb([
|
||||
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
|
||||
...karaokeFrames(2, '飛び上がる', 10_000, 6, 40),
|
||||
]);
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
|
||||
assert.equal(summary.burstGroups, 2);
|
||||
assert.equal(summary.removedLines, 10);
|
||||
// One surviving line per session, not one across both.
|
||||
assert.equal(countLines(db), 2);
|
||||
assert.equal(wordFrequency(db), 2);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a gap between runs splits them', () => {
|
||||
const { db, dbPath } = createDb([
|
||||
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
|
||||
...karaokeFrames(1, '飛び上がる', 60_000, 6, 40),
|
||||
]);
|
||||
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db);
|
||||
|
||||
assert.equal(summary.burstGroups, 2);
|
||||
assert.equal(countLines(db), 2);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('a dry run reports what an apply would do and writes nothing', () => {
|
||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
|
||||
|
||||
try {
|
||||
const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
|
||||
|
||||
assert.equal(preview.dryRun, true);
|
||||
assert.equal(preview.removedLines, 39);
|
||||
assert.equal(countLines(db), 40);
|
||||
assert.equal(wordFrequency(db), 40);
|
||||
|
||||
const applied = cleanupDuplicateSubtitleLines(db);
|
||||
assert.equal(applied.removedLines, preview.removedLines);
|
||||
assert.equal(applied.removedWordOccurrences, preview.removedWordOccurrences);
|
||||
assert.equal(countLines(db), 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('the lookback window leaves older bursts alone', () => {
|
||||
const recentMs = BASE_MS;
|
||||
const oldMs = BASE_MS - 40 * DAY_MS;
|
||||
const { db, dbPath } = createDb([
|
||||
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40).map((line) => ({
|
||||
...line,
|
||||
createdMs: oldMs,
|
||||
})),
|
||||
...karaokeFrames(2, '飛び上がる', 10_000, 6, 40).map((line) => ({
|
||||
...line,
|
||||
createdMs: recentMs,
|
||||
})),
|
||||
]);
|
||||
|
||||
globalThis.__subminerTestNowMs = BASE_MS;
|
||||
try {
|
||||
const summary = cleanupDuplicateSubtitleLines(db, { lookbackDays: 30 });
|
||||
|
||||
assert.equal(summary.lookbackDays, 30);
|
||||
assert.equal(summary.scannedLines, 6);
|
||||
assert.equal(summary.burstGroups, 1);
|
||||
assert.equal(summary.removedLines, 5);
|
||||
// Six untouched old frames plus the one surviving recent line.
|
||||
assert.equal(countLines(db), 7);
|
||||
assert.equal(wordFrequency(db), 7);
|
||||
} finally {
|
||||
globalThis.__subminerTestNowMs = undefined;
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
@@ -329,3 +329,28 @@ test('upgrading an older database backfills seen_ms from the subtitle lines', ()
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('an extreme-moving delete keeps subtraction-exact frequency instead of re-summing', () => {
|
||||
const { db, dbPath } = createDb([
|
||||
{ session: 1, wordId: 7, dayOffset: 0, count: 2 },
|
||||
{ session: 2, wordId: 7, dayOffset: 3, count: 1 },
|
||||
]);
|
||||
|
||||
try {
|
||||
// Simulate drift: the stored total is higher than the occurrences justify.
|
||||
// The extremes move via index seeks while the count stays a pure
|
||||
// subtraction; drifted counts reconcile only at the zero-crossing repair
|
||||
// or via the cleanup command, never by rescanning every occurrence here.
|
||||
db.prepare('UPDATE imm_words SET frequency = 10 WHERE id = 7').run();
|
||||
|
||||
deleteSession(db, 1);
|
||||
|
||||
const word = readWord(db, 7);
|
||||
assert.equal(word?.frequency, 10 - 2);
|
||||
assert.equal(word?.firstSeen, Math.floor((BASE_MS + 3 * DAY_MS) / 1000));
|
||||
assert.equal(word?.lastSeen, Math.floor((BASE_MS + 3 * DAY_MS) / 1000));
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { startSessionRecord } from '../session.js';
|
||||
import { applySessionLifetimeSummary, rebuildLifetimeSummaries } from '../lifetime.js';
|
||||
import { deleteMaintenanceBatch } from '../query-delete-maintenance.js';
|
||||
import { toDbTimestamp } from '../query-shared.js';
|
||||
import {
|
||||
BASE_MS,
|
||||
DAY_MS,
|
||||
cleanRow,
|
||||
createDb,
|
||||
seedAnime,
|
||||
seedEndedSession,
|
||||
seedVideo,
|
||||
snapshotAnime,
|
||||
snapshotGlobal,
|
||||
snapshotMedia,
|
||||
} from './lifetime-test-fixtures.js';
|
||||
|
||||
test('fractional lifetime metrics stay normalized across apply, rebuild, and delete', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const videoId = seedVideo(db, null, 'fractional-metrics');
|
||||
const seedFractionalSession = (
|
||||
startedAtMs: number,
|
||||
metrics: { activeMs: number; cards: number; lines: number; tokens: number },
|
||||
) => {
|
||||
const { state } = startSessionRecord(db, videoId, startedAtMs);
|
||||
state.activeWatchedMs = metrics.activeMs;
|
||||
state.cardsMined = metrics.cards;
|
||||
state.linesSeen = metrics.lines;
|
||||
state.tokensSeen = metrics.tokens;
|
||||
const endedAtMs = startedAtMs + 2_000;
|
||||
db.prepare(
|
||||
`UPDATE imm_sessions SET
|
||||
ended_at_ms = ?,
|
||||
active_watched_ms = ?,
|
||||
cards_mined = ?,
|
||||
lines_seen = ?,
|
||||
tokens_seen = ?
|
||||
WHERE session_id = ?`,
|
||||
).run(
|
||||
toDbTimestamp(endedAtMs),
|
||||
metrics.activeMs,
|
||||
metrics.cards,
|
||||
metrics.lines,
|
||||
metrics.tokens,
|
||||
state.sessionId,
|
||||
);
|
||||
return { state, endedAtMs };
|
||||
};
|
||||
const readMediaMetrics = () =>
|
||||
cleanRow<{
|
||||
total_sessions: number;
|
||||
total_active_ms: number;
|
||||
total_cards: number;
|
||||
total_lines_seen: number;
|
||||
total_tokens_seen: number;
|
||||
}>(
|
||||
db
|
||||
.prepare(
|
||||
`SELECT total_sessions, total_active_ms, total_cards,
|
||||
total_lines_seen, total_tokens_seen
|
||||
FROM imm_lifetime_media WHERE video_id = ?`,
|
||||
)
|
||||
.get(videoId),
|
||||
);
|
||||
|
||||
const withoutTelemetry = seedFractionalSession(BASE_MS, {
|
||||
activeMs: 1_234.9,
|
||||
cards: 2.8,
|
||||
lines: 3.7,
|
||||
tokens: 4.6,
|
||||
});
|
||||
applySessionLifetimeSummary(db, withoutTelemetry.state, withoutTelemetry.endedAtMs);
|
||||
assert.deepEqual(readMediaMetrics(), {
|
||||
total_sessions: 1,
|
||||
total_active_ms: 1_234,
|
||||
total_cards: 2,
|
||||
total_lines_seen: 3,
|
||||
total_tokens_seen: 4,
|
||||
});
|
||||
|
||||
const withTelemetry = seedFractionalSession(BASE_MS + DAY_MS, {
|
||||
activeMs: 9_999.9,
|
||||
cards: 9.9,
|
||||
lines: 9.9,
|
||||
tokens: 9.9,
|
||||
});
|
||||
db.prepare(
|
||||
`INSERT INTO imm_session_telemetry (
|
||||
session_id, sample_ms, active_watched_ms, cards_mined, lines_seen, tokens_seen
|
||||
) VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
).run(withTelemetry.state.sessionId, withTelemetry.endedAtMs, 2_345.9, 5.8, 6.7, 7.6);
|
||||
applySessionLifetimeSummary(db, withTelemetry.state, withTelemetry.endedAtMs);
|
||||
assert.deepEqual(readMediaMetrics(), {
|
||||
total_sessions: 2,
|
||||
total_active_ms: 3_579,
|
||||
total_cards: 7,
|
||||
total_lines_seen: 9,
|
||||
total_tokens_seen: 11,
|
||||
});
|
||||
|
||||
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: withTelemetry.state.sessionId }]);
|
||||
const retainedMetrics = {
|
||||
total_sessions: 1,
|
||||
total_active_ms: 1_234,
|
||||
total_cards: 2,
|
||||
total_lines_seen: 3,
|
||||
total_tokens_seen: 4,
|
||||
};
|
||||
assert.deepEqual(readMediaMetrics(), retainedMetrics, 'delete subtracts floored telemetry');
|
||||
|
||||
rebuildLifetimeSummaries(db);
|
||||
assert.deepEqual(
|
||||
readMediaMetrics(),
|
||||
retainedMetrics,
|
||||
'rebuild floors session-row fallback values',
|
||||
);
|
||||
|
||||
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: withoutTelemetry.state.sessionId }]);
|
||||
assert.deepEqual(snapshotMedia(db), [], 'delete subtracts the normalized metrics exactly');
|
||||
assert.deepEqual(snapshotGlobal(db), {
|
||||
total_sessions: 0,
|
||||
total_active_ms: 0,
|
||||
total_cards: 0,
|
||||
active_days: 0,
|
||||
episodes_started: 0,
|
||||
episodes_completed: 0,
|
||||
anime_completed: 0,
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('incremental delete maintenance matches a full rebuild when no history is pruned', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const animeA = seedAnime(db, 'Anime A', 2);
|
||||
const animeB = seedAnime(db, 'Anime B', 1);
|
||||
const videoA1 = seedVideo(db, animeA, 'anime-a-ep1', { watched: true });
|
||||
const videoA2 = seedVideo(db, animeA, 'anime-a-ep2', { watched: true });
|
||||
const videoB1 = seedVideo(db, animeB, 'anime-b-ep1', { watched: true });
|
||||
const videoLoose = seedVideo(db, null, 'loose-video');
|
||||
|
||||
seedEndedSession(db, videoA1, BASE_MS, { activeMs: 60_000, cards: 2, lines: 30, tokens: 200 });
|
||||
const deletedSessionId = seedEndedSession(db, videoA1, BASE_MS + DAY_MS, {
|
||||
activeMs: 45_000,
|
||||
cards: 1,
|
||||
lines: 20,
|
||||
tokens: 100,
|
||||
});
|
||||
seedEndedSession(db, videoA2, BASE_MS + 2 * DAY_MS, { activeMs: 90_000, cards: 3 });
|
||||
seedEndedSession(db, videoB1, BASE_MS + 3 * DAY_MS, { activeMs: 30_000 });
|
||||
seedEndedSession(db, videoLoose, BASE_MS + 4 * DAY_MS, { activeMs: 15_000 });
|
||||
|
||||
rebuildLifetimeSummaries(db);
|
||||
|
||||
deleteMaintenanceBatch(db, [
|
||||
{ kind: 'session', sessionId: deletedSessionId },
|
||||
{ kind: 'video', videoId: videoLoose },
|
||||
{ kind: 'anime', animeId: animeB },
|
||||
]);
|
||||
|
||||
const incrementalGlobal = snapshotGlobal(db);
|
||||
const incrementalMedia = snapshotMedia(db);
|
||||
const incrementalAnime = snapshotAnime(db);
|
||||
|
||||
// With every session still retained, subtracting must land on exactly the
|
||||
// state a from-scratch rebuild computes.
|
||||
rebuildLifetimeSummaries(db);
|
||||
assert.deepEqual(incrementalGlobal, snapshotGlobal(db));
|
||||
assert.deepEqual(incrementalMedia, snapshotMedia(db));
|
||||
assert.deepEqual(incrementalAnime, snapshotAnime(db));
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('deleting a retained session preserves lifetime history from pruned sessions', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const animeId = seedAnime(db, 'Pruned Anime', null);
|
||||
const videoId = seedVideo(db, animeId, 'pruned-ep1');
|
||||
const prunedSessionId = seedEndedSession(db, videoId, BASE_MS, {
|
||||
activeMs: 120_000,
|
||||
cards: 4,
|
||||
lines: 50,
|
||||
tokens: 400,
|
||||
});
|
||||
const retainedSessionId = seedEndedSession(db, videoId, BASE_MS + DAY_MS, {
|
||||
activeMs: 30_000,
|
||||
cards: 1,
|
||||
lines: 10,
|
||||
tokens: 80,
|
||||
});
|
||||
rebuildLifetimeSummaries(db);
|
||||
|
||||
// Simulate raw-session retention pruning the older session. Lifetime
|
||||
// summaries intentionally keep its contribution.
|
||||
db.prepare('DELETE FROM imm_sessions WHERE session_id = ?').run(prunedSessionId);
|
||||
|
||||
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: retainedSessionId }]);
|
||||
|
||||
const globalRow = snapshotGlobal(db);
|
||||
assert.equal(globalRow.total_sessions, 1, 'pruned session contribution survives the delete');
|
||||
assert.equal(globalRow.total_active_ms, 120_000);
|
||||
assert.equal(globalRow.total_cards, 4);
|
||||
assert.equal(globalRow.episodes_started, 1);
|
||||
// The pruned session's day stays counted (pruning never subtracts); only
|
||||
// the deleted retained session's day is dropped.
|
||||
assert.equal(globalRow.active_days, 1);
|
||||
|
||||
const mediaRow = db
|
||||
.prepare(
|
||||
'SELECT total_sessions, total_active_ms, total_cards FROM imm_lifetime_media WHERE video_id = ?',
|
||||
)
|
||||
.get(videoId);
|
||||
assert.deepEqual(
|
||||
cleanRow<{ total_sessions: number; total_active_ms: number; total_cards: number }>(mediaRow),
|
||||
{ total_sessions: 1, total_active_ms: 120_000, total_cards: 4 },
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('active_days only drops when the last session of a local day is deleted', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const videoId = seedVideo(db, null, 'same-day');
|
||||
const firstSessionId = seedEndedSession(db, videoId, BASE_MS, { activeMs: 10_000 });
|
||||
const secondSessionId = seedEndedSession(db, videoId, BASE_MS + 3_600_000, {
|
||||
activeMs: 20_000,
|
||||
});
|
||||
rebuildLifetimeSummaries(db);
|
||||
assert.equal(snapshotGlobal(db).active_days, 1);
|
||||
|
||||
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: firstSessionId }]);
|
||||
assert.equal(snapshotGlobal(db).active_days, 1, 'day still has a session');
|
||||
|
||||
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: secondSessionId }]);
|
||||
assert.equal(snapshotGlobal(db).active_days, 0, 'day lost its last session');
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('deleting a video updates anime and global rollups without a rebuild', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const animeId = seedAnime(db, 'Two Episode Anime', 2);
|
||||
const videoEp1 = seedVideo(db, animeId, 'two-ep-1', { watched: true });
|
||||
const videoEp2 = seedVideo(db, animeId, 'two-ep-2', { watched: true });
|
||||
seedEndedSession(db, videoEp1, BASE_MS, { activeMs: 60_000, cards: 2 });
|
||||
seedEndedSession(db, videoEp2, BASE_MS + DAY_MS, { activeMs: 40_000, cards: 1 });
|
||||
rebuildLifetimeSummaries(db);
|
||||
assert.equal(snapshotGlobal(db).anime_completed, 1);
|
||||
|
||||
deleteMaintenanceBatch(db, [{ kind: 'video', videoId: videoEp2 }]);
|
||||
|
||||
const globalRow = snapshotGlobal(db);
|
||||
assert.equal(globalRow.total_sessions, 1);
|
||||
assert.equal(globalRow.total_active_ms, 60_000);
|
||||
assert.equal(globalRow.episodes_started, 1);
|
||||
assert.equal(globalRow.episodes_completed, 1);
|
||||
assert.equal(globalRow.anime_completed, 0, 'anime no longer has all episodes completed');
|
||||
|
||||
const animeRow = db
|
||||
.prepare(
|
||||
'SELECT total_sessions, episodes_started, episodes_completed FROM imm_lifetime_anime WHERE anime_id = ?',
|
||||
)
|
||||
.get(animeId);
|
||||
assert.deepEqual(
|
||||
cleanRow<{
|
||||
total_sessions: number;
|
||||
episodes_started: number;
|
||||
episodes_completed: number;
|
||||
}>(animeRow),
|
||||
{ total_sessions: 1, episodes_started: 1, episodes_completed: 1 },
|
||||
);
|
||||
|
||||
deleteMaintenanceBatch(db, [{ kind: 'video', videoId: videoEp1 }]);
|
||||
assert.equal(
|
||||
(db.prepare('SELECT COUNT(*) AS total FROM imm_lifetime_anime').get() as { total: number })
|
||||
.total,
|
||||
0,
|
||||
'anime lifetime row is dropped once no episodes remain',
|
||||
);
|
||||
assert.deepEqual(snapshotGlobal(db), {
|
||||
total_sessions: 0,
|
||||
total_active_ms: 0,
|
||||
total_cards: 0,
|
||||
active_days: 0,
|
||||
episodes_started: 0,
|
||||
episodes_completed: 0,
|
||||
anime_completed: 0,
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,94 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { rebuildLifetimeSummaries, repairLifetimeSummariesFromMedia } from '../lifetime.js';
|
||||
import {
|
||||
BASE_MS,
|
||||
DAY_MS,
|
||||
cleanRow,
|
||||
createDb,
|
||||
seedAnime,
|
||||
seedEndedSession,
|
||||
seedVideo,
|
||||
snapshotAnime,
|
||||
snapshotGlobal,
|
||||
snapshotMedia,
|
||||
} from './lifetime-test-fixtures.js';
|
||||
|
||||
test('repair after a video moves between anime matches a full rebuild', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const animeA = seedAnime(db, 'Move Source', 2);
|
||||
const animeB = seedAnime(db, 'Move Target', 2);
|
||||
const movedVideo = seedVideo(db, animeA, 'moved-ep', { watched: true });
|
||||
const stayingVideo = seedVideo(db, animeA, 'staying-ep');
|
||||
const targetVideo = seedVideo(db, animeB, 'target-ep', { watched: true });
|
||||
seedEndedSession(db, movedVideo, BASE_MS, { activeMs: 60_000, cards: 2 });
|
||||
seedEndedSession(db, stayingVideo, BASE_MS + DAY_MS, { activeMs: 30_000 });
|
||||
seedEndedSession(db, targetVideo, BASE_MS + 2 * DAY_MS, { activeMs: 45_000, cards: 1 });
|
||||
rebuildLifetimeSummaries(db);
|
||||
|
||||
// Simulate a library merge reassigning the episode to the other anime.
|
||||
db.prepare('UPDATE imm_videos SET anime_id = ? WHERE video_id = ?').run(animeB, movedVideo);
|
||||
repairLifetimeSummariesFromMedia(db);
|
||||
|
||||
const repairedGlobal = snapshotGlobal(db);
|
||||
const repairedMedia = snapshotMedia(db);
|
||||
const repairedAnime = snapshotAnime(db);
|
||||
|
||||
rebuildLifetimeSummaries(db);
|
||||
assert.deepEqual(repairedGlobal, snapshotGlobal(db));
|
||||
assert.deepEqual(repairedMedia, snapshotMedia(db));
|
||||
assert.deepEqual(repairedAnime, snapshotAnime(db));
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('repair preserves lifetime history from pruned sessions where a rebuild would not', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const animeId = seedAnime(db, 'Repair Anime', null);
|
||||
const videoId = seedVideo(db, animeId, 'repair-ep');
|
||||
const prunedSessionId = seedEndedSession(db, videoId, BASE_MS, {
|
||||
activeMs: 90_000,
|
||||
cards: 3,
|
||||
});
|
||||
seedEndedSession(db, videoId, BASE_MS + DAY_MS, { activeMs: 30_000, cards: 1 });
|
||||
rebuildLifetimeSummaries(db);
|
||||
|
||||
db.prepare('DELETE FROM imm_sessions WHERE session_id = ?').run(prunedSessionId);
|
||||
repairLifetimeSummariesFromMedia(db);
|
||||
|
||||
const globalRow = snapshotGlobal(db);
|
||||
assert.equal(globalRow.total_sessions, 2, 'repair keeps the pruned session contribution');
|
||||
assert.equal(globalRow.total_active_ms, 120_000);
|
||||
assert.equal(globalRow.total_cards, 4);
|
||||
assert.equal(globalRow.active_days, 2, 'repair never subtracts active days');
|
||||
|
||||
const animeRow = db
|
||||
.prepare('SELECT total_sessions FROM imm_lifetime_anime WHERE anime_id = ?')
|
||||
.get(animeId);
|
||||
assert.equal(cleanRow<{ total_sessions: number }>(animeRow).total_sessions, 2);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('repair leaves a caller-owned transaction intact when its begin fails', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
db.exec('BEGIN');
|
||||
const animeId = seedAnime(db, 'Caller Transaction', null);
|
||||
|
||||
assert.throws(() => repairLifetimeSummariesFromMedia(db), /transaction/i);
|
||||
assert.ok(
|
||||
db.prepare('SELECT 1 FROM imm_anime WHERE anime_id = ?').get(animeId),
|
||||
'the repair did not roll back the caller transaction',
|
||||
);
|
||||
|
||||
db.exec('ROLLBACK');
|
||||
assert.equal(db.prepare('SELECT 1 FROM imm_anime WHERE anime_id = ?').get(animeId), undefined);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,157 @@
|
||||
import { Database } from '../sqlite.js';
|
||||
import type { DatabaseSync } from '../sqlite.js';
|
||||
import {
|
||||
applyPragmas,
|
||||
ensureSchema,
|
||||
getOrCreateAnimeRecord,
|
||||
getOrCreateVideoRecord,
|
||||
linkVideoToAnimeRecord,
|
||||
} from '../storage.js';
|
||||
import { startSessionRecord } from '../session.js';
|
||||
import { toDbTimestamp } from '../query-shared.js';
|
||||
|
||||
const SOURCE_TYPE_LOCAL = 1;
|
||||
export const DAY_MS = 86_400_000;
|
||||
// Noon UTC keeps every seeded timestamp on the same local day regardless of
|
||||
// the timezone the test host runs in.
|
||||
export const BASE_MS = Date.UTC(2026, 0, 5, 12, 0, 0);
|
||||
|
||||
export function createDb(): DatabaseSync {
|
||||
const db = new Database(':memory:');
|
||||
applyPragmas(db);
|
||||
ensureSchema(db);
|
||||
return db;
|
||||
}
|
||||
|
||||
export function seedAnime(db: DatabaseSync, title: string, episodesTotal: number | null): number {
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: title,
|
||||
canonicalTitle: title,
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: null,
|
||||
});
|
||||
if (episodesTotal !== null) {
|
||||
db.prepare('UPDATE imm_anime SET episodes_total = ? WHERE anime_id = ?').run(
|
||||
episodesTotal,
|
||||
animeId,
|
||||
);
|
||||
}
|
||||
return animeId;
|
||||
}
|
||||
|
||||
export function seedVideo(
|
||||
db: DatabaseSync,
|
||||
animeId: number | null,
|
||||
name: string,
|
||||
options: { watched?: boolean } = {},
|
||||
): number {
|
||||
const videoId = getOrCreateVideoRecord(db, `local:/tmp/${name}.mkv`, {
|
||||
canonicalTitle: name,
|
||||
sourcePath: `/tmp/${name}.mkv`,
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
if (animeId !== null) {
|
||||
linkVideoToAnimeRecord(db, videoId, {
|
||||
animeId,
|
||||
parsedBasename: `${name}.mkv`,
|
||||
parsedTitle: name,
|
||||
parsedSeason: 1,
|
||||
parsedEpisode: 1,
|
||||
parserSource: 'test',
|
||||
parserConfidence: 1,
|
||||
parseMetadataJson: null,
|
||||
});
|
||||
}
|
||||
if (options.watched) {
|
||||
db.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?').run(videoId);
|
||||
}
|
||||
return videoId;
|
||||
}
|
||||
|
||||
export function seedEndedSession(
|
||||
db: DatabaseSync,
|
||||
videoId: number,
|
||||
startedAtMs: number,
|
||||
metrics: { activeMs: number; cards?: number; lines?: number; tokens?: number },
|
||||
): number {
|
||||
const sessionId = startSessionRecord(db, videoId, startedAtMs).sessionId;
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE imm_sessions SET
|
||||
ended_at_ms = ?,
|
||||
active_watched_ms = ?,
|
||||
total_watched_ms = ?,
|
||||
cards_mined = ?,
|
||||
lines_seen = ?,
|
||||
tokens_seen = ?
|
||||
WHERE session_id = ?
|
||||
`,
|
||||
).run(
|
||||
toDbTimestamp(startedAtMs + metrics.activeMs),
|
||||
metrics.activeMs,
|
||||
metrics.activeMs,
|
||||
metrics.cards ?? 0,
|
||||
metrics.lines ?? 0,
|
||||
metrics.tokens ?? 0,
|
||||
sessionId,
|
||||
);
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
// libsql attaches a per-query `_metadata` property to result rows; strip it so
|
||||
// row snapshots can be compared with deepEqual.
|
||||
export function cleanRow<T>(row: unknown): T {
|
||||
const { _metadata: _ignored, ...rest } = row as Record<string, unknown>;
|
||||
return rest as T;
|
||||
}
|
||||
|
||||
export interface GlobalSnapshot {
|
||||
total_sessions: number;
|
||||
total_active_ms: number;
|
||||
total_cards: number;
|
||||
active_days: number;
|
||||
episodes_started: number;
|
||||
episodes_completed: number;
|
||||
anime_completed: number;
|
||||
}
|
||||
|
||||
export function snapshotGlobal(db: DatabaseSync): GlobalSnapshot {
|
||||
const row = db
|
||||
.prepare(
|
||||
`SELECT total_sessions, total_active_ms, total_cards, active_days,
|
||||
episodes_started, episodes_completed, anime_completed
|
||||
FROM imm_lifetime_global WHERE global_id = 1`,
|
||||
)
|
||||
.get();
|
||||
return cleanRow<GlobalSnapshot>(row);
|
||||
}
|
||||
|
||||
export function snapshotMedia(db: DatabaseSync): unknown[] {
|
||||
return db
|
||||
.prepare(
|
||||
`SELECT video_id, total_sessions, total_active_ms, total_cards,
|
||||
total_lines_seen, total_tokens_seen, completed,
|
||||
CAST(first_watched_ms AS REAL) AS first_watched,
|
||||
CAST(last_watched_ms AS REAL) AS last_watched
|
||||
FROM imm_lifetime_media ORDER BY video_id`,
|
||||
)
|
||||
.all()
|
||||
.map((row) => cleanRow(row));
|
||||
}
|
||||
|
||||
export function snapshotAnime(db: DatabaseSync): unknown[] {
|
||||
return db
|
||||
.prepare(
|
||||
`SELECT anime_id, total_sessions, total_active_ms, total_cards,
|
||||
total_lines_seen, total_tokens_seen, episodes_started, episodes_completed,
|
||||
CAST(first_watched_ms AS REAL) AS first_watched,
|
||||
CAST(last_watched_ms AS REAL) AS last_watched
|
||||
FROM imm_lifetime_anime ORDER BY anime_id`,
|
||||
)
|
||||
.all()
|
||||
.map((row) => cleanRow(row));
|
||||
}
|
||||
@@ -50,6 +50,7 @@ import {
|
||||
updateAnimeAnilistInfo,
|
||||
upsertCoverArt,
|
||||
} from '../query-maintenance.js';
|
||||
import { deleteMaintenanceBatch } from '../query-delete-maintenance.js';
|
||||
import { getLocalEpochDay } from '../query-shared.js';
|
||||
import { EVENT_CARD_MINED, EVENT_SUBTITLE_LINE, SOURCE_TYPE_LOCAL } from '../types.js';
|
||||
|
||||
@@ -985,3 +986,197 @@ test('split maintenance helpers delete multiple sessions and whole videos with d
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('delete maintenance batch preserves retained data across overlapping session, video, and anime targets', () => {
|
||||
const { db, dbPath, stmts } = createDb();
|
||||
|
||||
try {
|
||||
const retainedAnimeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: 'Retained Anime',
|
||||
canonicalTitle: 'Retained Anime',
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: null,
|
||||
});
|
||||
const deletedAnimeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: 'Deleted Anime',
|
||||
canonicalTitle: 'Deleted Anime',
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: null,
|
||||
});
|
||||
const retainedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/batch-retain.mkv', {
|
||||
canonicalTitle: 'Batch Retain',
|
||||
sourcePath: '/tmp/batch-retain.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
const deletedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/batch-video.mkv', {
|
||||
canonicalTitle: 'Batch Video',
|
||||
sourcePath: '/tmp/batch-video.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
const animeVideoId = getOrCreateVideoRecord(db, 'local:/tmp/batch-anime.mkv', {
|
||||
canonicalTitle: 'Batch Anime',
|
||||
sourcePath: '/tmp/batch-anime.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
for (const [videoId, animeId, episode] of [
|
||||
[retainedVideoId, retainedAnimeId, 1],
|
||||
[deletedVideoId, retainedAnimeId, 2],
|
||||
[animeVideoId, deletedAnimeId, 1],
|
||||
] as const) {
|
||||
linkVideoToAnimeRecord(db, videoId, {
|
||||
animeId,
|
||||
parsedBasename: `batch-${episode}.mkv`,
|
||||
parsedTitle: animeId === retainedAnimeId ? 'Retained Anime' : 'Deleted Anime',
|
||||
parsedSeason: 1,
|
||||
parsedEpisode: episode,
|
||||
parserSource: 'test',
|
||||
parserConfidence: 1,
|
||||
parseMetadataJson: null,
|
||||
});
|
||||
}
|
||||
|
||||
const startedAtMs = 1_700_000_000_000;
|
||||
const deletedSessionId = startSessionRecord(db, retainedVideoId, startedAtMs).sessionId;
|
||||
const retainedSessionId = startSessionRecord(
|
||||
db,
|
||||
retainedVideoId,
|
||||
startedAtMs + 1_000,
|
||||
).sessionId;
|
||||
const videoSessionId = startSessionRecord(db, deletedVideoId, startedAtMs + 2_000).sessionId;
|
||||
const animeSessionId = startSessionRecord(db, animeVideoId, startedAtMs + 3_000).sessionId;
|
||||
for (const [sessionId, sessionStartedAtMs] of [
|
||||
[deletedSessionId, startedAtMs],
|
||||
[retainedSessionId, startedAtMs + 1_000],
|
||||
[videoSessionId, startedAtMs + 2_000],
|
||||
[animeSessionId, startedAtMs + 3_000],
|
||||
] as const) {
|
||||
finalizeSessionMetrics(db, sessionId, sessionStartedAtMs);
|
||||
}
|
||||
|
||||
for (const [index, sessionId, videoId, animeId] of [
|
||||
[1, deletedSessionId, retainedVideoId, retainedAnimeId],
|
||||
[2, retainedSessionId, retainedVideoId, retainedAnimeId],
|
||||
[3, videoSessionId, deletedVideoId, retainedAnimeId],
|
||||
[4, animeSessionId, animeVideoId, deletedAnimeId],
|
||||
] as const) {
|
||||
insertWordOccurrence(db, stmts, {
|
||||
sessionId,
|
||||
videoId,
|
||||
animeId,
|
||||
lineIndex: index,
|
||||
text: '猫日',
|
||||
word: { headword: '猫', word: '猫', reading: 'ねこ' },
|
||||
});
|
||||
insertKanjiOccurrence(db, stmts, {
|
||||
sessionId,
|
||||
videoId,
|
||||
animeId,
|
||||
lineIndex: index + 10,
|
||||
text: '猫日',
|
||||
kanji: '日',
|
||||
});
|
||||
}
|
||||
|
||||
const rollupDay = getLocalEpochDay(db, startedAtMs);
|
||||
const rollupMonth = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT CAST(strftime('%Y%m', CAST(? AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) AS rollupMonth`,
|
||||
)
|
||||
.get(startedAtMs) as { rollupMonth: number }
|
||||
).rollupMonth;
|
||||
for (const videoId of [retainedVideoId, deletedVideoId, animeVideoId]) {
|
||||
db.prepare(
|
||||
`INSERT INTO imm_daily_rollups (
|
||||
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, 99, 99, 99, 99, 99, ?, ?)`,
|
||||
).run(rollupDay, videoId, startedAtMs, startedAtMs);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_monthly_rollups (
|
||||
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, 99, 99, 99, 99, 99, ?, ?)`,
|
||||
).run(rollupMonth, videoId, startedAtMs, startedAtMs);
|
||||
}
|
||||
|
||||
deleteMaintenanceBatch(db, [
|
||||
{ kind: 'session', sessionId: deletedSessionId },
|
||||
{ kind: 'session', sessionId: videoSessionId },
|
||||
{ kind: 'video', videoId: deletedVideoId },
|
||||
{ kind: 'video', videoId: animeVideoId },
|
||||
{ kind: 'anime', animeId: deletedAnimeId },
|
||||
]);
|
||||
|
||||
assert.deepEqual(db.prepare('SELECT session_id FROM imm_sessions').all(), [
|
||||
{ session_id: retainedSessionId },
|
||||
]);
|
||||
assert.deepEqual(db.prepare('SELECT video_id FROM imm_videos').all(), [
|
||||
{ video_id: retainedVideoId },
|
||||
]);
|
||||
assert.deepEqual(db.prepare('SELECT anime_id FROM imm_anime').all(), [
|
||||
{ anime_id: retainedAnimeId },
|
||||
]);
|
||||
assert.equal(
|
||||
(
|
||||
db.prepare(`SELECT frequency FROM imm_words WHERE headword = '猫'`).get() as {
|
||||
frequency: number;
|
||||
}
|
||||
).frequency,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
db.prepare(`SELECT frequency FROM imm_kanji WHERE kanji = '日'`).get() as {
|
||||
frequency: number;
|
||||
}
|
||||
).frequency,
|
||||
1,
|
||||
);
|
||||
assert.deepEqual(
|
||||
db.prepare('SELECT video_id, total_sessions FROM imm_daily_rollups').all() as Array<{
|
||||
video_id: number;
|
||||
total_sessions: number;
|
||||
}>,
|
||||
[{ video_id: retainedVideoId, total_sessions: 1 }],
|
||||
);
|
||||
assert.deepEqual(
|
||||
db.prepare('SELECT video_id, total_sessions FROM imm_monthly_rollups').all() as Array<{
|
||||
video_id: number;
|
||||
total_sessions: number;
|
||||
}>,
|
||||
[{ video_id: retainedVideoId, total_sessions: 1 }],
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('delete maintenance batch chunks id lists below the SQLite variable limit', () => {
|
||||
const { db, dbPath } = createDb();
|
||||
|
||||
try {
|
||||
const ids = Array.from({ length: 32_767 }, (_, index) => index + 1);
|
||||
|
||||
assert.doesNotThrow(() => {
|
||||
deleteMaintenanceBatch(db, [
|
||||
{ kind: 'sessions', sessionIds: ids },
|
||||
...ids.map((videoId) => ({ kind: 'video' as const, videoId })),
|
||||
...ids.map((animeId) => ({ kind: 'anime' as const, animeId })),
|
||||
]);
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -8,6 +8,7 @@ export interface AnimeSeasonRepairSummary {
|
||||
repaired: number;
|
||||
movedVideos: number;
|
||||
deletedAnimeRows: number;
|
||||
affectedAnimeIds: number[];
|
||||
}
|
||||
|
||||
interface AnimeRow {
|
||||
@@ -38,6 +39,7 @@ function emptySummary(scanned = 0): AnimeSeasonRepairSummary {
|
||||
repaired: 0,
|
||||
movedVideos: 0,
|
||||
deletedAnimeRows: 0,
|
||||
affectedAnimeIds: [],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -49,6 +51,7 @@ function mergeSummary(
|
||||
target.repaired += source.repaired;
|
||||
target.movedVideos += source.movedVideos;
|
||||
target.deletedAnimeRows += source.deletedAnimeRows;
|
||||
target.affectedAnimeIds = [...new Set([...target.affectedAnimeIds, ...source.affectedAnimeIds])];
|
||||
return target;
|
||||
}
|
||||
|
||||
@@ -184,6 +187,7 @@ function redistributeAnimeRowByParsedSeasonsInTransaction(
|
||||
|
||||
const videos = getParsedVideos(db, animeId);
|
||||
const summary = emptySummary(1);
|
||||
summary.affectedAnimeIds.push(animeId);
|
||||
const updatedAt = toDbTimestamp(nowMs());
|
||||
const targetBySeason = new Map<number, number>();
|
||||
|
||||
@@ -233,6 +237,9 @@ function redistributeAnimeRowByParsedSeasonsInTransaction(
|
||||
|
||||
if (videoUpdate.changes > 0 || lineUpdate.changes > 0) {
|
||||
summary.movedVideos += 1;
|
||||
if (!summary.affectedAnimeIds.includes(targetAnimeId)) {
|
||||
summary.affectedAnimeIds.push(targetAnimeId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { DeleteMaintenanceScheduler } from './delete-maintenance-scheduler';
|
||||
import type { DeleteMaintenanceTask } from './delete-maintenance';
|
||||
|
||||
test('scheduler batches same-turn requests and balances busy state', async () => {
|
||||
const tasks: DeleteMaintenanceTask[] = [];
|
||||
const states: string[] = [];
|
||||
const scheduler = new DeleteMaintenanceScheduler({
|
||||
batchWindowMs: 0,
|
||||
runTask: async (task) => {
|
||||
tasks.push(task);
|
||||
},
|
||||
onBusy: () => states.push('busy'),
|
||||
onIdle: () => states.push('idle'),
|
||||
});
|
||||
|
||||
const first = scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 }));
|
||||
const second = scheduler.enqueue(() => ({ kind: 'sessions', sessionIds: [2, 3] }));
|
||||
const third = scheduler.enqueue(() => null);
|
||||
await Promise.all([first, second, third]);
|
||||
|
||||
assert.deepEqual(tasks, [
|
||||
{
|
||||
kind: 'batch',
|
||||
tasks: [
|
||||
{ kind: 'session', sessionId: 1 },
|
||||
{ kind: 'sessions', sessionIds: [2, 3] },
|
||||
],
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(states, ['busy', 'idle']);
|
||||
});
|
||||
|
||||
test('scheduler rejects enqueue after destruction without entering busy state', async () => {
|
||||
let busyCalls = 0;
|
||||
let runCalls = 0;
|
||||
const scheduler = new DeleteMaintenanceScheduler({
|
||||
batchWindowMs: 0,
|
||||
runTask: async () => {
|
||||
runCalls += 1;
|
||||
},
|
||||
onBusy: () => {
|
||||
busyCalls += 1;
|
||||
},
|
||||
onIdle: () => {},
|
||||
});
|
||||
scheduler.destroy();
|
||||
|
||||
await assert.rejects(
|
||||
scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 })),
|
||||
/shutting down/,
|
||||
);
|
||||
assert.equal(busyCalls, 0);
|
||||
assert.equal(runCalls, 0);
|
||||
});
|
||||
|
||||
test('scheduler rejects every request in a batch when the maintenance task fails', async () => {
|
||||
const failure = new Error('maintenance failed');
|
||||
const scheduler = new DeleteMaintenanceScheduler({
|
||||
batchWindowMs: 0,
|
||||
runTask: async () => {
|
||||
throw failure;
|
||||
},
|
||||
onBusy: () => {},
|
||||
onIdle: () => {},
|
||||
});
|
||||
|
||||
const first = scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 }));
|
||||
const second = scheduler.enqueue(() => ({ kind: 'session', sessionId: 2 }));
|
||||
|
||||
const results = await Promise.allSettled([first, second]);
|
||||
assert.deepEqual(
|
||||
results.map((result) => (result.status === 'rejected' ? result.reason : null)),
|
||||
[failure, failure],
|
||||
);
|
||||
});
|
||||
|
||||
test('scheduler rejects only the request whose task resolution fails', async () => {
|
||||
const failure = new Error('resolution failed');
|
||||
const tasks: DeleteMaintenanceTask[] = [];
|
||||
const scheduler = new DeleteMaintenanceScheduler({
|
||||
batchWindowMs: 0,
|
||||
runTask: async (task) => {
|
||||
tasks.push(task);
|
||||
},
|
||||
onBusy: () => {},
|
||||
onIdle: () => {},
|
||||
});
|
||||
|
||||
const failed = scheduler.enqueue(() => {
|
||||
throw failure;
|
||||
});
|
||||
const succeeded = scheduler.enqueue(() => ({ kind: 'session', sessionId: 2 }));
|
||||
|
||||
const results = await Promise.allSettled([failed, succeeded]);
|
||||
assert.equal(results[0]?.status, 'rejected');
|
||||
assert.equal(results[0]?.status === 'rejected' ? results[0].reason : null, failure);
|
||||
assert.equal(results[1]?.status, 'fulfilled');
|
||||
assert.deepEqual(tasks, [{ kind: 'session', sessionId: 2 }]);
|
||||
});
|
||||
|
||||
test('scheduler does not schedule another drain when the queue is empty', async () => {
|
||||
const originalSetTimeout = globalThis.setTimeout;
|
||||
let timerCalls = 0;
|
||||
globalThis.setTimeout = ((handler: TimerHandler, timeout?: number, ...args: unknown[]) => {
|
||||
timerCalls += 1;
|
||||
return originalSetTimeout(handler, timeout, ...args);
|
||||
}) as typeof setTimeout;
|
||||
|
||||
try {
|
||||
const scheduler = new DeleteMaintenanceScheduler({
|
||||
batchWindowMs: 0,
|
||||
runTask: async () => {},
|
||||
onBusy: () => {},
|
||||
onIdle: () => {},
|
||||
});
|
||||
|
||||
await scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 }));
|
||||
assert.equal(timerCalls, 1);
|
||||
} finally {
|
||||
globalThis.setTimeout = originalSetTimeout;
|
||||
}
|
||||
});
|
||||
|
||||
test('scheduler serializes batches and rejects requests queued at destruction', async () => {
|
||||
const releases: Array<() => void> = [];
|
||||
let activeTasks = 0;
|
||||
let maxActiveTasks = 0;
|
||||
const scheduler = new DeleteMaintenanceScheduler({
|
||||
batchWindowMs: 0,
|
||||
runTask: async () => {
|
||||
activeTasks += 1;
|
||||
maxActiveTasks = Math.max(maxActiveTasks, activeTasks);
|
||||
await new Promise<void>((resolve) => releases.push(resolve));
|
||||
activeTasks -= 1;
|
||||
},
|
||||
onBusy: () => {},
|
||||
onIdle: () => {},
|
||||
});
|
||||
|
||||
const first = scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 }));
|
||||
const maxPollAttempts = 100;
|
||||
let pollAttempts = 0;
|
||||
while (releases.length === 0 && pollAttempts < maxPollAttempts) {
|
||||
pollAttempts += 1;
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
assert.ok(
|
||||
releases.length > 0,
|
||||
`runTask did not produce a release after ${maxPollAttempts} polling attempts`,
|
||||
);
|
||||
const queued = scheduler.enqueue(() => ({ kind: 'session', sessionId: 2 }));
|
||||
scheduler.destroy();
|
||||
|
||||
await assert.rejects(queued, /shutting down/);
|
||||
releases[0]?.();
|
||||
await first;
|
||||
assert.equal(maxActiveTasks, 1);
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import type { DeleteMaintenanceOperation, DeleteMaintenanceTask } from './delete-maintenance';
|
||||
|
||||
type ResolveDeleteMaintenanceOperation = () =>
|
||||
| DeleteMaintenanceOperation
|
||||
| null
|
||||
| Promise<DeleteMaintenanceOperation | null>;
|
||||
|
||||
interface PendingDeleteMaintenanceRequest {
|
||||
resolveTask: ResolveDeleteMaintenanceOperation;
|
||||
resolve: () => void;
|
||||
reject: (error: unknown) => void;
|
||||
}
|
||||
|
||||
interface DeleteMaintenanceSchedulerOptions {
|
||||
batchWindowMs: number;
|
||||
runTask: (task: DeleteMaintenanceTask) => Promise<void>;
|
||||
onBusy: () => void;
|
||||
onIdle: () => void;
|
||||
}
|
||||
|
||||
export class DeleteMaintenanceScheduler {
|
||||
private readonly pendingRequests: PendingDeleteMaintenanceRequest[] = [];
|
||||
private running = false;
|
||||
private drainTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private pendingTaskCount = 0;
|
||||
private destroyed = false;
|
||||
|
||||
constructor(private readonly options: DeleteMaintenanceSchedulerOptions) {}
|
||||
|
||||
enqueue(resolveTask: ResolveDeleteMaintenanceOperation): Promise<void> {
|
||||
if (this.destroyed) {
|
||||
return Promise.reject(new Error('Immersion tracker is shutting down'));
|
||||
}
|
||||
|
||||
if (this.pendingTaskCount === 0) this.options.onBusy();
|
||||
this.pendingTaskCount += 1;
|
||||
|
||||
const result = new Promise<void>((resolve, reject) => {
|
||||
this.pendingRequests.push({ resolveTask, resolve, reject });
|
||||
this.scheduleDrain();
|
||||
});
|
||||
|
||||
return result.finally(() => {
|
||||
this.pendingTaskCount -= 1;
|
||||
if (this.pendingTaskCount === 0) this.options.onIdle();
|
||||
});
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
if (this.drainTimer) {
|
||||
clearTimeout(this.drainTimer);
|
||||
this.drainTimer = null;
|
||||
}
|
||||
const error = new Error('Immersion tracker is shutting down');
|
||||
for (const request of this.pendingRequests.splice(0)) request.reject(error);
|
||||
}
|
||||
|
||||
private scheduleDrain(): void {
|
||||
if (this.destroyed || this.running || this.drainTimer || this.pendingRequests.length === 0) {
|
||||
return;
|
||||
}
|
||||
this.drainTimer = setTimeout(() => {
|
||||
this.drainTimer = null;
|
||||
void this.drain();
|
||||
}, this.options.batchWindowMs);
|
||||
}
|
||||
|
||||
private async drain(): Promise<void> {
|
||||
if (this.running || this.pendingRequests.length === 0) return;
|
||||
this.running = true;
|
||||
const requests = this.pendingRequests.splice(0);
|
||||
const runnable: Array<{
|
||||
request: PendingDeleteMaintenanceRequest;
|
||||
task: DeleteMaintenanceOperation;
|
||||
}> = [];
|
||||
|
||||
for (const request of requests) {
|
||||
try {
|
||||
const task = await request.resolveTask();
|
||||
if (task) runnable.push({ request, task });
|
||||
else request.resolve();
|
||||
} catch (error) {
|
||||
request.reject(error);
|
||||
}
|
||||
}
|
||||
|
||||
if (runnable.length > 0) {
|
||||
const task: DeleteMaintenanceTask =
|
||||
runnable.length === 1
|
||||
? runnable[0]!.task
|
||||
: { kind: 'batch', tasks: runnable.map((entry) => entry.task) };
|
||||
try {
|
||||
await this.options.runTask(task);
|
||||
for (const { request } of runnable) request.resolve();
|
||||
} catch (error) {
|
||||
for (const { request } of runnable) request.reject(error);
|
||||
}
|
||||
}
|
||||
|
||||
this.running = false;
|
||||
this.scheduleDrain();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
DeleteMaintenanceWorkerRuntime,
|
||||
resolveDeleteMaintenanceWorkerPath,
|
||||
} from './delete-maintenance-worker-runtime';
|
||||
import { executeDeleteMaintenanceTask } from './delete-maintenance';
|
||||
import { startSessionRecord } from './session';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas, ensureSchema, getOrCreateVideoRecord } from './storage';
|
||||
|
||||
type FakeWorkerListener = (value: never) => void;
|
||||
|
||||
function createFakeWorker() {
|
||||
const listeners = new Map<string, FakeWorkerListener>();
|
||||
const terminationState = { calls: 0 };
|
||||
const worker = {
|
||||
once(event: string, listener: FakeWorkerListener) {
|
||||
listeners.set(event, listener);
|
||||
return this;
|
||||
},
|
||||
terminate: async () => {
|
||||
terminationState.calls += 1;
|
||||
return 0;
|
||||
},
|
||||
};
|
||||
return { worker, listeners, terminationState };
|
||||
}
|
||||
|
||||
type FakeWorker = ReturnType<typeof createFakeWorker>['worker'];
|
||||
|
||||
test('a delete batch never runs a full lifetime rebuild', () => {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-delete-batch-test-'));
|
||||
const dbPath = path.join(tempDir, 'immersion.sqlite');
|
||||
let db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
applyPragmas(db);
|
||||
ensureSchema(db);
|
||||
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/batch-delete.mkv', {
|
||||
canonicalTitle: 'Batch Delete',
|
||||
sourcePath: '/tmp/batch-delete.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: 1,
|
||||
});
|
||||
const firstSessionId = startSessionRecord(db, videoId, 1_000).sessionId;
|
||||
const secondSessionId = startSessionRecord(db, videoId, 2_000).sessionId;
|
||||
const deletedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/batch-delete-video.mkv', {
|
||||
canonicalTitle: 'Batch Delete Video',
|
||||
sourcePath: '/tmp/batch-delete-video.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: 1,
|
||||
});
|
||||
startSessionRecord(db, deletedVideoId, 3_000);
|
||||
db.exec(`
|
||||
CREATE TABLE delete_rebuild_audit (id INTEGER PRIMARY KEY);
|
||||
CREATE TRIGGER count_delete_lifetime_rebuild
|
||||
AFTER UPDATE OF last_rebuilt_ms ON imm_lifetime_global
|
||||
BEGIN
|
||||
INSERT INTO delete_rebuild_audit (id) VALUES (NULL);
|
||||
END;
|
||||
`);
|
||||
db.close();
|
||||
|
||||
executeDeleteMaintenanceTask(dbPath, {
|
||||
kind: 'batch',
|
||||
tasks: [
|
||||
{ kind: 'session', sessionId: firstSessionId },
|
||||
{ kind: 'video', videoId: deletedVideoId },
|
||||
],
|
||||
});
|
||||
|
||||
db = new Database(dbPath);
|
||||
const audit = db.prepare('SELECT COUNT(*) AS total FROM delete_rebuild_audit').get() as {
|
||||
total: number;
|
||||
};
|
||||
const retainedSession = db
|
||||
.prepare('SELECT session_id AS sessionId FROM imm_sessions WHERE video_id = ?')
|
||||
.get(videoId) as { sessionId: number } | null;
|
||||
const deletedVideo = db
|
||||
.prepare('SELECT video_id AS videoId FROM imm_videos WHERE video_id = ?')
|
||||
.get(deletedVideoId) as { videoId: number } | null;
|
||||
assert.equal(retainedSession?.sessionId, secondSessionId);
|
||||
assert.equal(deletedVideo, undefined);
|
||||
assert.equal(
|
||||
audit.total,
|
||||
0,
|
||||
'delete maintenance subtracts incrementally instead of rewriting last_rebuilt_ms',
|
||||
);
|
||||
} finally {
|
||||
try {
|
||||
db.close();
|
||||
} catch {
|
||||
// The setup connection closes before maintenance runs.
|
||||
}
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('delete worker module resolves in the current layout', () => {
|
||||
// If this resolves to null, every delete silently runs on the serving thread
|
||||
// and blocks the stats API for the whole maintenance run.
|
||||
const workerPath = resolveDeleteMaintenanceWorkerPath();
|
||||
assert.ok(workerPath, 'delete-maintenance worker module must resolve');
|
||||
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
|
||||
});
|
||||
|
||||
test(
|
||||
'compiled delete worker removes data through its separate database connection',
|
||||
{ skip: resolveDeleteMaintenanceWorkerPath() === null },
|
||||
async () => {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-delete-worker-test-'));
|
||||
const dbPath = path.join(tempDir, 'immersion.sqlite');
|
||||
const runtime = new DeleteMaintenanceWorkerRuntime();
|
||||
let db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
applyPragmas(db);
|
||||
ensureSchema(db);
|
||||
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/worker-delete.mkv', {
|
||||
canonicalTitle: 'Worker Delete',
|
||||
sourcePath: '/tmp/worker-delete.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: 1,
|
||||
});
|
||||
const firstSessionId = startSessionRecord(db, videoId, 1_000).sessionId;
|
||||
const secondSessionId = startSessionRecord(db, videoId, 2_000).sessionId;
|
||||
db.close();
|
||||
|
||||
await runtime.run(dbPath, {
|
||||
kind: 'batch',
|
||||
tasks: [
|
||||
{ kind: 'session', sessionId: firstSessionId },
|
||||
{ kind: 'session', sessionId: secondSessionId },
|
||||
],
|
||||
});
|
||||
|
||||
db = new Database(dbPath);
|
||||
const row = db
|
||||
.prepare('SELECT COUNT(*) AS total FROM imm_sessions WHERE video_id = ?')
|
||||
.get(videoId) as { total: number };
|
||||
assert.equal(row.total, 0);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
try {
|
||||
db.close();
|
||||
} catch {
|
||||
// The setup connection is already closed before the worker starts.
|
||||
}
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test('worker runtime warns before falling back when no emitted worker is available', async () => {
|
||||
const warnings: unknown[][] = [];
|
||||
const fallbackTasks: unknown[] = [];
|
||||
const runtime = new DeleteMaintenanceWorkerRuntime({
|
||||
resolveWorkerPath: () => null,
|
||||
warn: (...args) => warnings.push(args),
|
||||
executeFallback: (_dbPath, task) => fallbackTasks.push(task),
|
||||
});
|
||||
|
||||
await runtime.run('/tmp/fallback.sqlite', { kind: 'session', sessionId: 1 });
|
||||
|
||||
assert.equal(warnings.length, 1);
|
||||
assert.match(String(warnings[0]?.[0]), /worker unavailable/i);
|
||||
assert.deepEqual(fallbackTasks, [{ kind: 'session', sessionId: 1 }]);
|
||||
});
|
||||
|
||||
test('worker runtime terminates a worker after successful settlement', async () => {
|
||||
const { worker, listeners, terminationState } = createFakeWorker();
|
||||
const runtime = new DeleteMaintenanceWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/delete-worker.js',
|
||||
createWorker: async () => worker,
|
||||
});
|
||||
|
||||
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
listeners.get('message')?.({ ok: true } as never);
|
||||
await result;
|
||||
|
||||
assert.equal(terminationState.calls, 1);
|
||||
});
|
||||
|
||||
test('worker runtime falls back to the current thread when the worker crashes', async () => {
|
||||
const { worker, listeners, terminationState } = createFakeWorker();
|
||||
const fallbackTasks: unknown[] = [];
|
||||
const warnings: string[] = [];
|
||||
const runtime = new DeleteMaintenanceWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/delete-worker.js',
|
||||
createWorker: async () => worker,
|
||||
executeFallback: (dbPath, task) => {
|
||||
fallbackTasks.push({ dbPath, task });
|
||||
},
|
||||
warn: (message) => {
|
||||
warnings.push(message);
|
||||
},
|
||||
});
|
||||
|
||||
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
listeners.get('error')?.(new Error('worker failed') as never);
|
||||
|
||||
await result;
|
||||
assert.equal(terminationState.calls, 1);
|
||||
assert.deepEqual(fallbackTasks, [
|
||||
{ dbPath: '/tmp/test.sqlite', task: { kind: 'session', sessionId: 1 } },
|
||||
]);
|
||||
assert.equal(warnings.length, 1);
|
||||
});
|
||||
|
||||
test('worker runtime falls back when a worker exits cleanly without a response', async () => {
|
||||
const { worker, listeners, terminationState } = createFakeWorker();
|
||||
const fallbackTasks: unknown[] = [];
|
||||
const runtime = new DeleteMaintenanceWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/delete-worker.js',
|
||||
createWorker: async () => worker,
|
||||
executeFallback: (dbPath, task) => {
|
||||
fallbackTasks.push({ dbPath, task });
|
||||
},
|
||||
});
|
||||
const task = { kind: 'session' as const, sessionId: 1 };
|
||||
|
||||
const result = runtime.run('/tmp/test.sqlite', task);
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
listeners.get('exit')?.(0 as never);
|
||||
|
||||
await result;
|
||||
assert.equal(terminationState.calls, 1);
|
||||
assert.deepEqual(fallbackTasks, [{ dbPath: '/tmp/test.sqlite', task }]);
|
||||
});
|
||||
|
||||
test('worker runtime surfaces a task failure without rerunning it', async () => {
|
||||
const { worker, listeners, terminationState } = createFakeWorker();
|
||||
const fallbackTasks: unknown[] = [];
|
||||
const runtime = new DeleteMaintenanceWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/delete-worker.js',
|
||||
createWorker: async () => worker,
|
||||
executeFallback: () => {
|
||||
fallbackTasks.push('ran');
|
||||
},
|
||||
});
|
||||
|
||||
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
listeners.get('message')?.({ ok: false, error: 'constraint violated' } as never);
|
||||
|
||||
await assert.rejects(result, /constraint violated/);
|
||||
assert.equal(terminationState.calls, 1);
|
||||
assert.equal(fallbackTasks.length, 0);
|
||||
});
|
||||
|
||||
test('worker runtime terminates a worker created after shutdown begins', async () => {
|
||||
const { worker, listeners, terminationState } = createFakeWorker();
|
||||
const createGate: { resolve?: (worker: FakeWorker) => void } = {};
|
||||
const fallbackTasks: unknown[] = [];
|
||||
const runtime = new DeleteMaintenanceWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/delete-worker.js',
|
||||
createWorker: () =>
|
||||
new Promise((resolve) => {
|
||||
createGate.resolve = resolve;
|
||||
}),
|
||||
executeFallback: (_dbPath, task) => fallbackTasks.push(task),
|
||||
});
|
||||
|
||||
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
runtime.destroy();
|
||||
createGate.resolve?.(worker);
|
||||
|
||||
await assert.rejects(result, /shut down/);
|
||||
assert.equal(terminationState.calls, 1);
|
||||
assert.equal(listeners.size, 0);
|
||||
assert.deepEqual(fallbackTasks, []);
|
||||
});
|
||||
|
||||
test('worker runtime does not fall back when worker creation fails during shutdown', async () => {
|
||||
const createGate: { reject?: (error: Error) => void } = {};
|
||||
const fallbackTasks: unknown[] = [];
|
||||
const runtime = new DeleteMaintenanceWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/delete-worker.js',
|
||||
createWorker: () =>
|
||||
new Promise((_resolve, reject) => {
|
||||
createGate.reject = reject;
|
||||
}),
|
||||
executeFallback: (_dbPath, task) => fallbackTasks.push(task),
|
||||
});
|
||||
|
||||
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
runtime.destroy();
|
||||
createGate.reject?.(new Error('creation failed'));
|
||||
|
||||
await assert.rejects(result, /shut down/);
|
||||
assert.deepEqual(fallbackTasks, []);
|
||||
});
|
||||
@@ -0,0 +1,150 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createLogger } from '../../../logger';
|
||||
import { executeDeleteMaintenanceTask, type DeleteMaintenanceTask } from './delete-maintenance';
|
||||
|
||||
interface DeleteMaintenanceWorkerResponse {
|
||||
ok?: unknown;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
export type RunDeleteMaintenanceTask = (
|
||||
dbPath: string,
|
||||
task: DeleteMaintenanceTask,
|
||||
) => Promise<void>;
|
||||
|
||||
interface DeleteMaintenanceWorkerHandle {
|
||||
once(event: 'message', listener: (message: DeleteMaintenanceWorkerResponse) => void): this;
|
||||
once(event: 'error', listener: (error: Error) => void): this;
|
||||
once(event: 'exit', listener: (code: number) => void): this;
|
||||
terminate(): Promise<number>;
|
||||
}
|
||||
|
||||
interface DeleteMaintenanceWorkerRuntimeOptions {
|
||||
resolveWorkerPath?: () => string | null;
|
||||
createWorker?: (
|
||||
workerPath: string,
|
||||
workerData: { dbPath: string; task: DeleteMaintenanceTask },
|
||||
) => Promise<DeleteMaintenanceWorkerHandle>;
|
||||
executeFallback?: typeof executeDeleteMaintenanceTask;
|
||||
warn?: (message: string, ...meta: unknown[]) => void;
|
||||
}
|
||||
|
||||
export function resolveDeleteMaintenanceWorkerPath(): string | null {
|
||||
// When the process runs TypeScript directly (Bun from source), the emitted
|
||||
// .js sibling doesn't exist — spawn the .ts module instead, which such
|
||||
// runtimes transpile for workers too. Compiled layouts keep using the .js.
|
||||
const fileName = __filename.endsWith('.ts')
|
||||
? 'delete-maintenance-worker-thread.ts'
|
||||
: 'delete-maintenance-worker-thread.js';
|
||||
const workerPath = path.join(__dirname, fileName);
|
||||
return fs.existsSync(workerPath) ? workerPath : null;
|
||||
}
|
||||
|
||||
const logger = createLogger('main:immersion-tracker:delete-worker');
|
||||
|
||||
export class DeleteMaintenanceWorkerRuntime {
|
||||
private readonly activeWorkers = new Set<DeleteMaintenanceWorkerHandle>();
|
||||
private destroyed = false;
|
||||
|
||||
constructor(private readonly options: DeleteMaintenanceWorkerRuntimeOptions = {}) {}
|
||||
|
||||
async run(dbPath: string, task: DeleteMaintenanceTask): Promise<void> {
|
||||
if (this.destroyed) {
|
||||
throw new Error('Delete maintenance worker is shut down');
|
||||
}
|
||||
|
||||
let worker: DeleteMaintenanceWorkerHandle;
|
||||
try {
|
||||
const workerPath = (this.options.resolveWorkerPath ?? resolveDeleteMaintenanceWorkerPath)();
|
||||
if (!workerPath) throw new Error('Emitted delete-maintenance worker module was not found');
|
||||
const createWorker =
|
||||
this.options.createWorker ??
|
||||
(async (resolvedPath, workerData) => {
|
||||
const { Worker } = await import('node:worker_threads');
|
||||
return new Worker(resolvedPath, { workerData });
|
||||
});
|
||||
worker = await createWorker(workerPath, { dbPath, task });
|
||||
} catch (error) {
|
||||
if (this.destroyed) {
|
||||
throw new Error('Delete maintenance worker is shut down');
|
||||
}
|
||||
(this.options.warn ?? logger.warn)(
|
||||
'Delete maintenance worker unavailable; running maintenance on the current thread',
|
||||
error,
|
||||
);
|
||||
(this.options.executeFallback ?? executeDeleteMaintenanceTask)(dbPath, task);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.destroyed) {
|
||||
await worker.terminate().catch(() => undefined);
|
||||
throw new Error('Delete maintenance worker is shut down');
|
||||
}
|
||||
|
||||
type WorkerOutcome =
|
||||
| { kind: 'ok' }
|
||||
| { kind: 'task-error'; detail: string }
|
||||
| { kind: 'worker-failure'; error: Error };
|
||||
|
||||
const outcome = await new Promise<WorkerOutcome>((resolve) => {
|
||||
let settled = false;
|
||||
this.activeWorkers.add(worker);
|
||||
|
||||
const settle = (result: WorkerOutcome) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
this.activeWorkers.delete(worker);
|
||||
resolve(result);
|
||||
void worker.terminate();
|
||||
};
|
||||
|
||||
worker.once('message', (message: DeleteMaintenanceWorkerResponse) => {
|
||||
if (message.ok === true) {
|
||||
settle({ kind: 'ok' });
|
||||
return;
|
||||
}
|
||||
const detail = typeof message.error === 'string' ? message.error : 'unknown worker error';
|
||||
settle({ kind: 'task-error', detail });
|
||||
});
|
||||
worker.once('error', (error) => settle({ kind: 'worker-failure', error }));
|
||||
worker.once('exit', (code) => {
|
||||
settle({
|
||||
kind: 'worker-failure',
|
||||
error: new Error(
|
||||
code === 0
|
||||
? 'Delete maintenance worker exited without a response'
|
||||
: `Delete maintenance worker exited with code ${code}`,
|
||||
),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
if (outcome.kind === 'ok') return;
|
||||
// The maintenance itself failed inside the worker — rerunning it on this
|
||||
// thread would hit the same error, so surface it instead.
|
||||
if (outcome.kind === 'task-error') {
|
||||
throw new Error(`Delete maintenance failed: ${outcome.detail}`);
|
||||
}
|
||||
if (this.destroyed) {
|
||||
throw new Error('Delete maintenance worker is shut down');
|
||||
}
|
||||
// The worker died without reporting a result (failed to load, crashed).
|
||||
// Its transaction rolled back with its connection, and a rerun re-plans
|
||||
// against the current rows, so falling back on this thread is safe.
|
||||
(this.options.warn ?? logger.warn)(
|
||||
'Delete maintenance worker failed; running maintenance on the current thread',
|
||||
outcome.error,
|
||||
);
|
||||
(this.options.executeFallback ?? executeDeleteMaintenanceTask)(dbPath, task);
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
for (const worker of this.activeWorkers) {
|
||||
void worker.terminate();
|
||||
}
|
||||
this.activeWorkers.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
import { executeDeleteMaintenanceTask, type DeleteMaintenanceTask } from './delete-maintenance';
|
||||
|
||||
interface DeleteMaintenanceWorkerData {
|
||||
dbPath: string;
|
||||
task: DeleteMaintenanceTask;
|
||||
}
|
||||
|
||||
if (!parentPort) {
|
||||
throw new Error('delete maintenance worker missing parent port');
|
||||
}
|
||||
|
||||
const port = parentPort;
|
||||
const request = workerData as DeleteMaintenanceWorkerData;
|
||||
|
||||
try {
|
||||
executeDeleteMaintenanceTask(request.dbPath, request.task);
|
||||
port.postMessage({ ok: true });
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
port.postMessage({ ok: false, error: message });
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas } from './storage';
|
||||
import {
|
||||
deleteMaintenanceBatch,
|
||||
type DeleteMaintenanceOperation,
|
||||
} from './query-delete-maintenance';
|
||||
|
||||
export type { DeleteMaintenanceOperation } from './query-delete-maintenance';
|
||||
|
||||
export type DeleteMaintenanceTask =
|
||||
| DeleteMaintenanceOperation
|
||||
| { kind: 'batch'; tasks: DeleteMaintenanceOperation[] };
|
||||
|
||||
export function executeDeleteMaintenanceTask(dbPath: string, task: DeleteMaintenanceTask): void {
|
||||
const db = new Database(dbPath);
|
||||
try {
|
||||
applyPragmas(db);
|
||||
deleteMaintenanceBatch(db, task.kind === 'batch' ? task.tasks : [task]);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
/*
|
||||
* Retroactive removal of animation-burst subtitle lines from the stats database.
|
||||
*
|
||||
* Before the live ingest gate existed, a karaoke OP recorded one line -- and one count
|
||||
* for every word in it -- per animation frame, which is enough to put an OP lyric at the
|
||||
* top of "Top Repeated Words" for good. This module finds those runs in what is already
|
||||
* stored and takes them back down to one line.
|
||||
*
|
||||
* Only timing is available here: the stored text has been stripped of ASS markup, so the
|
||||
* authoring evidence the file-level parser uses (`\t`, `\move`, karaoke timing, a
|
||||
* changing override signature) is long gone. What is left is a run of identical,
|
||||
* contiguous, short-lived lines inside a single session.
|
||||
*
|
||||
* The run has to be as long as the timing-only rule in `subtitle-cue-dedup` demands, but
|
||||
* its short frames may be as long as the animation-frame bound rather than the much
|
||||
* tighter timing-only one. A qualifying run may end with one longer hold, which is a
|
||||
* common karaoke shape. Five or more repeats of the same text, each ending where the next
|
||||
* begins, is already conclusive on its own -- no dialogue does that -- and the tighter
|
||||
* bound would walk straight past the heavier typesetting that motivated this, where
|
||||
* frames sit nearer a quarter of a second. Both bounds are options, so a cautious run can
|
||||
* ask for more, and a dry run always reports before anything is removed.
|
||||
*
|
||||
* Scope: subtitle lines, their word/kanji occurrences, and the `imm_words`/`imm_kanji`
|
||||
* aggregates those occurrences feed. Session telemetry (`lines_seen`, `tokens_seen`) and
|
||||
* the rollups derived from it are left alone; they are cumulative samples taken at record
|
||||
* time, and for sessions whose raw rows have since been pruned they cannot be recomputed.
|
||||
*/
|
||||
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import {
|
||||
ANIMATION_FRAME_MAX_SECONDS,
|
||||
DUPLICATE_CUE_GAP_TOLERANCE_SECONDS,
|
||||
MIN_STREAM_RESIDUE_FRAMES,
|
||||
MIN_TIMING_ONLY_FRAMES,
|
||||
TIMING_ONLY_FRAME_MAX_SECONDS,
|
||||
} from '../subtitle-burst-constants';
|
||||
import {
|
||||
applyLexicalRemovals,
|
||||
makePlaceholders,
|
||||
planLexicalRemovalsForLines,
|
||||
toDbTimestamp,
|
||||
} from './query-shared';
|
||||
import { nowMs } from './time';
|
||||
|
||||
const MS_PER_DAY = 86_400_000;
|
||||
/** SQLite caps bound parameters per statement; stay well under it. */
|
||||
const ID_BATCH_SIZE = 400;
|
||||
const DEFAULT_SAMPLE_LIMIT = 20;
|
||||
|
||||
export interface DuplicateSubtitleLineCleanupOptions {
|
||||
/** Only consider lines recorded within this many days. Null or omitted = all history. */
|
||||
lookbackDays?: number | null;
|
||||
/** Measure without writing. */
|
||||
dryRun?: boolean;
|
||||
/** Identical contiguous lines needed before a run counts as an animation. */
|
||||
minRunLength?: number;
|
||||
/** Longest a single event may last and still look like an animation frame. */
|
||||
maxFrameSeconds?: number;
|
||||
/** How many of the largest runs to describe in the summary. */
|
||||
sampleLimit?: number;
|
||||
}
|
||||
|
||||
export interface DuplicateSubtitleLineBurst {
|
||||
sessionId: number;
|
||||
videoId: number;
|
||||
text: string;
|
||||
/** Kept line, extended to cover the whole run. */
|
||||
keptLineId: number;
|
||||
removedLineIds: number[];
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
}
|
||||
|
||||
export interface DuplicateSubtitleLineSample {
|
||||
videoId: number;
|
||||
videoTitle: string | null;
|
||||
text: string;
|
||||
frames: number;
|
||||
removedLines: number;
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
}
|
||||
|
||||
export interface DuplicateSubtitleLineCleanupSummary {
|
||||
dryRun: boolean;
|
||||
lookbackDays: number | null;
|
||||
scannedLines: number;
|
||||
burstGroups: number;
|
||||
removedLines: number;
|
||||
removedWordOccurrences: number;
|
||||
removedKanjiOccurrences: number;
|
||||
samples: DuplicateSubtitleLineSample[];
|
||||
}
|
||||
|
||||
export interface StoredSubtitleLineRow {
|
||||
lineId: number;
|
||||
sessionId: number;
|
||||
videoId: number;
|
||||
text: string;
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
}
|
||||
|
||||
interface ResolvedBounds {
|
||||
lookbackDays: number | null;
|
||||
minRunLength: number;
|
||||
maxFrameMs: number;
|
||||
/** Shorter runs qualify only when every event sits under this much stricter bound. */
|
||||
residueMinRunLength: number;
|
||||
strictFrameMs: number;
|
||||
gapToleranceMs: number;
|
||||
sampleLimit: number;
|
||||
}
|
||||
|
||||
function resolveBounds(options: DuplicateSubtitleLineCleanupOptions): ResolvedBounds {
|
||||
const lookbackDays =
|
||||
typeof options.lookbackDays === 'number' && Number.isFinite(options.lookbackDays)
|
||||
? Math.max(1, Math.floor(options.lookbackDays))
|
||||
: null;
|
||||
const minRunLength =
|
||||
typeof options.minRunLength === 'number' && Number.isFinite(options.minRunLength)
|
||||
? Math.max(2, Math.floor(options.minRunLength))
|
||||
: MIN_TIMING_ONLY_FRAMES;
|
||||
const maxFrameSeconds =
|
||||
typeof options.maxFrameSeconds === 'number' &&
|
||||
Number.isFinite(options.maxFrameSeconds) &&
|
||||
options.maxFrameSeconds > 0
|
||||
? options.maxFrameSeconds
|
||||
: ANIMATION_FRAME_MAX_SECONDS;
|
||||
const sampleLimit =
|
||||
typeof options.sampleLimit === 'number' && options.sampleLimit >= 0
|
||||
? Math.floor(options.sampleLimit)
|
||||
: DEFAULT_SAMPLE_LIMIT;
|
||||
return {
|
||||
lookbackDays,
|
||||
minRunLength,
|
||||
maxFrameMs: Math.round(maxFrameSeconds * 1000),
|
||||
residueMinRunLength: Math.max(MIN_STREAM_RESIDUE_FRAMES, minRunLength - 1),
|
||||
strictFrameMs: Math.round(TIMING_ONLY_FRAME_MAX_SECONDS * 1000),
|
||||
gapToleranceMs: Math.round(DUPLICATE_CUE_GAP_TOLERANCE_SECONDS * 1000),
|
||||
sampleLimit,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* `CREATED_DATE` holds epoch milliseconds on rows this app wrote, but older and synced
|
||||
* rows can carry seconds, so normalize before comparing against the cutoff.
|
||||
*/
|
||||
const CREATED_MS_SQL = `
|
||||
CASE
|
||||
WHEN sl.CREATED_DATE < 10000000000 THEN sl.CREATED_DATE * 1000
|
||||
ELSE sl.CREATED_DATE
|
||||
END`;
|
||||
|
||||
function readCandidateLines(db: DatabaseSync, bounds: ResolvedBounds): StoredSubtitleLineRow[] {
|
||||
const scope =
|
||||
bounds.lookbackDays === null
|
||||
? ''
|
||||
: `AND sl.CREATED_DATE IS NOT NULL AND ${CREATED_MS_SQL} >= ?`;
|
||||
const params = bounds.lookbackDays === null ? [] : [nowMs() - bounds.lookbackDays * MS_PER_DAY];
|
||||
return db
|
||||
.prepare(
|
||||
`SELECT
|
||||
sl.line_id AS lineId,
|
||||
sl.session_id AS sessionId,
|
||||
sl.video_id AS videoId,
|
||||
sl.text AS text,
|
||||
sl.segment_start_ms AS startMs,
|
||||
sl.segment_end_ms AS endMs
|
||||
FROM imm_subtitle_lines sl
|
||||
WHERE sl.segment_start_ms IS NOT NULL
|
||||
AND sl.segment_end_ms IS NOT NULL
|
||||
${scope}
|
||||
ORDER BY sl.session_id, sl.video_id, sl.segment_start_ms, sl.line_id`,
|
||||
)
|
||||
.all(...params) as StoredSubtitleLineRow[];
|
||||
}
|
||||
|
||||
function isBurst(run: StoredSubtitleLineRow[], bounds: ResolvedBounds): boolean {
|
||||
const isShortFrame = (row: StoredSubtitleLineRow): boolean =>
|
||||
row.endMs - row.startMs <= bounds.maxFrameMs;
|
||||
// The residue the live gate leaves behind: it records the first frames of a burst
|
||||
// before the run is long enough to recognise, so one frame fewer than the timing-only
|
||||
// minimum, every one under the strict timing-only bound. No dialogue holds identical
|
||||
// sub-tenth-second lines back to back that many times.
|
||||
if (
|
||||
run.length >= bounds.residueMinRunLength &&
|
||||
run.every((row) => row.endMs - row.startMs <= bounds.strictFrameMs)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (run.length < bounds.minRunLength) {
|
||||
return false;
|
||||
}
|
||||
if (run.every(isShortFrame)) {
|
||||
return true;
|
||||
}
|
||||
// Karaoke commonly finishes its short animation frames with one long hold. Only the
|
||||
// final event may exceed the frame bound, and the short frames before it must already
|
||||
// meet the minimum run length on their own.
|
||||
return (
|
||||
run.length - 1 >= bounds.minRunLength &&
|
||||
run.slice(0, -1).every(isShortFrame) &&
|
||||
!isShortFrame(run[run.length - 1]!)
|
||||
);
|
||||
}
|
||||
|
||||
function toBurst(run: StoredSubtitleLineRow[]): DuplicateSubtitleLineBurst {
|
||||
const [first] = run;
|
||||
return {
|
||||
sessionId: first!.sessionId,
|
||||
videoId: first!.videoId,
|
||||
text: first!.text,
|
||||
keptLineId: first!.lineId,
|
||||
removedLineIds: run.slice(1).map((row) => row.lineId),
|
||||
startMs: first!.startMs,
|
||||
endMs: run.reduce((latest, row) => Math.max(latest, row.endMs), first!.endMs),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Group stored lines into animation runs.
|
||||
*
|
||||
* Rows are bucketed per (session, video, text) before chaining, the way the file-level
|
||||
* dedup buckets cues: dual-line karaoke interleaves two texts frame by frame, and
|
||||
* chaining across the interleave would break every run at length one.
|
||||
*
|
||||
* Runs never cross a session, which is what keeps a rewatch intact: the same episode
|
||||
* watched twice stores the same line twice, and those two belong to different sessions.
|
||||
*/
|
||||
export function findDuplicateSubtitleLineBursts(
|
||||
rows: readonly StoredSubtitleLineRow[],
|
||||
options: DuplicateSubtitleLineCleanupOptions = {},
|
||||
): DuplicateSubtitleLineBurst[] {
|
||||
const bounds = resolveBounds(options);
|
||||
|
||||
// Insertion order preserves the query's startMs ordering within each bucket.
|
||||
const rowsByKey = new Map<string, StoredSubtitleLineRow[]>();
|
||||
for (const row of rows) {
|
||||
const key = `${row.sessionId}|${row.videoId}|${row.text}`;
|
||||
const bucket = rowsByKey.get(key);
|
||||
if (bucket) {
|
||||
bucket.push(row);
|
||||
} else {
|
||||
rowsByKey.set(key, [row]);
|
||||
}
|
||||
}
|
||||
|
||||
const bursts: DuplicateSubtitleLineBurst[] = [];
|
||||
for (const bucket of rowsByKey.values()) {
|
||||
if (bucket.length < 2) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let run: StoredSubtitleLineRow[] = [];
|
||||
let chainEndMs = 0;
|
||||
|
||||
const closeRun = (): void => {
|
||||
if (run.length > 1 && isBurst(run, bounds)) {
|
||||
bursts.push(toBurst(run));
|
||||
}
|
||||
run = [];
|
||||
};
|
||||
|
||||
for (const row of bucket) {
|
||||
if (run.length > 0 && row.startMs <= chainEndMs + bounds.gapToleranceMs) {
|
||||
run.push(row);
|
||||
chainEndMs = Math.max(chainEndMs, row.endMs);
|
||||
continue;
|
||||
}
|
||||
closeRun();
|
||||
run = [row];
|
||||
chainEndMs = row.endMs;
|
||||
}
|
||||
closeRun();
|
||||
}
|
||||
|
||||
return bursts;
|
||||
}
|
||||
|
||||
function chunk<T>(values: T[], size: number): T[][] {
|
||||
const chunks: T[][] = [];
|
||||
for (let i = 0; i < values.length; i += size) {
|
||||
chunks.push(values.slice(i, i + size));
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
function buildSamples(
|
||||
db: DatabaseSync,
|
||||
bursts: DuplicateSubtitleLineBurst[],
|
||||
sampleLimit: number,
|
||||
): DuplicateSubtitleLineSample[] {
|
||||
if (sampleLimit === 0 || bursts.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const largest = [...bursts]
|
||||
.sort((a, b) => b.removedLineIds.length - a.removedLineIds.length)
|
||||
.slice(0, sampleLimit);
|
||||
const videoIds = [...new Set(largest.map((burst) => burst.videoId))];
|
||||
const titles = new Map<number, string>();
|
||||
for (const batch of chunk(videoIds, ID_BATCH_SIZE)) {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT video_id AS videoId, canonical_title AS title
|
||||
FROM imm_videos
|
||||
WHERE video_id IN (${makePlaceholders(batch)})`,
|
||||
)
|
||||
.all(...batch) as Array<{ videoId: number; title: string | null }>;
|
||||
for (const row of rows) {
|
||||
if (row.title) titles.set(row.videoId, row.title);
|
||||
}
|
||||
}
|
||||
|
||||
return largest.map((burst) => ({
|
||||
videoId: burst.videoId,
|
||||
videoTitle: titles.get(burst.videoId) ?? null,
|
||||
text: burst.text,
|
||||
frames: burst.removedLineIds.length + 1,
|
||||
removedLines: burst.removedLineIds.length,
|
||||
startMs: burst.startMs,
|
||||
endMs: burst.endMs,
|
||||
}));
|
||||
}
|
||||
|
||||
function sumRemovedOccurrences(
|
||||
db: DatabaseSync,
|
||||
table: 'imm_word_line_occurrences' | 'imm_kanji_line_occurrences',
|
||||
lineIds: number[],
|
||||
): number {
|
||||
let total = 0;
|
||||
for (const batch of chunk(lineIds, ID_BATCH_SIZE)) {
|
||||
const row = db
|
||||
.prepare(
|
||||
`SELECT COALESCE(SUM(occurrence_count), 0) AS total
|
||||
FROM ${table}
|
||||
WHERE line_id IN (${makePlaceholders(batch)})`,
|
||||
)
|
||||
.get(...batch) as { total: number } | null;
|
||||
total += row?.total ?? 0;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
function applyBursts(db: DatabaseSync, bursts: DuplicateSubtitleLineBurst[]): void {
|
||||
const removedLineIds = bursts.flatMap((burst) => burst.removedLineIds);
|
||||
const currentMs = toDbTimestamp(nowMs());
|
||||
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
for (const batch of chunk(removedLineIds, ID_BATCH_SIZE)) {
|
||||
const placeholders = makePlaceholders(batch);
|
||||
// Measured before the delete, applied after it: `applyLexicalRemovals` checks the
|
||||
// surviving occurrences to decide whether a zeroed count really means the word is
|
||||
// gone, so the rows it inspects have to be the post-delete ones.
|
||||
const plan = planLexicalRemovalsForLines(db, batch);
|
||||
db.prepare(`DELETE FROM imm_word_line_occurrences WHERE line_id IN (${placeholders})`).run(
|
||||
...batch,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_kanji_line_occurrences WHERE line_id IN (${placeholders})`).run(
|
||||
...batch,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_subtitle_lines WHERE line_id IN (${placeholders})`).run(...batch);
|
||||
applyLexicalRemovals(db, plan);
|
||||
}
|
||||
|
||||
const extendStmt = db.prepare(
|
||||
`UPDATE imm_subtitle_lines
|
||||
SET segment_end_ms = ?, LAST_UPDATE_DATE = ?
|
||||
WHERE line_id = ? AND (segment_end_ms IS NULL OR segment_end_ms < ?)`,
|
||||
);
|
||||
for (const burst of bursts) {
|
||||
extendStmt.run(burst.endMs, currentMs, burst.keptLineId, burst.endMs);
|
||||
}
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
try {
|
||||
db.exec('ROLLBACK');
|
||||
} catch {
|
||||
// Surface the transaction failure, not the rollback's.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapse stored animation bursts down to one line each.
|
||||
*
|
||||
* A dry run measures exactly what an apply would remove, using the same scan, so the
|
||||
* numbers shown in a confirmation prompt are the numbers that will happen.
|
||||
*/
|
||||
export function cleanupDuplicateSubtitleLines(
|
||||
db: DatabaseSync,
|
||||
options: DuplicateSubtitleLineCleanupOptions = {},
|
||||
): DuplicateSubtitleLineCleanupSummary {
|
||||
const bounds = resolveBounds(options);
|
||||
const dryRun = options.dryRun === true;
|
||||
const rows = readCandidateLines(db, bounds);
|
||||
const bursts = findDuplicateSubtitleLineBursts(rows, options);
|
||||
const removedLineIds = bursts.flatMap((burst) => burst.removedLineIds);
|
||||
|
||||
const summary: DuplicateSubtitleLineCleanupSummary = {
|
||||
dryRun,
|
||||
lookbackDays: bounds.lookbackDays,
|
||||
scannedLines: rows.length,
|
||||
burstGroups: bursts.length,
|
||||
removedLines: removedLineIds.length,
|
||||
removedWordOccurrences: sumRemovedOccurrences(db, 'imm_word_line_occurrences', removedLineIds),
|
||||
removedKanjiOccurrences: sumRemovedOccurrences(
|
||||
db,
|
||||
'imm_kanji_line_occurrences',
|
||||
removedLineIds,
|
||||
),
|
||||
samples: buildSamples(db, bursts, bounds.sampleLimit),
|
||||
};
|
||||
|
||||
if (dryRun || removedLineIds.length === 0) {
|
||||
return summary;
|
||||
}
|
||||
|
||||
applyBursts(db, bursts);
|
||||
return summary;
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { finalizeSessionRecord } from './session';
|
||||
import { nowMs } from './time';
|
||||
import { toDbTimestamp } from './query-shared';
|
||||
import { forEachIdChunk, makePlaceholders, toDbTimestamp } from './query-shared';
|
||||
import type { LifetimeRebuildSummary, SessionState } from './types';
|
||||
|
||||
interface TelemetryRow {
|
||||
@@ -21,10 +21,8 @@ interface AnimeRow {
|
||||
}
|
||||
|
||||
function asPositiveNumber(value: number | null, fallback: number): number {
|
||||
if (value === null || !Number.isFinite(value)) {
|
||||
return fallback;
|
||||
}
|
||||
return Math.max(0, Math.floor(value));
|
||||
const resolved = value !== null && Number.isFinite(value) ? value : fallback;
|
||||
return Number.isFinite(resolved) ? Math.floor(Math.max(resolved, 0)) : 0;
|
||||
}
|
||||
|
||||
interface ExistenceRow {
|
||||
@@ -68,10 +66,10 @@ const RETAINED_SESSION_METRICS_CTE = `
|
||||
v.anime_id,
|
||||
s.started_at_ms,
|
||||
s.ended_at_ms,
|
||||
MAX(COALESCE(t.active_watched_ms, s.active_watched_ms, 0), 0) AS active_ms,
|
||||
MAX(COALESCE(t.cards_mined, s.cards_mined, 0), 0) AS cards_mined,
|
||||
MAX(COALESCE(t.lines_seen, s.lines_seen, 0), 0) AS lines_seen,
|
||||
MAX(COALESCE(t.tokens_seen, s.tokens_seen, 0), 0) AS tokens_seen,
|
||||
CAST(MAX(COALESCE(t.active_watched_ms, s.active_watched_ms, 0), 0) AS INTEGER) AS active_ms,
|
||||
CAST(MAX(COALESCE(t.cards_mined, s.cards_mined, 0), 0) AS INTEGER) AS cards_mined,
|
||||
CAST(MAX(COALESCE(t.lines_seen, s.lines_seen, 0), 0) AS INTEGER) AS lines_seen,
|
||||
CAST(MAX(COALESCE(t.tokens_seen, s.tokens_seen, 0), 0) AS INTEGER) AS tokens_seen,
|
||||
CASE WHEN v.watched > 0 THEN 1 ELSE 0 END AS completed
|
||||
FROM imm_sessions s
|
||||
JOIN imm_videos v
|
||||
@@ -599,18 +597,10 @@ export function applySessionLifetimeSummary(
|
||||
.get(video.anime_id) as AnimeRow | null | undefined) ?? null)
|
||||
: null;
|
||||
|
||||
const activeMs = telemetry
|
||||
? asPositiveNumber(telemetry.active_watched_ms, session.activeWatchedMs)
|
||||
: session.activeWatchedMs;
|
||||
const cardsMined = telemetry
|
||||
? asPositiveNumber(telemetry.cards_mined, session.cardsMined)
|
||||
: session.cardsMined;
|
||||
const linesSeen = telemetry
|
||||
? asPositiveNumber(telemetry.lines_seen, session.linesSeen)
|
||||
: session.linesSeen;
|
||||
const tokensSeen = telemetry
|
||||
? asPositiveNumber(telemetry.tokens_seen, session.tokensSeen)
|
||||
: session.tokensSeen;
|
||||
const activeMs = asPositiveNumber(telemetry?.active_watched_ms ?? null, session.activeWatchedMs);
|
||||
const cardsMined = asPositiveNumber(telemetry?.cards_mined ?? null, session.cardsMined);
|
||||
const linesSeen = asPositiveNumber(telemetry?.lines_seen ?? null, session.linesSeen);
|
||||
const tokensSeen = asPositiveNumber(telemetry?.tokens_seen ?? null, session.tokensSeen);
|
||||
const watched = video?.watched ?? 0;
|
||||
const isFirstSessionForVideoRun =
|
||||
mediaLifetime === null &&
|
||||
@@ -708,6 +698,466 @@ export function rebuildLifetimeSummariesInTransaction(
|
||||
return rebuildLifetimeSummariesInternal(db, rebuiltAtMs);
|
||||
}
|
||||
|
||||
const LOCAL_DAY_EXPR = `CAST(
|
||||
julianday(CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5
|
||||
AS INTEGER
|
||||
)`;
|
||||
|
||||
interface LifetimeMediaRemoval {
|
||||
videoId: number;
|
||||
sessions: number;
|
||||
activeMs: number;
|
||||
cards: number;
|
||||
linesSeen: number;
|
||||
tokensSeen: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* What a pending delete removes from the lifetime summary tables.
|
||||
*
|
||||
* Lifetime totals intentionally outlive raw-session retention, so they can
|
||||
* never be rebuilt from `imm_sessions` without collapsing history to the
|
||||
* retention window. Deletes instead subtract exactly what the deleted rows
|
||||
* contributed: this plan is measured before the rows are removed and applied
|
||||
* after.
|
||||
*/
|
||||
export interface LifetimeRemovalPlan {
|
||||
/** Per surviving video: summed metrics of its deleted, lifetime-applied sessions. */
|
||||
mediaRemovals: LifetimeMediaRemoval[];
|
||||
/** Surviving anime whose lifetime rows must be recomputed from their media rows. */
|
||||
affectedAnimeIds: number[];
|
||||
/** Local-day keys touched by deleted applied sessions, for active_days upkeep. */
|
||||
affectedDayKeys: number[];
|
||||
}
|
||||
|
||||
export function planLifetimeRemovals(
|
||||
db: DatabaseSync,
|
||||
args: {
|
||||
/** Every session being deleted, including ones expanded from video/anime deletes. */
|
||||
deletedSessionIds: number[];
|
||||
/** Deleted sessions whose video survives the delete. */
|
||||
sessionIdsOnSurvivingVideos: number[];
|
||||
deletedVideoIds: number[];
|
||||
deletedAnimeIds: number[];
|
||||
},
|
||||
): LifetimeRemovalPlan {
|
||||
const mediaRemovalsByVideo = new Map<number, LifetimeMediaRemoval>();
|
||||
forEachIdChunk(args.sessionIdsOnSurvivingVideos, (chunk) => {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
s.video_id AS videoId,
|
||||
COUNT(*) AS sessions,
|
||||
COALESCE(SUM(CAST(MAX(COALESCE(t.active_watched_ms, s.active_watched_ms, 0), 0) AS INTEGER)), 0) AS activeMs,
|
||||
COALESCE(SUM(CAST(MAX(COALESCE(t.cards_mined, s.cards_mined, 0), 0) AS INTEGER)), 0) AS cards,
|
||||
COALESCE(SUM(CAST(MAX(COALESCE(t.lines_seen, s.lines_seen, 0), 0) AS INTEGER)), 0) AS linesSeen,
|
||||
COALESCE(SUM(CAST(MAX(COALESCE(t.tokens_seen, s.tokens_seen, 0), 0) AS INTEGER)), 0) AS tokensSeen
|
||||
FROM imm_sessions s
|
||||
JOIN imm_lifetime_applied_sessions a ON a.session_id = s.session_id
|
||||
LEFT JOIN imm_session_telemetry t
|
||||
ON t.telemetry_id = (
|
||||
SELECT telemetry_id
|
||||
FROM imm_session_telemetry
|
||||
WHERE session_id = s.session_id
|
||||
ORDER BY sample_ms DESC, telemetry_id DESC
|
||||
LIMIT 1
|
||||
)
|
||||
WHERE s.session_id IN (${makePlaceholders(chunk)})
|
||||
GROUP BY s.video_id
|
||||
`,
|
||||
)
|
||||
.all(...chunk) as LifetimeMediaRemoval[];
|
||||
for (const row of rows) {
|
||||
const existing = mediaRemovalsByVideo.get(row.videoId);
|
||||
if (!existing) {
|
||||
mediaRemovalsByVideo.set(row.videoId, { ...row });
|
||||
continue;
|
||||
}
|
||||
existing.sessions += row.sessions;
|
||||
existing.activeMs += row.activeMs;
|
||||
existing.cards += row.cards;
|
||||
existing.linesSeen += row.linesSeen;
|
||||
existing.tokensSeen += row.tokensSeen;
|
||||
}
|
||||
});
|
||||
|
||||
const deletedAnimeIds = new Set(args.deletedAnimeIds);
|
||||
const affectedAnimeIds = new Set<number>();
|
||||
forEachIdChunk(args.deletedVideoIds, (chunk) => {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT DISTINCT anime_id AS animeId FROM imm_videos
|
||||
WHERE video_id IN (${makePlaceholders(chunk)}) AND anime_id IS NOT NULL`,
|
||||
)
|
||||
.all(...chunk) as Array<{ animeId: number }>;
|
||||
for (const row of rows) affectedAnimeIds.add(row.animeId);
|
||||
});
|
||||
forEachIdChunk(args.sessionIdsOnSurvivingVideos, (chunk) => {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT DISTINCT v.anime_id AS animeId
|
||||
FROM imm_sessions s
|
||||
JOIN imm_videos v ON v.video_id = s.video_id
|
||||
WHERE s.session_id IN (${makePlaceholders(chunk)}) AND v.anime_id IS NOT NULL`,
|
||||
)
|
||||
.all(...chunk) as Array<{ animeId: number }>;
|
||||
for (const row of rows) affectedAnimeIds.add(row.animeId);
|
||||
});
|
||||
for (const animeId of deletedAnimeIds) affectedAnimeIds.delete(animeId);
|
||||
|
||||
const affectedDayKeys = new Set<number>();
|
||||
forEachIdChunk(args.deletedSessionIds, (chunk) => {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT DISTINCT ${LOCAL_DAY_EXPR} AS dayKey
|
||||
FROM imm_sessions s
|
||||
JOIN imm_lifetime_applied_sessions a ON a.session_id = s.session_id
|
||||
WHERE s.session_id IN (${makePlaceholders(chunk)})`,
|
||||
)
|
||||
.all(...chunk) as Array<{ dayKey: number }>;
|
||||
for (const row of rows) affectedDayKeys.add(row.dayKey);
|
||||
});
|
||||
|
||||
return {
|
||||
mediaRemovals: [...mediaRemovalsByVideo.values()],
|
||||
affectedAnimeIds: [...affectedAnimeIds],
|
||||
affectedDayKeys: [...affectedDayKeys],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a removal plan after the underlying rows are gone.
|
||||
*
|
||||
* Media rows are adjusted by subtraction (pruned-session history stays intact),
|
||||
* affected anime rows are recomputed from their surviving media rows, and the
|
||||
* global row is re-derived from the media/anime tables. `active_days` is the
|
||||
* one metric that can't be derived, so a touched day is only decremented when
|
||||
* no ended session remains on that local day; days whose sessions were pruned
|
||||
* by retention keep their count because pruning never subtracts.
|
||||
*/
|
||||
export function applyLifetimeRemovals(db: DatabaseSync, plan: LifetimeRemovalPlan): void {
|
||||
const updatedAtMs = toDbTimestamp(nowMs());
|
||||
|
||||
const subtractMediaStmt = db.prepare(
|
||||
`
|
||||
UPDATE imm_lifetime_media SET
|
||||
total_sessions = MAX(total_sessions - ?, 0),
|
||||
total_active_ms = MAX(total_active_ms - ?, 0),
|
||||
total_cards = MAX(total_cards - ?, 0),
|
||||
total_lines_seen = MAX(total_lines_seen - ?, 0),
|
||||
total_tokens_seen = MAX(total_tokens_seen - ?, 0),
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE video_id = ?
|
||||
`,
|
||||
);
|
||||
const dropEmptyMediaStmt = db.prepare(
|
||||
'DELETE FROM imm_lifetime_media WHERE video_id = ? AND total_sessions <= 0',
|
||||
);
|
||||
const remainingSessionRangeStmt = db.prepare(
|
||||
`
|
||||
SELECT
|
||||
MIN(CAST(started_at_ms AS REAL)) AS minStartedMs,
|
||||
MAX(CAST(ended_at_ms AS REAL)) AS maxEndedMs
|
||||
FROM imm_sessions
|
||||
WHERE video_id = ? AND ended_at_ms IS NOT NULL
|
||||
`,
|
||||
);
|
||||
const storedMediaRangeStmt = db.prepare(
|
||||
`
|
||||
SELECT CAST(first_watched_ms AS REAL) AS firstWatchedMs
|
||||
FROM imm_lifetime_media
|
||||
WHERE video_id = ?
|
||||
`,
|
||||
);
|
||||
const refreshMediaRangeStmt = db.prepare(
|
||||
`
|
||||
UPDATE imm_lifetime_media SET
|
||||
first_watched_ms = ?,
|
||||
last_watched_ms = ?
|
||||
WHERE video_id = ?
|
||||
`,
|
||||
);
|
||||
|
||||
for (const removal of plan.mediaRemovals) {
|
||||
subtractMediaStmt.run(
|
||||
removal.sessions,
|
||||
removal.activeMs,
|
||||
removal.cards,
|
||||
removal.linesSeen,
|
||||
removal.tokensSeen,
|
||||
updatedAtMs,
|
||||
removal.videoId,
|
||||
);
|
||||
dropEmptyMediaStmt.run(removal.videoId);
|
||||
const stored = storedMediaRangeStmt.get(removal.videoId) as {
|
||||
firstWatchedMs: number | null;
|
||||
} | null;
|
||||
if (!stored) continue;
|
||||
const range = remainingSessionRangeStmt.get(removal.videoId) as {
|
||||
minStartedMs: number | null;
|
||||
maxEndedMs: number | null;
|
||||
} | null;
|
||||
// Retained sessions are always newer than pruned ones, so the surviving
|
||||
// range is authoritative for last_watched while first_watched can only
|
||||
// keep or extend the stored (possibly pruned-history) minimum. When no
|
||||
// session survives, the stored values are all that's left.
|
||||
if (range && range.minStartedMs !== null && range.maxEndedMs !== null) {
|
||||
const firstWatchedMs =
|
||||
stored.firstWatchedMs === null
|
||||
? range.minStartedMs
|
||||
: Math.min(stored.firstWatchedMs, range.minStartedMs);
|
||||
refreshMediaRangeStmt.run(
|
||||
toDbTimestamp(firstWatchedMs),
|
||||
toDbTimestamp(range.maxEndedMs),
|
||||
removal.videoId,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
recomputeLifetimeAnimeFromMedia(db, plan.affectedAnimeIds, updatedAtMs);
|
||||
|
||||
// One pass over the sessions rather than a probe per affected day: the local
|
||||
// day is a computed expression with no index, so each probe would be a table
|
||||
// scan — and the miss case (the day we need to count) is the full-scan one.
|
||||
let removedDays = 0;
|
||||
if (plan.affectedDayKeys.length > 0) {
|
||||
const survivingDayKeys = new Set(
|
||||
(
|
||||
db
|
||||
.prepare(
|
||||
`SELECT DISTINCT ${LOCAL_DAY_EXPR} AS dayKey
|
||||
FROM imm_sessions
|
||||
WHERE ended_at_ms IS NOT NULL`,
|
||||
)
|
||||
.all() as Array<{ dayKey: number }>
|
||||
).map((row) => row.dayKey),
|
||||
);
|
||||
for (const dayKey of plan.affectedDayKeys) {
|
||||
if (!survivingDayKeys.has(dayKey)) removedDays += 1;
|
||||
}
|
||||
}
|
||||
|
||||
recomputeLifetimeGlobalFromSummaries(db, { removedActiveDays: removedDays, updatedAtMs });
|
||||
}
|
||||
|
||||
/**
|
||||
* Recompute lifetime anime rows exactly from their surviving media rows.
|
||||
*
|
||||
* Media rows are the durable per-video ledger (they outlive session pruning and
|
||||
* follow a video when it moves between anime), so this is the correct refresh
|
||||
* after merges, moves, and deletes. Anime with no media rows left are dropped.
|
||||
*/
|
||||
export function recomputeLifetimeAnimeFromMedia(
|
||||
db: DatabaseSync,
|
||||
animeIds: number[],
|
||||
updatedAtMs = toDbTimestamp(nowMs()),
|
||||
): void {
|
||||
if (animeIds.length === 0) return;
|
||||
|
||||
const animeSummaryStmt = db.prepare(
|
||||
`
|
||||
SELECT
|
||||
COUNT(*) AS episodeRows,
|
||||
COALESCE(SUM(m.total_sessions), 0) AS totalSessions,
|
||||
COALESCE(SUM(m.total_active_ms), 0) AS totalActiveMs,
|
||||
COALESCE(SUM(m.total_cards), 0) AS totalCards,
|
||||
COALESCE(SUM(m.total_lines_seen), 0) AS totalLinesSeen,
|
||||
COALESCE(SUM(m.total_tokens_seen), 0) AS totalTokensSeen,
|
||||
COALESCE(SUM(CASE WHEN m.completed > 0 THEN 1 ELSE 0 END), 0) AS episodesCompleted,
|
||||
MIN(CAST(m.first_watched_ms AS REAL)) AS firstWatchedMs,
|
||||
MAX(CAST(m.last_watched_ms AS REAL)) AS lastWatchedMs
|
||||
FROM imm_lifetime_media m
|
||||
JOIN imm_videos v ON v.video_id = m.video_id
|
||||
WHERE v.anime_id = ?
|
||||
`,
|
||||
);
|
||||
const dropAnimeStmt = db.prepare('DELETE FROM imm_lifetime_anime WHERE anime_id = ?');
|
||||
const upsertAnimeStmt = db.prepare(
|
||||
`
|
||||
INSERT INTO imm_lifetime_anime(
|
||||
anime_id,
|
||||
total_sessions,
|
||||
total_active_ms,
|
||||
total_cards,
|
||||
total_lines_seen,
|
||||
total_tokens_seen,
|
||||
episodes_started,
|
||||
episodes_completed,
|
||||
first_watched_ms,
|
||||
last_watched_ms,
|
||||
CREATED_DATE,
|
||||
LAST_UPDATE_DATE
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(anime_id) DO UPDATE SET
|
||||
total_sessions = excluded.total_sessions,
|
||||
total_active_ms = excluded.total_active_ms,
|
||||
total_cards = excluded.total_cards,
|
||||
total_lines_seen = excluded.total_lines_seen,
|
||||
total_tokens_seen = excluded.total_tokens_seen,
|
||||
episodes_started = excluded.episodes_started,
|
||||
episodes_completed = excluded.episodes_completed,
|
||||
first_watched_ms = excluded.first_watched_ms,
|
||||
last_watched_ms = excluded.last_watched_ms,
|
||||
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
|
||||
`,
|
||||
);
|
||||
|
||||
for (const animeId of animeIds) {
|
||||
const summary = animeSummaryStmt.get(animeId) as {
|
||||
episodeRows: number;
|
||||
totalSessions: number;
|
||||
totalActiveMs: number;
|
||||
totalCards: number;
|
||||
totalLinesSeen: number;
|
||||
totalTokensSeen: number;
|
||||
episodesCompleted: number;
|
||||
firstWatchedMs: number | null;
|
||||
lastWatchedMs: number | null;
|
||||
};
|
||||
if (Number(summary.episodeRows) === 0) {
|
||||
dropAnimeStmt.run(animeId);
|
||||
continue;
|
||||
}
|
||||
upsertAnimeStmt.run(
|
||||
animeId,
|
||||
summary.totalSessions,
|
||||
summary.totalActiveMs,
|
||||
summary.totalCards,
|
||||
summary.totalLinesSeen,
|
||||
summary.totalTokensSeen,
|
||||
summary.episodeRows,
|
||||
summary.episodesCompleted,
|
||||
summary.firstWatchedMs === null ? null : toDbTimestamp(summary.firstWatchedMs),
|
||||
summary.lastWatchedMs === null ? null : toDbTimestamp(summary.lastWatchedMs),
|
||||
updatedAtMs,
|
||||
updatedAtMs,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-derive the global lifetime row from the media/anime summary tables.
|
||||
*
|
||||
* Every global metric except active_days is a pure aggregate of those tables;
|
||||
* active_days can't be derived, so callers pass how many day slots their change
|
||||
* removed (0 for moves/merges, which never touch sessions).
|
||||
*/
|
||||
export function recomputeLifetimeGlobalFromSummaries(
|
||||
db: DatabaseSync,
|
||||
options: { removedActiveDays?: number; updatedAtMs?: string } = {},
|
||||
): void {
|
||||
const updatedAtMs = options.updatedAtMs ?? toDbTimestamp(nowMs());
|
||||
const mediaTotals = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
COUNT(*) AS episodesStarted,
|
||||
COALESCE(SUM(total_sessions), 0) AS totalSessions,
|
||||
COALESCE(SUM(total_active_ms), 0) AS totalActiveMs,
|
||||
COALESCE(SUM(total_cards), 0) AS totalCards,
|
||||
COALESCE(SUM(CASE WHEN completed > 0 THEN 1 ELSE 0 END), 0) AS episodesCompleted
|
||||
FROM imm_lifetime_media
|
||||
`,
|
||||
)
|
||||
.get() as {
|
||||
episodesStarted: number;
|
||||
totalSessions: number;
|
||||
totalActiveMs: number;
|
||||
totalCards: number;
|
||||
episodesCompleted: number;
|
||||
};
|
||||
const animeCompletedRow = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT COUNT(*) AS animeCompleted
|
||||
FROM imm_lifetime_anime la
|
||||
JOIN imm_anime a ON a.anime_id = la.anime_id
|
||||
WHERE a.episodes_total IS NOT NULL
|
||||
AND a.episodes_total > 0
|
||||
AND la.episodes_completed >= a.episodes_total
|
||||
`,
|
||||
)
|
||||
.get() as { animeCompleted: number };
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE imm_lifetime_global SET
|
||||
total_sessions = ?,
|
||||
total_active_ms = ?,
|
||||
total_cards = ?,
|
||||
episodes_started = ?,
|
||||
episodes_completed = ?,
|
||||
anime_completed = ?,
|
||||
active_days = MAX(active_days - ?, 0),
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE global_id = 1
|
||||
`,
|
||||
).run(
|
||||
mediaTotals.totalSessions,
|
||||
mediaTotals.totalActiveMs,
|
||||
mediaTotals.totalCards,
|
||||
mediaTotals.episodesStarted,
|
||||
mediaTotals.episodesCompleted,
|
||||
animeCompletedRow.animeCompleted,
|
||||
options.removedActiveDays ?? 0,
|
||||
updatedAtMs,
|
||||
);
|
||||
}
|
||||
|
||||
export interface LifetimeRepairSummary {
|
||||
recomputedAnime: number;
|
||||
repairedAtMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-destructive lifetime repair: recompute every anime row and the global row
|
||||
* from the per-video media ledger.
|
||||
*
|
||||
* Unlike {@link rebuildLifetimeSummaries}, this never resets the tables from
|
||||
* retained sessions, so lifetime history older than the session retention
|
||||
* window survives. The one exception is a database whose lifetime tables were
|
||||
* never populated — there is no ledger to repair from, so it bootstraps with
|
||||
* the full rebuild instead.
|
||||
*/
|
||||
export function repairLifetimeSummariesFromMedia(db: DatabaseSync): LifetimeRepairSummary {
|
||||
const repairedAtMs = nowMs();
|
||||
let transactionStarted = false;
|
||||
try {
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
transactionStarted = true;
|
||||
if (shouldBackfillLifetimeSummaries(db)) {
|
||||
const rebuilt = rebuildLifetimeSummariesInTransaction(db, repairedAtMs);
|
||||
const animeRow = db
|
||||
.prepare('SELECT COUNT(*) AS count FROM imm_lifetime_anime')
|
||||
.get() as ExistenceRow;
|
||||
db.exec('COMMIT');
|
||||
return { recomputedAnime: Number(animeRow.count), repairedAtMs: rebuilt.rebuiltAtMs };
|
||||
}
|
||||
|
||||
const animeIds = new Set<number>();
|
||||
for (const row of db
|
||||
.prepare('SELECT DISTINCT anime_id AS animeId FROM imm_videos WHERE anime_id IS NOT NULL')
|
||||
.all() as Array<{ animeId: number }>) {
|
||||
animeIds.add(row.animeId);
|
||||
}
|
||||
for (const row of db
|
||||
.prepare('SELECT anime_id AS animeId FROM imm_lifetime_anime')
|
||||
.all() as Array<{ animeId: number }>) {
|
||||
animeIds.add(row.animeId);
|
||||
}
|
||||
const updatedAtMs = toDbTimestamp(repairedAtMs);
|
||||
recomputeLifetimeAnimeFromMedia(db, [...animeIds], updatedAtMs);
|
||||
recomputeLifetimeGlobalFromSummaries(db, { updatedAtMs });
|
||||
db.exec('COMMIT');
|
||||
return { recomputedAnime: animeIds.size, repairedAtMs };
|
||||
} catch (error) {
|
||||
if (transactionStarted) db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function reconcileStaleActiveSessions(db: DatabaseSync): number {
|
||||
const sessions = getRetainedStaleActiveSessions(db);
|
||||
if (sessions.length === 0) {
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { applyLifetimeRemovals, planLifetimeRemovals } from './lifetime';
|
||||
import { getRollupGroupsForSessions, refreshRollupsForGroupsInTransaction } from './maintenance';
|
||||
import {
|
||||
applyLexicalRemovals,
|
||||
cleanupUnusedCoverArtBlobHash,
|
||||
deleteSessionsByIds,
|
||||
forEachIdChunk,
|
||||
makePlaceholders,
|
||||
planLexicalRemovalsForSessions,
|
||||
planLexicalRemovalsForVideos,
|
||||
type LexicalRemovalPlan,
|
||||
} from './query-shared';
|
||||
import type { RollupGroup } from './maintenance';
|
||||
|
||||
export type DeleteMaintenanceOperation =
|
||||
| { kind: 'session'; sessionId: number }
|
||||
| { kind: 'sessions'; sessionIds: number[] }
|
||||
| { kind: 'video'; videoId: number }
|
||||
| { kind: 'anime'; animeId: number };
|
||||
|
||||
function addOperationTargets(
|
||||
operations: DeleteMaintenanceOperation[],
|
||||
sessionIds: Set<number>,
|
||||
videoIds: Set<number>,
|
||||
animeIds: Set<number>,
|
||||
): void {
|
||||
for (const operation of operations) {
|
||||
switch (operation.kind) {
|
||||
case 'session':
|
||||
sessionIds.add(operation.sessionId);
|
||||
break;
|
||||
case 'sessions':
|
||||
for (const sessionId of operation.sessionIds) sessionIds.add(sessionId);
|
||||
break;
|
||||
case 'video':
|
||||
videoIds.add(operation.videoId);
|
||||
break;
|
||||
case 'anime':
|
||||
animeIds.add(operation.animeId);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function selectIds(
|
||||
db: DatabaseSync,
|
||||
buildSql: (placeholders: string) => string,
|
||||
params: number[],
|
||||
column: string,
|
||||
): number[] {
|
||||
if (params.length === 0) return [];
|
||||
const ids: number[] = [];
|
||||
forEachIdChunk(params, (chunk) => {
|
||||
const rows = db.prepare(buildSql(makePlaceholders(chunk))).all(...chunk) as Array<
|
||||
Record<string, number>
|
||||
>;
|
||||
for (const row of rows) ids.push(row[column]!);
|
||||
});
|
||||
return ids;
|
||||
}
|
||||
|
||||
function mergeLexicalPlanEntries(
|
||||
target: LexicalRemovalPlan['words'],
|
||||
byId: Map<number, LexicalRemovalPlan['words'][number]>,
|
||||
source: LexicalRemovalPlan['words'],
|
||||
): void {
|
||||
for (const entry of source) {
|
||||
const existing = byId.get(entry.id);
|
||||
if (!existing) {
|
||||
const added = { ...entry };
|
||||
target.push(added);
|
||||
byId.set(entry.id, added);
|
||||
continue;
|
||||
}
|
||||
existing.removedFrequency += entry.removedFrequency;
|
||||
if (
|
||||
entry.removedFirstSeenMs !== null &&
|
||||
(existing.removedFirstSeenMs === null ||
|
||||
entry.removedFirstSeenMs < existing.removedFirstSeenMs)
|
||||
) {
|
||||
existing.removedFirstSeenMs = entry.removedFirstSeenMs;
|
||||
}
|
||||
if (
|
||||
entry.removedLastSeenMs !== null &&
|
||||
(existing.removedLastSeenMs === null || entry.removedLastSeenMs > existing.removedLastSeenMs)
|
||||
) {
|
||||
existing.removedLastSeenMs = entry.removedLastSeenMs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface LexicalPlanEntryMaps {
|
||||
words: Map<number, LexicalRemovalPlan['words'][number]>;
|
||||
kanji: Map<number, LexicalRemovalPlan['kanji'][number]>;
|
||||
}
|
||||
|
||||
function mergeLexicalPlans(
|
||||
target: LexicalRemovalPlan,
|
||||
byId: LexicalPlanEntryMaps,
|
||||
source: LexicalRemovalPlan,
|
||||
): void {
|
||||
mergeLexicalPlanEntries(target.words, byId.words, source.words);
|
||||
mergeLexicalPlanEntries(target.kanji, byId.kanji, source.kanji);
|
||||
}
|
||||
|
||||
/**
|
||||
* Plan what the delete removes from imm_words/imm_kanji.
|
||||
*
|
||||
* Deleted videos are planned by video so orphaned subtitle lines (whose session
|
||||
* is already gone) still get subtracted; sessions on surviving videos are
|
||||
* planned by session. The two scopes are disjoint, so nothing is counted twice.
|
||||
*/
|
||||
function planLexicalRemovalsForDelete(
|
||||
db: DatabaseSync,
|
||||
sessionIdsOnSurvivingVideos: number[],
|
||||
videoIds: number[],
|
||||
): LexicalRemovalPlan {
|
||||
const combined: LexicalRemovalPlan = { words: [], kanji: [] };
|
||||
const byId: LexicalPlanEntryMaps = {
|
||||
words: new Map(),
|
||||
kanji: new Map(),
|
||||
};
|
||||
forEachIdChunk(sessionIdsOnSurvivingVideos, (chunk) => {
|
||||
mergeLexicalPlans(combined, byId, planLexicalRemovalsForSessions(db, chunk));
|
||||
});
|
||||
forEachIdChunk(videoIds, (chunk) => {
|
||||
mergeLexicalPlans(combined, byId, planLexicalRemovalsForVideos(db, chunk));
|
||||
});
|
||||
return combined;
|
||||
}
|
||||
|
||||
export function deleteMaintenanceBatch(
|
||||
db: DatabaseSync,
|
||||
operations: DeleteMaintenanceOperation[],
|
||||
): void {
|
||||
if (operations.length === 0) return;
|
||||
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const sessionIds = new Set<number>();
|
||||
const videoIds = new Set<number>();
|
||||
const animeIds = new Set<number>();
|
||||
addOperationTargets(operations, sessionIds, videoIds, animeIds);
|
||||
|
||||
const animeIdList = [...animeIds];
|
||||
for (const videoId of selectIds(
|
||||
db,
|
||||
(placeholders) => `SELECT video_id FROM imm_videos WHERE anime_id IN (${placeholders})`,
|
||||
animeIdList,
|
||||
'video_id',
|
||||
)) {
|
||||
videoIds.add(videoId);
|
||||
}
|
||||
|
||||
const videoIdList = [...videoIds];
|
||||
const sessionIdsOnDeletedVideos = new Set(
|
||||
selectIds(
|
||||
db,
|
||||
(placeholders) => `SELECT session_id FROM imm_sessions WHERE video_id IN (${placeholders})`,
|
||||
videoIdList,
|
||||
'session_id',
|
||||
),
|
||||
);
|
||||
for (const sessionId of sessionIdsOnDeletedVideos) sessionIds.add(sessionId);
|
||||
|
||||
const sessionIdList = [...sessionIds];
|
||||
const sessionIdsOnSurvivingVideos = sessionIdList.filter(
|
||||
(sessionId) => !sessionIdsOnDeletedVideos.has(sessionId),
|
||||
);
|
||||
|
||||
// Both plans must be measured before any rows are removed.
|
||||
const lexicalRemovals = planLexicalRemovalsForDelete(
|
||||
db,
|
||||
sessionIdsOnSurvivingVideos,
|
||||
videoIdList,
|
||||
);
|
||||
const lifetimeRemovals = planLifetimeRemovals(db, {
|
||||
deletedSessionIds: sessionIdList,
|
||||
sessionIdsOnSurvivingVideos,
|
||||
deletedVideoIds: videoIdList,
|
||||
deletedAnimeIds: animeIdList,
|
||||
});
|
||||
const affectedRollupGroups: RollupGroup[] = [];
|
||||
forEachIdChunk(sessionIdsOnSurvivingVideos, (chunk) => {
|
||||
affectedRollupGroups.push(...getRollupGroupsForSessions(db, chunk));
|
||||
});
|
||||
const coverBlobHashes = new Set<string>();
|
||||
if (videoIdList.length > 0) {
|
||||
forEachIdChunk(videoIdList, (chunk) => {
|
||||
const placeholders = makePlaceholders(chunk);
|
||||
const artRows = db
|
||||
.prepare(
|
||||
`SELECT cover_blob_hash AS coverBlobHash
|
||||
FROM imm_media_art
|
||||
WHERE video_id IN (${placeholders}) AND cover_blob_hash IS NOT NULL`,
|
||||
)
|
||||
.all(...chunk) as Array<{ coverBlobHash: string }>;
|
||||
for (const row of artRows) coverBlobHashes.add(row.coverBlobHash);
|
||||
});
|
||||
}
|
||||
|
||||
deleteSessionsByIds(db, sessionIdList);
|
||||
forEachIdChunk(sessionIdList, (chunk) => {
|
||||
const placeholders = makePlaceholders(chunk);
|
||||
db.prepare(
|
||||
`DELETE FROM imm_lifetime_applied_sessions WHERE session_id IN (${placeholders})`,
|
||||
).run(...chunk);
|
||||
});
|
||||
forEachIdChunk(videoIdList, (chunk) => {
|
||||
const placeholders = makePlaceholders(chunk);
|
||||
db.prepare(`DELETE FROM imm_subtitle_lines WHERE video_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_daily_rollups WHERE video_id IN (${placeholders})`).run(...chunk);
|
||||
db.prepare(`DELETE FROM imm_monthly_rollups WHERE video_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_media_art WHERE video_id IN (${placeholders})`).run(...chunk);
|
||||
db.prepare(`DELETE FROM imm_lifetime_media WHERE video_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_videos WHERE video_id IN (${placeholders})`).run(...chunk);
|
||||
});
|
||||
|
||||
for (const coverBlobHash of coverBlobHashes) {
|
||||
cleanupUnusedCoverArtBlobHash(db, coverBlobHash);
|
||||
}
|
||||
if (animeIdList.length > 0) {
|
||||
forEachIdChunk(animeIdList, (chunk) => {
|
||||
const placeholders = makePlaceholders(chunk);
|
||||
db.prepare(`DELETE FROM imm_lifetime_anime WHERE anime_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_anime WHERE anime_id IN (${placeholders})`).run(...chunk);
|
||||
});
|
||||
}
|
||||
|
||||
applyLexicalRemovals(db, lexicalRemovals);
|
||||
applyLifetimeRemovals(db, lifetimeRemovals);
|
||||
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -1,20 +1,21 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { buildCoverBlobReference, normalizeCoverBlobBytes } from './storage';
|
||||
import { rebuildLifetimeSummaries, rebuildLifetimeSummariesInTransaction } from './lifetime';
|
||||
import { getRollupGroupsForSessions, refreshRollupsForGroupsInTransaction } from './maintenance';
|
||||
import {
|
||||
recomputeLifetimeAnimeFromMedia,
|
||||
recomputeLifetimeGlobalFromSummaries,
|
||||
repairLifetimeSummariesFromMedia,
|
||||
shouldBackfillLifetimeSummaries,
|
||||
} from './lifetime';
|
||||
import { nowMs } from './time';
|
||||
import { resolveAnimeAnilistConflict } from './anime-season-repair';
|
||||
import { deleteMaintenanceBatch } from './query-delete-maintenance';
|
||||
import { PartOfSpeech, type MergedToken } from '../../../types';
|
||||
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
|
||||
import { deriveStoredPartOfSpeech } from '../tokenizer/part-of-speech';
|
||||
import {
|
||||
applyLexicalRemovals,
|
||||
cleanupUnusedCoverArtBlobHash,
|
||||
deleteSessionsByIds,
|
||||
findSharedCoverBlobHash,
|
||||
planLexicalRemovalsForSessions,
|
||||
planLexicalRemovalsForVideos,
|
||||
toDbMs,
|
||||
toDbTimestamp,
|
||||
} from './query-shared';
|
||||
@@ -425,7 +426,7 @@ export function updateAnimeAnilistInfo(
|
||||
} | null;
|
||||
if (!row?.anime_id) return;
|
||||
|
||||
const repair = resolveAnimeAnilistConflict(db, row.anime_id, info.anilistId);
|
||||
const conflictRepair = resolveAnimeAnilistConflict(db, row.anime_id, info.anilistId);
|
||||
const targetRow = db
|
||||
.prepare('SELECT anime_id FROM imm_videos WHERE video_id = ?')
|
||||
.get(videoId) as {
|
||||
@@ -454,8 +455,14 @@ export function updateAnimeAnilistInfo(
|
||||
toDbTimestamp(nowMs()),
|
||||
targetRow.anime_id,
|
||||
);
|
||||
if (repair.movedVideos > 0 || repair.deletedAnimeRows > 0) {
|
||||
rebuildLifetimeSummaries(db);
|
||||
if (shouldBackfillLifetimeSummaries(db)) {
|
||||
repairLifetimeSummariesFromMedia(db);
|
||||
} else {
|
||||
const affectedAnimeIds = new Set(conflictRepair.affectedAnimeIds);
|
||||
affectedAnimeIds.add(row.anime_id);
|
||||
affectedAnimeIds.add(targetRow.anime_id);
|
||||
recomputeLifetimeAnimeFromMedia(db, [...affectedAnimeIds]);
|
||||
recomputeLifetimeGlobalFromSummaries(db);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -482,136 +489,22 @@ export function isVideoWatched(db: DatabaseSync, videoId: number): boolean {
|
||||
}
|
||||
|
||||
export function deleteSession(db: DatabaseSync, sessionId: number): void {
|
||||
const sessionIds = [sessionId];
|
||||
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
// Measured inside the write lock: the plan records what the delete removes,
|
||||
// and applying a plan taken against a different snapshot would subtract the
|
||||
// wrong totals from imm_words/imm_kanji.
|
||||
const lexicalRemovals = planLexicalRemovalsForSessions(db, sessionIds);
|
||||
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
|
||||
deleteSessionsByIds(db, sessionIds);
|
||||
applyLexicalRemovals(db, lexicalRemovals);
|
||||
rebuildLifetimeSummariesInTransaction(db);
|
||||
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId }]);
|
||||
}
|
||||
|
||||
export function deleteSessions(db: DatabaseSync, sessionIds: number[]): void {
|
||||
if (sessionIds.length === 0) return;
|
||||
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const lexicalRemovals = planLexicalRemovalsForSessions(db, sessionIds);
|
||||
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
|
||||
deleteSessionsByIds(db, sessionIds);
|
||||
applyLexicalRemovals(db, lexicalRemovals);
|
||||
rebuildLifetimeSummariesInTransaction(db);
|
||||
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
deleteMaintenanceBatch(db, [{ kind: 'sessions', sessionIds }]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an entire library entry: every episode of the anime, all of their
|
||||
* sessions and derived stats, and the anime row itself.
|
||||
*
|
||||
* Mirrors {@link deleteVideo} per episode, but batches the lexical refresh and
|
||||
* lifetime rebuild into a single transaction so a multi-episode title doesn't
|
||||
* pay for one full rebuild per episode.
|
||||
*/
|
||||
export function deleteAnime(db: DatabaseSync, animeId: number): void {
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const videoIds = (
|
||||
db.prepare('SELECT video_id FROM imm_videos WHERE anime_id = ?').all(animeId) as Array<{
|
||||
video_id: number;
|
||||
}>
|
||||
).map((row) => row.video_id);
|
||||
|
||||
const lexicalRemovals = planLexicalRemovalsForVideos(db, videoIds);
|
||||
const coverBlobHashes: string[] = [];
|
||||
const sessionIds: number[] = [];
|
||||
for (const videoId of videoIds) {
|
||||
const artRow = db
|
||||
.prepare('SELECT cover_blob_hash AS coverBlobHash FROM imm_media_art WHERE video_id = ?')
|
||||
.get(videoId) as { coverBlobHash: string | null } | undefined;
|
||||
if (artRow?.coverBlobHash) {
|
||||
coverBlobHashes.push(artRow.coverBlobHash);
|
||||
}
|
||||
const sessions = db
|
||||
.prepare('SELECT session_id FROM imm_sessions WHERE video_id = ?')
|
||||
.all(videoId) as Array<{ session_id: number }>;
|
||||
sessionIds.push(...sessions.map((session) => session.session_id));
|
||||
}
|
||||
|
||||
deleteSessionsByIds(db, sessionIds);
|
||||
const deleteLinesStmt = db.prepare('DELETE FROM imm_subtitle_lines WHERE video_id = ?');
|
||||
const deleteDailyStmt = db.prepare('DELETE FROM imm_daily_rollups WHERE video_id = ?');
|
||||
const deleteMonthlyStmt = db.prepare('DELETE FROM imm_monthly_rollups WHERE video_id = ?');
|
||||
const deleteArtStmt = db.prepare('DELETE FROM imm_media_art WHERE video_id = ?');
|
||||
const deleteVideoStmt = db.prepare('DELETE FROM imm_videos WHERE video_id = ?');
|
||||
for (const videoId of videoIds) {
|
||||
deleteLinesStmt.run(videoId);
|
||||
deleteDailyStmt.run(videoId);
|
||||
deleteMonthlyStmt.run(videoId);
|
||||
deleteArtStmt.run(videoId);
|
||||
deleteVideoStmt.run(videoId);
|
||||
}
|
||||
for (const coverBlobHash of new Set(coverBlobHashes)) {
|
||||
cleanupUnusedCoverArtBlobHash(db, coverBlobHash);
|
||||
}
|
||||
db.prepare('DELETE FROM imm_lifetime_anime WHERE anime_id = ?').run(animeId);
|
||||
db.prepare('DELETE FROM imm_anime WHERE anime_id = ?').run(animeId);
|
||||
applyLexicalRemovals(db, lexicalRemovals);
|
||||
rebuildLifetimeSummariesInTransaction(db);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
deleteMaintenanceBatch(db, [{ kind: 'anime', animeId }]);
|
||||
}
|
||||
|
||||
export function deleteVideo(db: DatabaseSync, videoId: number): void {
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const artRow = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT cover_blob_hash AS coverBlobHash
|
||||
FROM imm_media_art
|
||||
WHERE video_id = ?
|
||||
`,
|
||||
)
|
||||
.get(videoId) as { coverBlobHash: string | null } | undefined;
|
||||
const lexicalRemovals = planLexicalRemovalsForVideos(db, [videoId]);
|
||||
const sessions = db
|
||||
.prepare('SELECT session_id FROM imm_sessions WHERE video_id = ?')
|
||||
.all(videoId) as Array<{ session_id: number }>;
|
||||
|
||||
deleteSessionsByIds(
|
||||
db,
|
||||
sessions.map((session) => session.session_id),
|
||||
);
|
||||
db.prepare('DELETE FROM imm_subtitle_lines WHERE video_id = ?').run(videoId);
|
||||
db.prepare('DELETE FROM imm_daily_rollups WHERE video_id = ?').run(videoId);
|
||||
db.prepare('DELETE FROM imm_monthly_rollups WHERE video_id = ?').run(videoId);
|
||||
db.prepare('DELETE FROM imm_media_art WHERE video_id = ?').run(videoId);
|
||||
cleanupUnusedCoverArtBlobHash(db, artRow?.coverBlobHash ?? null);
|
||||
db.prepare('DELETE FROM imm_videos WHERE video_id = ?').run(videoId);
|
||||
applyLexicalRemovals(db, lexicalRemovals);
|
||||
rebuildLifetimeSummariesInTransaction(db);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
deleteMaintenanceBatch(db, [{ kind: 'video', videoId }]);
|
||||
}
|
||||
|
||||
@@ -80,6 +80,14 @@ export function makePlaceholders(values: number[]): string {
|
||||
return values.map(() => '?').join(',');
|
||||
}
|
||||
|
||||
export const SQLITE_ID_CHUNK_SIZE = 1_000;
|
||||
|
||||
export function forEachIdChunk(ids: number[], callback: (chunk: number[]) => void): void {
|
||||
for (let start = 0; start < ids.length; start += SQLITE_ID_CHUNK_SIZE) {
|
||||
callback(ids.slice(start, start + SQLITE_ID_CHUNK_SIZE));
|
||||
}
|
||||
}
|
||||
|
||||
export function resolvedCoverBlobExpr(mediaAlias: string, blobStoreAlias: string): string {
|
||||
return `COALESCE(${blobStoreAlias}.cover_blob, CASE WHEN ${mediaAlias}.cover_blob_hash IS NULL THEN ${mediaAlias}.cover_blob ELSE NULL END)`;
|
||||
}
|
||||
@@ -268,6 +276,19 @@ export function planLexicalRemovalsForSessions(
|
||||
return planLexicalRemovals(db, `sl.session_id IN (${makePlaceholders(sessionIds)})`, sessionIds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Measure what deleting these individual subtitle lines removes from the vocabulary
|
||||
* tables. Used by the duplicate-line cleanup, which drops animation frames out of the
|
||||
* middle of sessions that otherwise stay intact.
|
||||
*/
|
||||
export function planLexicalRemovalsForLines(
|
||||
db: DatabaseSync,
|
||||
lineIds: number[],
|
||||
): LexicalRemovalPlan {
|
||||
if (lineIds.length === 0) return EMPTY_LEXICAL_REMOVAL_PLAN;
|
||||
return planLexicalRemovals(db, `sl.line_id IN (${makePlaceholders(lineIds)})`, lineIds);
|
||||
}
|
||||
|
||||
/** Measure what deleting these videos removes from the vocabulary tables. */
|
||||
export function planLexicalRemovalsForVideos(
|
||||
db: DatabaseSync,
|
||||
@@ -286,10 +307,13 @@ function toStoredSeenSeconds(ms: number | null): number | null {
|
||||
* Apply a removal plan to the vocabulary aggregates.
|
||||
*
|
||||
* Frequencies are adjusted by subtraction, which is exact and touches only the
|
||||
* affected rows. `first_seen`/`last_seen` only need a rescan when the removed
|
||||
* lines held the current extreme, and rows whose frequency reaches zero are
|
||||
* verified against the surviving occurrences before deletion — so stored counts
|
||||
* that have drifted still converge on the truth instead of dropping a live row.
|
||||
* affected rows. When the removed lines held a `first_seen`/`last_seen`
|
||||
* extreme, the new extremes come from MIN/MAX index-endpoint seeks on the
|
||||
* occurrence covering index — never a re-aggregation of every occurrence, which
|
||||
* for common particles means scanning the whole library. The full re-aggregate
|
||||
* survives only as the repair path: rows whose stored frequency reaches zero
|
||||
* while occurrences remain (drift), and rows with undated pre-migration
|
||||
* occurrences the seeks would skip.
|
||||
*/
|
||||
export function applyLexicalRemovals(db: DatabaseSync, plan: LexicalRemovalPlan): void {
|
||||
applyRemovalsForEntity(db, 'word', plan.words);
|
||||
@@ -318,6 +342,21 @@ function applyRemovalsForEntity(
|
||||
`SELECT 1 AS found FROM ${occurrenceTable} WHERE ${col} = ? LIMIT 1`,
|
||||
);
|
||||
const deleteStmt = db.prepare(`DELETE FROM ${entityTable} WHERE id = ?`);
|
||||
// Seeks to the front of this entity's index range, where NULL seen_ms sorts.
|
||||
const hasUndatedOccurrenceStmt = db.prepare(
|
||||
`SELECT 1 AS found FROM ${occurrenceTable} WHERE ${col} = ? AND seen_ms IS NULL LIMIT 1`,
|
||||
);
|
||||
// Kept as separate single-aggregate statements so SQLite's min/max
|
||||
// optimization turns each into an index-endpoint seek instead of a scan.
|
||||
const minSeenStmt = db.prepare(
|
||||
`SELECT MIN(seen_ms) AS value FROM ${occurrenceTable} WHERE ${col} = ?`,
|
||||
);
|
||||
const maxSeenStmt = db.prepare(
|
||||
`SELECT MAX(seen_ms) AS value FROM ${occurrenceTable} WHERE ${col} = ?`,
|
||||
);
|
||||
const updateAggregatesStmt = db.prepare(
|
||||
`UPDATE ${entityTable} SET frequency = ?, first_seen = ?, last_seen = ? WHERE id = ?`,
|
||||
);
|
||||
|
||||
const needsExactRefresh: number[] = [];
|
||||
|
||||
@@ -350,7 +389,26 @@ function applyRemovalsForEntity(
|
||||
current.lastSeen === null ||
|
||||
(removedLastSeen !== null && removedLastSeen >= current.lastSeen);
|
||||
if (firstSeenMayHaveMoved || lastSeenMayHaveMoved) {
|
||||
needsExactRefresh.push(removal.id);
|
||||
// Undated pre-migration occurrences are invisible to the seeks below;
|
||||
// fall back to the full re-aggregate that resolves their dates.
|
||||
if (hasUndatedOccurrenceStmt.get(removal.id)) {
|
||||
needsExactRefresh.push(removal.id);
|
||||
continue;
|
||||
}
|
||||
const minSeenMs = (minSeenStmt.get(removal.id) as { value: number | null }).value;
|
||||
const maxSeenMs = (maxSeenStmt.get(removal.id) as { value: number | null }).value;
|
||||
if (minSeenMs === null || maxSeenMs === null) {
|
||||
// Frequency says occurrences remain but none exist: stale row, let the
|
||||
// exact refresh reconcile (it deletes rows with nothing left).
|
||||
needsExactRefresh.push(removal.id);
|
||||
continue;
|
||||
}
|
||||
updateAggregatesStmt.run(
|
||||
nextFrequency,
|
||||
Math.floor(Number(minSeenMs) / 1000),
|
||||
Math.floor(Number(maxSeenMs) / 1000),
|
||||
removal.id,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -490,17 +548,19 @@ export function deleteSessionsByIds(db: DatabaseSync, sessionIds: number[]): voi
|
||||
return;
|
||||
}
|
||||
|
||||
const placeholders = makePlaceholders(sessionIds);
|
||||
db.prepare(`DELETE FROM imm_subtitle_lines WHERE session_id IN (${placeholders})`).run(
|
||||
...sessionIds,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_session_telemetry WHERE session_id IN (${placeholders})`).run(
|
||||
...sessionIds,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_session_events WHERE session_id IN (${placeholders})`).run(
|
||||
...sessionIds,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_sessions WHERE session_id IN (${placeholders})`).run(...sessionIds);
|
||||
forEachIdChunk(sessionIds, (chunk) => {
|
||||
const placeholders = makePlaceholders(chunk);
|
||||
db.prepare(`DELETE FROM imm_subtitle_lines WHERE session_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_session_telemetry WHERE session_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_session_events WHERE session_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_sessions WHERE session_id IN (${placeholders})`).run(...chunk);
|
||||
});
|
||||
}
|
||||
|
||||
export function toDbMs(ms: number | bigint): bigint {
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
export { Texthooker } from './texthooker';
|
||||
export { hasMpvWebsocketPlugin, SubtitleWebSocket } from './subtitle-ws';
|
||||
export {
|
||||
hasMpvWebsocketPlugin,
|
||||
isSubtitleAnnotationUpgrade,
|
||||
SubtitleWebSocket,
|
||||
} from './subtitle-ws';
|
||||
export { registerGlobalShortcuts } from './shortcut';
|
||||
export { createIpcDepsRuntime, registerIpcHandlers } from './ipc';
|
||||
export { shortcutMatchesInputForLocalFallback } from './shortcut-fallback';
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import electron from 'electron';
|
||||
import type { BrowserWindow as ElectronBrowserWindow, IpcMainEvent } from 'electron';
|
||||
import type {
|
||||
ChangelogSnapshot,
|
||||
CompiledSessionBinding,
|
||||
ControllerConfigUpdate,
|
||||
PlaylistBrowserMutationResult,
|
||||
@@ -122,6 +123,7 @@ export interface IpcServiceDeps {
|
||||
removeCharacterDictionaryManagedEntry?: (mediaId: number) => Promise<unknown>;
|
||||
moveCharacterDictionaryManagedEntry?: (mediaId: number, direction: 1 | -1) => Promise<unknown>;
|
||||
appendClipboardVideoToQueue: () => { ok: boolean; message: string };
|
||||
getChangelogSnapshot?: (options?: { refresh?: boolean }) => Promise<ChangelogSnapshot>;
|
||||
getPlaylistBrowserSnapshot: () => Promise<PlaylistBrowserSnapshot>;
|
||||
appendPlaylistBrowserFile: (filePath: string) => Promise<PlaylistBrowserMutationResult>;
|
||||
playPlaylistBrowserIndex: (index: number) => Promise<PlaylistBrowserMutationResult>;
|
||||
@@ -297,6 +299,7 @@ export interface IpcDepsRuntimeOptions {
|
||||
removeCharacterDictionaryManagedEntry?: (mediaId: number) => Promise<unknown>;
|
||||
moveCharacterDictionaryManagedEntry?: (mediaId: number, direction: 1 | -1) => Promise<unknown>;
|
||||
appendClipboardVideoToQueue: () => { ok: boolean; message: string };
|
||||
getChangelogSnapshot?: (options?: { refresh?: boolean }) => Promise<ChangelogSnapshot>;
|
||||
getPlaylistBrowserSnapshot: () => Promise<PlaylistBrowserSnapshot>;
|
||||
appendPlaylistBrowserFile: (filePath: string) => Promise<PlaylistBrowserMutationResult>;
|
||||
playPlaylistBrowserIndex: (index: number) => Promise<PlaylistBrowserMutationResult>;
|
||||
@@ -418,6 +421,7 @@ export function createIpcDepsRuntime(options: IpcDepsRuntimeOptions): IpcService
|
||||
entries: [],
|
||||
})),
|
||||
appendClipboardVideoToQueue: options.appendClipboardVideoToQueue,
|
||||
getChangelogSnapshot: options.getChangelogSnapshot,
|
||||
getPlaylistBrowserSnapshot: options.getPlaylistBrowserSnapshot,
|
||||
appendPlaylistBrowserFile: options.appendPlaylistBrowserFile,
|
||||
playPlaylistBrowserIndex: options.playPlaylistBrowserIndex,
|
||||
@@ -820,6 +824,17 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar
|
||||
return deps.appendClipboardVideoToQueue();
|
||||
});
|
||||
|
||||
ipc.handle(IPC_CHANNELS.request.getChangelogSnapshot, async (_event, payload: unknown) => {
|
||||
const refresh =
|
||||
typeof payload === 'object' && payload !== null && 'refresh' in payload
|
||||
? (payload as { refresh?: unknown }).refresh === true
|
||||
: false;
|
||||
if (!deps.getChangelogSnapshot) {
|
||||
throw new Error('Changelog service is unavailable.');
|
||||
}
|
||||
return await deps.getChangelogSnapshot({ refresh });
|
||||
});
|
||||
|
||||
ipc.handle(IPC_CHANNELS.request.getPlaylistBrowserSnapshot, async () => {
|
||||
return await deps.getPlaylistBrowserSnapshot();
|
||||
});
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user