Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
695613b1e7
|
||
|
|
20797772f8 | ||
|
|
72c312810a
|
||
|
|
b61f050e78
|
||
|
|
2a77ba9dd4 | ||
|
|
4ed878270f
|
||
|
|
1d1b0c7bb6
|
||
|
|
db61ce358d
|
||
|
|
e9778a945a | ||
|
|
7de73e16a1 | ||
|
|
273652f781 | ||
|
|
14c0086520
|
||
|
|
ff0bc0af6d | ||
|
|
00b1b79bf4 | ||
|
|
e11a5fea0d
|
||
|
|
f73fe179d0
|
||
|
|
2938e7a32a | ||
|
|
82f6b4705a | ||
|
|
a02c33dac4 |
@@ -274,7 +274,8 @@ jobs:
|
|||||||
config.example.jsonc \
|
config.example.jsonc \
|
||||||
plugin/subminer \
|
plugin/subminer \
|
||||||
plugin/subminer.conf \
|
plugin/subminer.conf \
|
||||||
assets/themes/subminer.rasi
|
assets/themes/subminer.rasi \
|
||||||
|
assets/thumbnailers/subminer-ffmpegthumbnailer.thumbnailer
|
||||||
|
|
||||||
- name: Generate checksums
|
- name: Generate checksums
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -273,7 +273,8 @@ jobs:
|
|||||||
config.example.jsonc \
|
config.example.jsonc \
|
||||||
plugin/subminer \
|
plugin/subminer \
|
||||||
plugin/subminer.conf \
|
plugin/subminer.conf \
|
||||||
assets/themes/subminer.rasi
|
assets/themes/subminer.rasi \
|
||||||
|
assets/thumbnailers/subminer-ffmpegthumbnailer.thumbnailer
|
||||||
|
|
||||||
- name: Generate checksums
|
- name: Generate checksums
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -2,8 +2,10 @@
|
|||||||
|
|
||||||
APP_NAME := subminer
|
APP_NAME := subminer
|
||||||
THEME_SOURCE := assets/themes/subminer.rasi
|
THEME_SOURCE := assets/themes/subminer.rasi
|
||||||
|
THUMBNAILER_SOURCE := assets/thumbnailers/subminer-ffmpegthumbnailer.thumbnailer
|
||||||
LAUNCHER_OUT := dist/launcher/$(APP_NAME)
|
LAUNCHER_OUT := dist/launcher/$(APP_NAME)
|
||||||
THEME_FILE := subminer.rasi
|
THEME_FILE := subminer.rasi
|
||||||
|
THUMBNAILER_FILE := subminer-ffmpegthumbnailer.thumbnailer
|
||||||
|
|
||||||
# Default install prefix for the wrapper script.
|
# Default install prefix for the wrapper script.
|
||||||
PREFIX ?= $(HOME)/.local
|
PREFIX ?= $(HOME)/.local
|
||||||
@@ -221,11 +223,13 @@ docs-dev: ensure-bun
|
|||||||
|
|
||||||
|
|
||||||
install-linux: build-launcher
|
install-linux: build-launcher
|
||||||
@printf '%s\n' "[INFO] Installing Linux wrapper/theme artifacts"
|
@printf '%s\n' "[INFO] Installing Linux wrapper/support artifacts"
|
||||||
@install -d "$(BINDIR)"
|
@install -d "$(BINDIR)"
|
||||||
@install -m 0755 "$(LAUNCHER_OUT)" "$(BINDIR)/$(APP_NAME)"
|
@install -m 0755 "$(LAUNCHER_OUT)" "$(BINDIR)/$(APP_NAME)"
|
||||||
@install -d "$(LINUX_DATA_DIR)/themes"
|
@install -d "$(LINUX_DATA_DIR)/themes"
|
||||||
@install -m 0644 "./$(THEME_SOURCE)" "$(LINUX_DATA_DIR)/themes/$(THEME_FILE)"
|
@install -m 0644 "./$(THEME_SOURCE)" "$(LINUX_DATA_DIR)/themes/$(THEME_FILE)"
|
||||||
|
@install -d "$(LINUX_DATA_DIR)/thumbnailers"
|
||||||
|
@install -m 0644 "./$(THUMBNAILER_SOURCE)" "$(LINUX_DATA_DIR)/thumbnailers/$(THUMBNAILER_FILE)"
|
||||||
@install -d "$(LINUX_DATA_DIR)/plugin/subminer"
|
@install -d "$(LINUX_DATA_DIR)/plugin/subminer"
|
||||||
@cp -R ./plugin/subminer/. "$(LINUX_DATA_DIR)/plugin/subminer/"
|
@cp -R ./plugin/subminer/. "$(LINUX_DATA_DIR)/plugin/subminer/"
|
||||||
@if [ -n "$(APPIMAGE_SRC)" ]; then \
|
@if [ -n "$(APPIMAGE_SRC)" ]; then \
|
||||||
@@ -234,7 +238,7 @@ install-linux: build-launcher
|
|||||||
printf '%s\n' "[WARN] No release/SubMiner-*.AppImage found; skipping AppImage install"; \
|
printf '%s\n' "[WARN] No release/SubMiner-*.AppImage found; skipping AppImage install"; \
|
||||||
printf '%s\n' " Build one with: make build"; \
|
printf '%s\n' " Build one with: make build"; \
|
||||||
fi
|
fi
|
||||||
@printf '%s\n' "Installed to:" " $(BINDIR)/subminer" " $(LINUX_DATA_DIR)/themes/$(THEME_FILE)"
|
@printf '%s\n' "Installed to:" " $(BINDIR)/subminer" " $(LINUX_DATA_DIR)/themes/$(THEME_FILE)" " $(LINUX_DATA_DIR)/thumbnailers/$(THUMBNAILER_FILE)"
|
||||||
|
|
||||||
install-macos: build-launcher
|
install-macos: build-launcher
|
||||||
@printf '%s\n' "[INFO] Installing macOS wrapper/theme/app artifacts"
|
@printf '%s\n' "[INFO] Installing macOS wrapper/theme/app artifacts"
|
||||||
@@ -275,8 +279,9 @@ uninstall:
|
|||||||
uninstall-linux:
|
uninstall-linux:
|
||||||
@rm -f "$(BINDIR)/subminer" "$(BINDIR)/SubMiner.AppImage"
|
@rm -f "$(BINDIR)/subminer" "$(BINDIR)/SubMiner.AppImage"
|
||||||
@rm -f "$(LINUX_DATA_DIR)/themes/$(THEME_FILE)"
|
@rm -f "$(LINUX_DATA_DIR)/themes/$(THEME_FILE)"
|
||||||
|
@rm -f "$(LINUX_DATA_DIR)/thumbnailers/$(THUMBNAILER_FILE)"
|
||||||
@rm -rf "$(LINUX_DATA_DIR)/plugin/subminer"
|
@rm -rf "$(LINUX_DATA_DIR)/plugin/subminer"
|
||||||
@printf '%s\n' "Removed:" " $(BINDIR)/subminer" " $(BINDIR)/SubMiner.AppImage" " $(LINUX_DATA_DIR)/themes/$(THEME_FILE)" " $(LINUX_DATA_DIR)/plugin/subminer"
|
@printf '%s\n' "Removed:" " $(BINDIR)/subminer" " $(BINDIR)/SubMiner.AppImage" " $(LINUX_DATA_DIR)/themes/$(THEME_FILE)" " $(LINUX_DATA_DIR)/thumbnailers/$(THUMBNAILER_FILE)" " $(LINUX_DATA_DIR)/plugin/subminer"
|
||||||
|
|
||||||
uninstall-macos:
|
uninstall-macos:
|
||||||
@rm -f "$(BINDIR)/subminer"
|
@rm -f "$(BINDIR)/subminer"
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ Integrates Yomitan and mpv - on-screen lookups, mine to Anki, and track immersio
|
|||||||
[](https://www.gnu.org/licenses/gpl-3.0)
|
[](https://www.gnu.org/licenses/gpl-3.0)
|
||||||
[](https://www.typescriptlang.org)
|
[](https://www.typescriptlang.org)
|
||||||
|
|
||||||
[](https://github.com/user-attachments/assets/89e61895-e2b7-4b47-8d50-a35afe4132b2)
|
[](https://github.com/user-attachments/assets/7abab8a9-4e4e-4f06-9f3c-9783e15a3807)
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
|
Before Width: | Height: | Size: 114 KiB |
|
Before Width: | Height: | Size: 23 MiB |
|
Before Width: | Height: | Size: 303 KiB |
|
Before Width: | Height: | Size: 3.0 MiB After Width: | Height: | Size: 3.8 MiB |
@@ -0,0 +1,4 @@
|
|||||||
|
[Thumbnailer Entry]
|
||||||
|
TryExec=ffmpegthumbnailer
|
||||||
|
Exec=ffmpegthumbnailer -i %i -o %o -s %s -f
|
||||||
|
MimeType=video/matroska;video/matroska-3d;video/x-matroska;video/x-matroska-3d;
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
type: fixed
|
||||||
|
area: subtitles
|
||||||
|
|
||||||
|
- Typeset ASS karaoke and animated signs no longer flood the primary overlay, subtitle sidebar, immersion history, or sentence mining with repeated glyph fragments or full-line color phases. Matching timed comments and full-line boundary events recover the complete authored line without merging ordinary repeated dialogue or separately positioned signs, and dialogue spoken while a song's animation is on screen is kept intact instead of being replaced by the lyric. Entrance and exit frames that run past the authored line timing still resolve to the clean line during lyric transitions, and dialogue spoken while a song's animation is on screen enters immersion and subtitle history without the fragment lines beside it.
|
||||||
|
- The secondary subtitle overlay drops layered duplicate lines from animated tracks, so a short stack of repeated words collapses to its distinct lines even when the full karaoke heuristic does not apply.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
type: fixed
|
||||||
|
area: character dictionary
|
||||||
|
|
||||||
|
- Reuse character dictionaries after MeCab completes without finding any name splits instead of regenerating character data and portraits on every launch.
|
||||||
|
- Restore inline character portraits when a cached portrait index finishes loading after subtitles have already been tokenized.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
type: fixed
|
||||||
|
area: dictionary
|
||||||
|
|
||||||
|
- Character dictionary generation, merged rebuilds, and imports no longer freeze the app (and trigger the compositor's "application not responding" dialog) on large dictionaries; snapshot reads/writes, archive building, and the character image/name lookup caches now do their heavy work off the UI's critical path.
|
||||||
|
- Desktop progress notifications now update in place on Linux AppImage installs too: the AppImage's bundled libraries broke the system notify-send helper, which silently forced the flickering close-and-reopen notification fallback.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
type: internal
|
||||||
|
area: docs
|
||||||
|
|
||||||
|
- Excluded the `/main/` and `/v/<version>/` docs trees from search indexing with a self-referential canonical, `noindex,follow`, and a matching `X-Robots-Tag` header, so crawlers spend their budget on the current docs instead of ~30 archived copies of every page.
|
||||||
|
- Restored `<lastmod>` dates in the docs sitemap, which were silently dropped because production builds render from an untracked release snapshot.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
type: fixed
|
||||||
|
area: overlay
|
||||||
|
|
||||||
|
- Dedicated overlay modals are prewarmed on macOS and Windows so shortcuts open them promptly on the first press. Windows now refreshes the hidden modal renderer between sessions to keep later modals interactive. On macOS, reused modals and the in-app stats window also open above fullscreen mpv on its current Space instead of appearing on another desktop or forcing a Space change.
|
||||||
|
- Updated subtitle ASS observation to mpv's current `sub-text/ass` property, removing its deprecation warning.
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
type: fixed
|
||||||
|
area: overlay
|
||||||
|
|
||||||
|
- Fixed the overlay getting stuck on "Overlay loading" forever when startup stalls: mpv IPC connection attempts now time out and retry, switching sockets aborts obsolete attempts, and the plugin replaces its spinner with an actionable error if overlay content is still not ready after 30 seconds.
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
type: fixed
|
||||||
|
area: subtitles
|
||||||
|
|
||||||
|
- Primary ASS subtitles now use the active parsed cue when it fully accounts for mpv's live text, preventing fill, border, blur, and shadow copies of the same full-span lyric from appearing repeatedly while preserving unmatched overlapping dialogue and signs.
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
type: fixed
|
||||||
|
area: overlay
|
||||||
|
|
||||||
|
- Secondary subtitles now parse the selected ASS/SRT/VTT source with the primary subtitle deduplication pipeline, preventing layered animation text from appearing several times in the overlay, mined cards, and statistics. Live mpv text remains the fallback for unreadable tracks.
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
type: fixed
|
||||||
|
area: overlay
|
||||||
|
|
||||||
|
- Fixed system-wide mouse lag on Windows while SubMiner is running: the overlay no longer installs Electron's global mouse hook for click-through forwarding, and the mpv window tracker no longer blocks the app on repeated PowerShell command-line lookups.
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
type: docs
|
|
||||||
area: documentation
|
|
||||||
|
|
||||||
- Hid the unfinished feature demos page from the documentation sidebar while keeping its direct URL available.
|
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
type: fixed
|
||||||
|
area: launcher
|
||||||
|
|
||||||
|
- Fixed missing MKV thumbnails in the Linux rofi picker when system thumbnailer registrations only advertise legacy Matroska MIME aliases.
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
type: fixed
|
||||||
|
area: stats
|
||||||
|
|
||||||
|
- Fixed Vocabulary totals and charts counting only the first browsing page instead of all tracked vocabulary, without delaying the rest of the page.
|
||||||
|
- New-word history now uses permanent daily lexical rollups that apply the same vocabulary filters as the totals and normalize legacy second/millisecond timestamps; versioned background rebuilds repair existing history across legacy rollup-state schemas without dropping playback writes or clearing watch-time, activity, efficiency, and library charts.
|
||||||
|
- Calendar-day chart labels now preserve the recorded local date in time zones west of UTC.
|
||||||
|
- Vocabulary summary cards and charts refresh automatically after the word exclusion list changes, and failed or unfinished loads use bounded retries before showing an inline error with a Retry control.
|
||||||
|
- Rapid exclusion edits no longer race each other; writes are sent in order so a slower earlier save cannot overwrite a newer list.
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { spawnSync } from 'node:child_process';
|
||||||
import { existsSync, readFileSync, statSync } from 'node:fs';
|
import { existsSync, readFileSync, statSync } from 'node:fs';
|
||||||
import { extname, join, posix, resolve, sep } from 'node:path';
|
import { extname, join, posix, resolve, sep } from 'node:path';
|
||||||
import type { DefaultTheme, HeadConfig, TransformContext, UserConfig } from 'vitepress';
|
import type { DefaultTheme, HeadConfig, TransformContext, UserConfig } from 'vitepress';
|
||||||
@@ -26,6 +27,9 @@ function optionalEnv(value: string | undefined): string | undefined {
|
|||||||
const base = normalizeBase(optionalEnv(process.env.SUBMINER_DOCS_BASE) ?? '/');
|
const base = normalizeBase(optionalEnv(process.env.SUBMINER_DOCS_BASE) ?? '/');
|
||||||
const outDir = optionalEnv(process.env.SUBMINER_DOCS_OUT_DIR);
|
const outDir = optionalEnv(process.env.SUBMINER_DOCS_OUT_DIR);
|
||||||
const docsSourceDir = optionalEnv(process.env.SUBMINER_DOCS_SOURCE_DIR) ?? process.cwd();
|
const docsSourceDir = optionalEnv(process.env.SUBMINER_DOCS_SOURCE_DIR) ?? process.cwd();
|
||||||
|
// The tracked `docs-site/` checkout, which stays a git working tree even when
|
||||||
|
// `docsSourceDir` points at an untracked release snapshot. Used for git lookups only.
|
||||||
|
const repoDocsDir = optionalEnv(process.env.SUBMINER_DOCS_REPO_DIR) ?? process.cwd();
|
||||||
const channel = normalizeChannel(optionalEnv(process.env.SUBMINER_DOCS_CHANNEL));
|
const channel = normalizeChannel(optionalEnv(process.env.SUBMINER_DOCS_CHANNEL));
|
||||||
const docsVersion = optionalEnv(process.env.SUBMINER_DOCS_VERSION);
|
const docsVersion = optionalEnv(process.env.SUBMINER_DOCS_VERSION);
|
||||||
const latestStable = optionalEnv(process.env.SUBMINER_DOCS_LATEST_STABLE) ?? 'v0.18.0';
|
const latestStable = optionalEnv(process.env.SUBMINER_DOCS_LATEST_STABLE) ?? 'v0.18.0';
|
||||||
@@ -82,15 +86,18 @@ function pageToRoute(page: string): string | null {
|
|||||||
return route ? `/${route}` : '/';
|
return route ? `/${route}` : '/';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only the root channel is indexable. `main` and every /v/<version>/ archive are
|
||||||
|
// near-verbatim copies of it, so they own their URL via a self-referential canonical
|
||||||
|
// and are excluded from the index instead of being consolidated onto root. Uniform
|
||||||
|
// self-canonical plus noindex avoids mixing noindex with a cross-page canonical,
|
||||||
|
// which Google treats as a conflicting signal.
|
||||||
|
const isIndexableChannel = channel === 'stable-root';
|
||||||
|
|
||||||
function pageToCanonicalHref(page: string): string | null {
|
function pageToCanonicalHref(page: string): string | null {
|
||||||
const route = pageToRoute(page);
|
const route = pageToRoute(page);
|
||||||
if (!route) return null;
|
if (!route) return null;
|
||||||
|
|
||||||
if (channel === 'main') {
|
if (!isIndexableChannel) {
|
||||||
return `${DOCS_HOSTNAME}${canonicalRouteWithBase(route)}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (channel === 'stable-archive' && docsVersion !== latestStable) {
|
|
||||||
return `${DOCS_HOSTNAME}${canonicalRouteWithBase(route)}`;
|
return `${DOCS_HOSTNAME}${canonicalRouteWithBase(route)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -106,7 +113,9 @@ function transformPageHead({ page }: TransformContext): HeadConfig[] {
|
|||||||
const href = pageToCanonicalHref(page);
|
const href = pageToCanonicalHref(page);
|
||||||
const head: HeadConfig[] = href ? [['link', { rel: 'canonical', href }]] : [];
|
const head: HeadConfig[] = href ? [['link', { rel: 'canonical', href }]] : [];
|
||||||
|
|
||||||
if (channel === 'main') {
|
// Crawlable so links still pass through, but out of the index: ~30 archived copies
|
||||||
|
// of every page otherwise soak up the crawl budget the current docs need.
|
||||||
|
if (!isIndexableChannel) {
|
||||||
head.push(['meta', { name: 'robots', content: 'noindex,follow' }]);
|
head.push(['meta', { name: 'robots', content: 'noindex,follow' }]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -287,6 +296,39 @@ const versionItems = [
|
|||||||
})),
|
})),
|
||||||
];
|
];
|
||||||
|
|
||||||
|
function sitemapUrlToPage(url: string): string {
|
||||||
|
const route = url.replace(/\.html$/, '').replace(/^\/+|\/+$/g, '');
|
||||||
|
return route ? `${route}.md` : 'index.md';
|
||||||
|
}
|
||||||
|
|
||||||
|
// VitePress derives <lastmod> by running `git log` inside its source dir. Production
|
||||||
|
// builds point that at an untracked snapshot of the release tag, so the lookup comes
|
||||||
|
// back empty and the sitemap ships with no dates at all. Resolve it from the tracked
|
||||||
|
// checkout at the ref being built instead.
|
||||||
|
function lastModifiedFor(url: string): string | undefined {
|
||||||
|
const ref = docsVersion && docsVersion !== 'main' ? docsVersion : 'HEAD';
|
||||||
|
const result = spawnSync('git', ['log', '-1', '--format=%cI', ref, '--', sitemapUrlToPage(url)], {
|
||||||
|
cwd: repoDocsDir,
|
||||||
|
encoding: 'utf8',
|
||||||
|
});
|
||||||
|
|
||||||
|
return (result.status === 0 && result.stdout.trim()) || undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only the root channel publishes a sitemap. Archived and `main` builds would emit
|
||||||
|
// their own copies listing the same canonical URLs, which just advertises the
|
||||||
|
// duplicate trees we are trying to keep out of the index.
|
||||||
|
const sitemap: UserConfig['sitemap'] = isIndexableChannel
|
||||||
|
? {
|
||||||
|
hostname: DOCS_HOSTNAME,
|
||||||
|
transformItems(items) {
|
||||||
|
return items
|
||||||
|
.filter((item) => item.url !== 'README' && item.url !== `${DOCS_HOSTNAME}/README`)
|
||||||
|
.map((item) => ({ ...item, lastmod: item.lastmod ?? lastModifiedFor(item.url) }));
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: undefined;
|
||||||
|
|
||||||
const nav: DefaultTheme.NavItem[] = [
|
const nav: DefaultTheme.NavItem[] = [
|
||||||
{ text: 'Home', link: '/' },
|
{ text: 'Home', link: '/' },
|
||||||
{ text: 'Get Started', link: '/installation' },
|
{ text: 'Get Started', link: '/installation' },
|
||||||
@@ -419,14 +461,7 @@ const config: UserConfig = {
|
|||||||
appearance: 'dark',
|
appearance: 'dark',
|
||||||
cleanUrls: true,
|
cleanUrls: true,
|
||||||
metaChunk: true,
|
metaChunk: true,
|
||||||
sitemap: {
|
sitemap,
|
||||||
hostname: DOCS_HOSTNAME,
|
|
||||||
transformItems(items) {
|
|
||||||
return items.filter(
|
|
||||||
(item) => item.url !== 'README' && item.url !== `${DOCS_HOSTNAME}/README`,
|
|
||||||
);
|
|
||||||
},
|
|
||||||
},
|
|
||||||
transformHead: transformPageHead,
|
transformHead: transformPageHead,
|
||||||
lastUpdated: true,
|
lastUpdated: true,
|
||||||
srcExclude: ['subagents/**', 'README.md'],
|
srcExclude: ['subagents/**', 'README.md'],
|
||||||
|
|||||||
@@ -38,8 +38,10 @@ bun run docs:dev
|
|||||||
The public docs root is stable-only:
|
The public docs root is stable-only:
|
||||||
|
|
||||||
- `/` serves the latest stable release docs.
|
- `/` serves the latest stable release docs.
|
||||||
- `/main/` serves development docs from `main` and is marked `noindex,follow`.
|
- `/main/` serves development docs from `main`.
|
||||||
- `/v/<version>/` serves stable release archives.
|
- `/v/<version>/` serves stable release archives.
|
||||||
- Prerelease tags do not update the docs site.
|
- Prerelease tags do not update the docs site.
|
||||||
|
|
||||||
|
Only `/` is indexable. `/main/` and every `/v/<version>/` page carries a self-referential canonical plus `noindex,follow`, and the generated `_headers` file repeats that as an `X-Robots-Tag`. They stay crawlable so their links still resolve, but ~30 archived copies of every page would otherwise consume the crawl budget the current docs need. Only the root build emits `sitemap.xml`, and its `<lastmod>` dates come from `git log` against the tracked checkout at the released tag, because the build renders from an untracked snapshot that VitePress cannot date itself.
|
||||||
|
|
||||||
Keep Cloudflare Git auto-deploy disabled. The production deploy is `.github/workflows/docs-pages.yml`, which uploads `.tmp/docs-versioned-site` with `--branch main` so tag-triggered runs update Production instead of creating preview deployments.
|
Keep Cloudflare Git auto-deploy disabled. The production deploy is `.github/workflows/docs-pages.yml`, which uploads `.tmp/docs-versioned-site` with `--branch main` so tag-triggered runs update Production instead of creating preview deployments.
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ Short recordings of SubMiner's key features and integrations from real playback
|
|||||||
<script setup>
|
<script setup>
|
||||||
import { withBase } from 'vitepress';
|
import { withBase } from 'vitepress';
|
||||||
|
|
||||||
const v = '20260301-1';
|
const v = '20260819-1';
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
## Anki Card Mining & Enrichment
|
## Anki Card Mining & Enrichment
|
||||||
|
|||||||
@@ -82,7 +82,7 @@ Expandable session history with new-word activity, cumulative totals, and pause/
|
|||||||
|
|
||||||
#### Vocabulary
|
#### Vocabulary
|
||||||
|
|
||||||
Top repeated words (click a bar to open the word), new-word timeline, cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons.
|
The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The word and kanji tables load first while those complete totals calculate separately. Top Repeated Words and New Words by Day use complete tracking history rather than the table's browsing page. New-word history is maintained as a permanent daily lexical rollup using the same token-visibility rules as the totals, including normalization of older timestamps stored in either seconds or milliseconds and retroactive corrections when tracked material is removed or reprocessed. On the first launch after an applicable upgrade, that history is version-rebuilt in the background and the chart refreshes when it is ready; if it remains unavailable, polling stops and an inline Retry control appears. The cards and charts also refresh automatically after the word exclusion list changes. The rest of the tab includes cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons.
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
@@ -138,6 +138,8 @@ Karaoke openings and animated signs are authored as one subtitle event per anima
|
|||||||
|
|
||||||
Recording now collapses those runs as they happen, matching what the subtitle sidebar shows:
|
Recording now collapses those runs as they happen, matching what the subtitle sidebar shows:
|
||||||
|
|
||||||
|
- When a typeset ASS file stores a clean lyric or sign in a timed authoring comment, or in full-line events surrounding generated fragments, the matching complete line is recorded once. The repeated glyph or clip-animation frames are not recorded. Dialogue spoken while such an animation is on screen records as itself, without the fragment lines beside it.
|
||||||
|
- When karaoke styling redraws the same complete lyric across consecutive color or highlight phases, those phases are combined into one line with their full timing. Repeated ordinary dialogue remains separate.
|
||||||
- When the active subtitle source has been parsed, its cue list has already had duplicate events and animation bursts merged. A line landing inside a surviving cue but after that cue's start is a frame the sidebar merged away, and is not recorded.
|
- When the active subtitle source has been parsed, its cue list has already had duplicate events and animation bursts merged. A line landing inside a surviving cue but after that cue's start is a frame the sidebar merged away, and is not recorded.
|
||||||
- When no parsed cue covers the live timing, including while a subtitle source is changing or shifted, the strict metadata-free rule applies: a run of identical, contiguous lines each shorter than 0.1s stops being recorded after a few frames. Runs are tracked per line of text, so dual-line karaoke (a kanji and a romaji line frame-flipped together) collapses both lines. Ordinary repeated dialogue, and lines held for a normal beat, always record.
|
- When no parsed cue covers the live timing, including while a subtitle source is changing or shifted, the strict metadata-free rule applies: a run of identical, contiguous lines each shorter than 0.1s stops being recorded after a few frames. Runs are tracked per line of text, so dual-line karaoke (a kanji and a romaji line frame-flipped together) collapses both lines. Ordinary repeated dialogue, and lines held for a normal beat, always record.
|
||||||
|
|
||||||
@@ -180,6 +182,7 @@ In practice:
|
|||||||
- Anime and episode pages keep lifetime totals from summary tables while session drill-down still reads retained sessions directly. With the current defaults, both are kept forever.
|
- Anime and episode pages keep lifetime totals from summary tables while session drill-down still reads retained sessions directly. With the current defaults, both are kept forever.
|
||||||
- Trends can read the full available history because daily/monthly rollups are also kept forever by default.
|
- Trends can read the full available history because daily/monthly rollups are also kept forever by default.
|
||||||
- Vocabulary and kanji totals are cumulative and not bounded by the raw session retention knobs.
|
- Vocabulary and kanji totals are cumulative and not bounded by the raw session retention knobs.
|
||||||
|
- New-word charts use their own permanent lexical daily rollups, which are not pruned by activity-rollup retention.
|
||||||
|
|
||||||
## Storage / Performance Model
|
## Storage / Performance Model
|
||||||
|
|
||||||
@@ -349,6 +352,7 @@ Rollup tables:
|
|||||||
|
|
||||||
- `imm_daily_rollups`
|
- `imm_daily_rollups`
|
||||||
- `imm_monthly_rollups`
|
- `imm_monthly_rollups`
|
||||||
|
- `imm_lexical_daily_rollups` - permanent first-discovery counts for vocabulary and kanji chart history
|
||||||
- `imm_rollup_state` - incremental rollup progress bookkeeping
|
- `imm_rollup_state` - incremental rollup progress bookkeeping
|
||||||
|
|
||||||
Vocabulary tables:
|
Vocabulary tables:
|
||||||
|
|||||||
@@ -88,7 +88,7 @@ features:
|
|||||||
<script setup>
|
<script setup>
|
||||||
import { withBase } from 'vitepress';
|
import { withBase } from 'vitepress';
|
||||||
|
|
||||||
const demoAssetVersion = '20260223-2';
|
const demoAssetVersion = '20260819-1';
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<div class="landing-shell">
|
<div class="landing-shell">
|
||||||
|
|||||||
@@ -392,7 +392,7 @@ subminer -u
|
|||||||
subminer --update
|
subminer --update
|
||||||
```
|
```
|
||||||
|
|
||||||
SubMiner verifies AppImage, launcher, and Linux support-asset downloads against `SHA256SUMS.txt`. On Linux those support assets include the launcher-managed runtime plugin copy under `SubMiner/plugin/subminer` plus the rofi theme at `SubMiner/themes/subminer.rasi`. If the binary is in a protected path, SubMiner shows the exact command to run rather than elevating itself.
|
SubMiner verifies AppImage, launcher, and Linux support-asset downloads against `SHA256SUMS.txt`. On Linux those support assets include the launcher-managed runtime plugin copy under `SubMiner/plugin/subminer`, the rofi theme at `SubMiner/themes/subminer.rasi`, and the scoped Matroska thumbnailer registration under `SubMiner/thumbnailers`. If the binary is in a protected path, SubMiner shows the exact command to run rather than elevating itself.
|
||||||
|
|
||||||
The tray "Check for Updates" entry installs the new app automatically on Linux, macOS, and Windows. On Linux it replaces the running `.AppImage` in place via `electron-updater` and refreshes the managed support assets from `subminer-assets.tar.gz`; AppImages managed by a system package (for example the AUR `/opt/SubMiner/SubMiner.AppImage`) are skipped so the package manager stays in charge.
|
The tray "Check for Updates" entry installs the new app automatically on Linux, macOS, and Windows. On Linux it replaces the running `.AppImage` in place via `electron-updater` and refreshes the managed support assets from `subminer-assets.tar.gz`; AppImages managed by a system package (for example the AUR `/opt/SubMiner/SubMiner.AppImage`) are skipped so the package manager stays in charge.
|
||||||
|
|
||||||
@@ -404,7 +404,7 @@ SubMiner is an overlay that sits on top of mpv. It connects to mpv through an IP
|
|||||||
|
|
||||||
The `subminer` launcher handles mpv IPC socket setup automatically. If you launch mpv yourself or from another tool, you must pass `--input-ipc-server=/tmp/subminer-socket` (or `\\.\pipe\subminer-socket` on Windows) - without it the overlay starts but subtitles won't appear.
|
The `subminer` launcher handles mpv IPC socket setup automatically. If you launch mpv yourself or from another tool, you must pass `--input-ipc-server=/tmp/subminer-socket` (or `\\.\pipe\subminer-socket` on Windows) - without it the overlay starts but subtitles won't appear.
|
||||||
|
|
||||||
The bundled mpv plugin is injected at runtime automatically - you don't need to install it separately. On Linux, the `subminer` launcher now checks for its managed runtime plugin copy and rofi theme before every mpv-managed launch and installs those support assets from the bundled app automatically if either one is missing. It provides in-player keybindings (the `y` chord) for controlling the overlay from within mpv. See [MPV Plugin](/mpv-plugin) for the full keybinding and configuration reference.
|
The bundled mpv plugin is injected at runtime automatically - you don't need to install it separately. On Linux, the `subminer` launcher checks for its managed runtime plugin copy, rofi theme, and scoped thumbnailer registration before every mpv-managed launch and installs those support assets from the bundled app automatically if one is missing. For a rofi picker launch, this check runs before the picker opens. It provides in-player keybindings (the `y` chord) for controlling the overlay from within mpv. See [MPV Plugin](/mpv-plugin) for the full keybinding and configuration reference.
|
||||||
|
|
||||||
## Platform Notes
|
## Platform Notes
|
||||||
|
|
||||||
@@ -456,18 +456,20 @@ sudo chmod +x /usr/local/bin/subminer
|
|||||||
|
|
||||||
### Linux Support Assets
|
### Linux Support Assets
|
||||||
|
|
||||||
SubMiner ships the Linux rofi theme plus the launcher-managed runtime plugin copy in `subminer-assets.tar.gz`:
|
SubMiner ships the Linux rofi theme, scoped Matroska thumbnailer registration, and launcher-managed runtime plugin copy in `subminer-assets.tar.gz`:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
wget https://github.com/ksyasuda/SubMiner/releases/latest/download/subminer-assets.tar.gz -O /tmp/subminer-assets.tar.gz
|
wget https://github.com/ksyasuda/SubMiner/releases/latest/download/subminer-assets.tar.gz -O /tmp/subminer-assets.tar.gz
|
||||||
tar -xzf /tmp/subminer-assets.tar.gz -C /tmp
|
tar -xzf /tmp/subminer-assets.tar.gz -C /tmp
|
||||||
mkdir -p ~/.local/share/SubMiner/themes
|
mkdir -p ~/.local/share/SubMiner/themes
|
||||||
cp /tmp/assets/themes/subminer.rasi ~/.local/share/SubMiner/themes/subminer.rasi
|
cp /tmp/assets/themes/subminer.rasi ~/.local/share/SubMiner/themes/subminer.rasi
|
||||||
|
mkdir -p ~/.local/share/SubMiner/thumbnailers
|
||||||
|
cp /tmp/assets/thumbnailers/subminer-ffmpegthumbnailer.thumbnailer ~/.local/share/SubMiner/thumbnailers/
|
||||||
mkdir -p ~/.local/share/SubMiner/plugin
|
mkdir -p ~/.local/share/SubMiner/plugin
|
||||||
cp -R /tmp/plugin/subminer ~/.local/share/SubMiner/plugin/subminer
|
cp -R /tmp/plugin/subminer ~/.local/share/SubMiner/plugin/subminer
|
||||||
```
|
```
|
||||||
|
|
||||||
`subminer -u` and the tray updater keep those Linux support assets in sync automatically once the `SubMiner` data dir exists. Normal Linux launcher playback also auto-installs the managed runtime plugin copy and rofi theme from the bundled app if either support asset is missing, so manual extraction is mainly useful for pre-seeding or custom setups.
|
`subminer -u` and the tray updater keep those Linux support assets in sync automatically once the `SubMiner` data dir exists. Normal Linux launcher playback also auto-installs all three assets from the bundled app if one is missing, so manual extraction is mainly useful for pre-seeding or custom setups. Rofi receives the SubMiner data path through its process-local `XDG_DATA_DIRS`, so the thumbnailer registration does not change the desktop-wide configuration.
|
||||||
|
|
||||||
Override the theme path with `SUBMINER_ROFI_THEME=/absolute/path/to/theme.rasi`.
|
Override the theme path with `SUBMINER_ROFI_THEME=/absolute/path/to/theme.rasi`.
|
||||||
|
|
||||||
|
|||||||
@@ -34,18 +34,22 @@ subminer -R -r -d ~/Anime # rofi picker, recursive
|
|||||||
subminer -R /directory # rofi picker, directory shortcut
|
subminer -R /directory # rofi picker, directory shortcut
|
||||||
```
|
```
|
||||||
|
|
||||||
rofi shows a GUI menu with icon thumbnails when available. SubMiner ships the rofi theme plus the Linux launcher-managed runtime plugin copy in the release assets tarball:
|
rofi shows a GUI menu with icon thumbnails when available. SubMiner ships the rofi theme, a scoped `ffmpegthumbnailer` MIME registration, and the Linux launcher-managed runtime plugin copy in the release assets tarball:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
wget https://github.com/ksyasuda/SubMiner/releases/latest/download/subminer-assets.tar.gz -O /tmp/subminer-assets.tar.gz
|
wget https://github.com/ksyasuda/SubMiner/releases/latest/download/subminer-assets.tar.gz -O /tmp/subminer-assets.tar.gz
|
||||||
tar -xzf /tmp/subminer-assets.tar.gz -C /tmp
|
tar -xzf /tmp/subminer-assets.tar.gz -C /tmp
|
||||||
mkdir -p ~/.local/share/SubMiner/themes
|
mkdir -p ~/.local/share/SubMiner/themes
|
||||||
cp /tmp/assets/themes/subminer.rasi ~/.local/share/SubMiner/themes/subminer.rasi
|
cp /tmp/assets/themes/subminer.rasi ~/.local/share/SubMiner/themes/subminer.rasi
|
||||||
|
mkdir -p ~/.local/share/SubMiner/thumbnailers
|
||||||
|
cp /tmp/assets/thumbnailers/subminer-ffmpegthumbnailer.thumbnailer ~/.local/share/SubMiner/thumbnailers/
|
||||||
mkdir -p ~/.local/share/SubMiner/plugin
|
mkdir -p ~/.local/share/SubMiner/plugin
|
||||||
cp -R /tmp/plugin/subminer ~/.local/share/SubMiner/plugin/subminer
|
cp -R /tmp/plugin/subminer ~/.local/share/SubMiner/plugin/subminer
|
||||||
```
|
```
|
||||||
|
|
||||||
Once the `SubMiner` data dir exists, `subminer -u` refreshes both assets automatically. Normal Linux launcher playback also checks for the managed runtime plugin copy and rofi theme before mpv launch and installs them from the bundled app automatically if either one is missing.
|
Once the `SubMiner` data dir exists, `subminer -u` refreshes these assets automatically. Normal Linux launcher playback checks for all three assets and installs them from the bundled app when one is missing. For `subminer -R`, this repair runs before rofi opens.
|
||||||
|
|
||||||
|
When `ffmpegthumbnailer` is installed, SubMiner prepends its own data directory to `XDG_DATA_DIRS` for the rofi process only. This lets rofi recognize the canonical Matroska MIME types used by newer GLib versions without changing the desktop-wide MIME or thumbnailer configuration. An existing registration in your own `$XDG_DATA_HOME/thumbnailers` still takes priority.
|
||||||
|
|
||||||
The theme is auto-detected from these paths (first match wins):
|
The theme is auto-detected from these paths (first match wins):
|
||||||
|
|
||||||
|
|||||||
|
Before Width: | Height: | Size: 56 KiB After Width: | Height: | Size: 160 KiB |
|
Before Width: | Height: | Size: 23 MiB |
|
Before Width: | Height: | Size: 303 KiB |
|
Before Width: | Height: | Size: 523 KiB |
|
Before Width: | Height: | Size: 3.0 MiB After Width: | Height: | Size: 3.8 MiB |
@@ -56,34 +56,43 @@ test('main docs canonical uses /main/ and emits noindex', async () => {
|
|||||||
{ rel: 'canonical', href: 'https://docs.subminer.moe/main/' },
|
{ rel: 'canonical', href: 'https://docs.subminer.moe/main/' },
|
||||||
]);
|
]);
|
||||||
expect(head).toContainEqual(['meta', { name: 'robots', content: 'noindex,follow' }]);
|
expect(head).toContainEqual(['meta', { name: 'robots', content: 'noindex,follow' }]);
|
||||||
|
expect(mainDocsConfig.sitemap).toBeUndefined();
|
||||||
|
|
||||||
process.env.SUBMINER_DOCS_CHANNEL = previousChannel;
|
process.env.SUBMINER_DOCS_CHANNEL = previousChannel;
|
||||||
process.env.SUBMINER_DOCS_BASE = previousBase;
|
process.env.SUBMINER_DOCS_BASE = previousBase;
|
||||||
});
|
});
|
||||||
|
|
||||||
test('latest stable archive canonical points to root equivalent', async () => {
|
test.each([
|
||||||
const previousChannel = process.env.SUBMINER_DOCS_CHANNEL;
|
['latest stable', 'v0.14.0', '/v/0.14.0/', 'https://docs.subminer.moe/v/0.14.0/usage'],
|
||||||
const previousBase = process.env.SUBMINER_DOCS_BASE;
|
['superseded', 'v0.12.0', '/v/0.12.0/', 'https://docs.subminer.moe/v/0.12.0/usage'],
|
||||||
const previousVersion = process.env.SUBMINER_DOCS_VERSION;
|
])(
|
||||||
const previousLatest = process.env.SUBMINER_DOCS_LATEST_STABLE;
|
'%s archive keeps a self-referential canonical and stays out of the index',
|
||||||
process.env.SUBMINER_DOCS_CHANNEL = 'stable-archive';
|
async (_label, version, base, expectedCanonical) => {
|
||||||
process.env.SUBMINER_DOCS_BASE = '/v/0.14.0/';
|
const previousChannel = process.env.SUBMINER_DOCS_CHANNEL;
|
||||||
process.env.SUBMINER_DOCS_VERSION = 'v0.14.0';
|
const previousBase = process.env.SUBMINER_DOCS_BASE;
|
||||||
process.env.SUBMINER_DOCS_LATEST_STABLE = 'v0.14.0';
|
const previousVersion = process.env.SUBMINER_DOCS_VERSION;
|
||||||
const { default: latestArchiveConfig } = await import('./.vitepress/config?latest-archive');
|
const previousLatest = process.env.SUBMINER_DOCS_LATEST_STABLE;
|
||||||
|
process.env.SUBMINER_DOCS_CHANNEL = 'stable-archive';
|
||||||
|
process.env.SUBMINER_DOCS_BASE = base;
|
||||||
|
process.env.SUBMINER_DOCS_VERSION = version;
|
||||||
|
process.env.SUBMINER_DOCS_LATEST_STABLE = 'v0.14.0';
|
||||||
|
try {
|
||||||
|
const { default: archiveConfig } = await import(`./.vitepress/config?archive-${version}`);
|
||||||
|
|
||||||
const head = await latestArchiveConfig.transformHead?.(makeTransformContext('usage.md'));
|
const head = await archiveConfig.transformHead?.(makeTransformContext('usage.md'));
|
||||||
|
|
||||||
expect(head).toContainEqual([
|
expect(head).toContainEqual(['link', { rel: 'canonical', href: expectedCanonical }]);
|
||||||
'link',
|
expect(head).toContainEqual(['meta', { name: 'robots', content: 'noindex,follow' }]);
|
||||||
{ rel: 'canonical', href: 'https://docs.subminer.moe/usage' },
|
// A sitemap here would advertise the archive tree we just excluded.
|
||||||
]);
|
expect(archiveConfig.sitemap).toBeUndefined();
|
||||||
|
} finally {
|
||||||
process.env.SUBMINER_DOCS_CHANNEL = previousChannel;
|
process.env.SUBMINER_DOCS_CHANNEL = previousChannel;
|
||||||
process.env.SUBMINER_DOCS_BASE = previousBase;
|
process.env.SUBMINER_DOCS_BASE = previousBase;
|
||||||
process.env.SUBMINER_DOCS_VERSION = previousVersion;
|
process.env.SUBMINER_DOCS_VERSION = previousVersion;
|
||||||
process.env.SUBMINER_DOCS_LATEST_STABLE = previousLatest;
|
process.env.SUBMINER_DOCS_LATEST_STABLE = previousLatest;
|
||||||
});
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
test('stable archive theme links stay on the selected version', async () => {
|
test('stable archive theme links stay on the selected version', async () => {
|
||||||
const previousCwd = process.cwd();
|
const previousCwd = process.cwd();
|
||||||
@@ -433,3 +442,22 @@ test('docs sitemap excludes duplicate README page from indexable URLs', async ()
|
|||||||
|
|
||||||
expect(transformedItems?.map((item) => item.url)).toEqual(['', 'usage']);
|
expect(transformedItems?.map((item) => item.url)).toEqual(['', 'usage']);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('docs sitemap dates every URL from the tracked checkout', async () => {
|
||||||
|
const previousRepoDir = process.env.SUBMINER_DOCS_REPO_DIR;
|
||||||
|
// Production builds render from an untracked snapshot, so the date has to come from
|
||||||
|
// the real checkout rather than VitePress's own srcDir git lookup.
|
||||||
|
process.env.SUBMINER_DOCS_REPO_DIR = docsSiteDir;
|
||||||
|
try {
|
||||||
|
const { default: sitemapConfig } = await import('./.vitepress/config?sitemap-lastmod');
|
||||||
|
|
||||||
|
const items = await sitemapConfig.sitemap?.transformItems?.([{ url: '' }, { url: 'usage' }]);
|
||||||
|
|
||||||
|
expect(items).toHaveLength(2);
|
||||||
|
for (const item of items ?? []) {
|
||||||
|
expect(item.lastmod).toMatch(/^\d{4}-\d{2}-\d{2}T/);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
process.env.SUBMINER_DOCS_REPO_DIR = previousRepoDir;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ When SubMiner parses the active subtitle source into a cue list, the sidebar bec
|
|||||||
- Clicking any cue seeks mpv to that timestamp.
|
- Clicking any cue seeks mpv to that timestamp.
|
||||||
- The sidebar stays synchronized with the overlay - media transitions and subtitle source changes update both simultaneously.
|
- The sidebar stays synchronized with the overlay - media transitions and subtitle source changes update both simultaneously.
|
||||||
|
|
||||||
|
For typeset ASS karaoke and animated signs, SubMiner collapses generated animation frames and repeated full-line color phases before they reach the sidebar. It recovers a clean complete line from a matching timed authoring comment or from full-line events surrounding generated fragments. Ordinary ASS comments, editor notes, alternate lines, repeated dialogue, and separately positioned signs remain distinct.
|
||||||
|
|
||||||
The sidebar only appears when a parsed cue list is available. External subtitle sources that SubMiner cannot parse (for example, embedded ASS tracks rendered directly by mpv) will not populate the sidebar.
|
The sidebar only appears when a parsed cue list is available. External subtitle sources that SubMiner cannot parse (for example, embedded ASS tracks rendered directly by mpv) will not populate the sidebar.
|
||||||
|
|
||||||
## Layout Modes
|
## Layout Modes
|
||||||
|
|||||||
@@ -70,18 +70,25 @@ interface SubtitleCue {
|
|||||||
startTime: number; // seconds
|
startTime: number; // seconds
|
||||||
endTime: number; // seconds
|
endTime: number; // seconds
|
||||||
text: string; // plain text, decoded from the source format
|
text: string; // plain text, decoded from the source format
|
||||||
|
source?: 'canonical-ass'; // recovered authored text for generated ASS animation
|
||||||
|
animationStartTime?: number; // full generated-frame envelope; entrance/exit frames
|
||||||
|
animationEndTime?: number; // run past the authored timing, live matching uses this
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**Supported formats:**
|
**Supported formats:**
|
||||||
|
|
||||||
- SRT/VTT: Regex-based parsing of timing lines + text content between timing blocks.
|
- SRT/VTT: Regex-based parsing of timing lines + text content between timing blocks.
|
||||||
- ASS: Parse `[Events]` section, extract `Dialogue:` lines, read the field order from the `Format:` row, and take everything after the Text field index as the text (Text can itself contain commas).
|
- ASS: Parse the `[Events]` section, read the field order from the `Format:` row, and extract timed `Dialogue:` lines. Timed `Comment:` lines are normally ignored, but can supply canonical authored text when they match a nearby generated animation from the same style and actor. Text can itself contain commas.
|
||||||
|
|
||||||
**ASS decoding.** The parser is where ASS text is decoded, once, via `assToPlainText()` in `src/core/services/ass-text.ts`. That decoder mirrors mpv's `ass_to_plaintext` so a cue read from a file reads identically to the same line arriving live on `sub-text`: `{...}` override blocks are markup, `\pN … \p0` vector drawing runs are dropped rather than shown as text, `\N`/`\n`/`\h` are the only escapes (`\{`, `\}` and `\\` are not), and an unclosed `{` is rendered verbatim. Every layer downstream — renderer, timing tracker, tokenizer, tokenization cache keys — receives plain text and uses `normalizePlainSubtitleText()` for whitespace only, so nothing decodes the same string twice and one authored line always maps to one cache key.
|
**ASS decoding.** The parser is where ASS text is decoded, once, via `assToPlainText()` in `src/core/services/ass-text.ts`. That decoder mirrors mpv's `ass_to_plaintext` so a cue read from a file reads identically to the same line arriving live on `sub-text`: `{...}` override blocks are markup, `\pN … \p0` vector drawing runs are dropped rather than shown as text, `\N`/`\n`/`\h` are the only escapes (`\{`, `\}` and `\\` are not), and an unclosed `{` is rendered verbatim. Every layer downstream — renderer, timing tracker, tokenizer, tokenization cache keys — receives plain text and uses `normalizePlainSubtitleText()` for whitespace only, so nothing decodes the same string twice and one authored line always maps to one cache key.
|
||||||
|
|
||||||
**Duplicate collapsing.** Typeset scripts emit one `Dialogue:` event per animation frame, plus layered copies of the same line. The parser collapses identical text over an identical span unconditionally, and collapses contiguous same-text runs of at least three events when the run looks like an animation. For ASS that means shared style and actor plus authoring evidence: a temporal tag (`\t`, `\move`, `\k`/`\kf`/`\ko`/`\K`, or anything wrapped in `\t(...)`), an animated `Effect` column (`Karaoke`, `Banner`, `Scroll`), or override values that change across the run. Static tags shared by every event (`\pos`, an identical `\clip`) are not evidence. SRT/VTT carry no such metadata, so there collapsing needs at least five contiguous events all under 0.1s — the frame timing left behind by ASS-to-SRT conversion. The parser keeps this authoring metadata (style, actor, layer, `Effect`, parsed override commands, source order) private; `parseSubtitleCues()` returns only `SubtitleCue`.
|
**Duplicate collapsing.** Typeset scripts emit one `Dialogue:` event per animation frame, plus layered copies of the same line. The parser collapses identical text over an identical span unconditionally, and collapses contiguous same-text runs of at least three events when the run looks like an animation. For ASS that means shared style and actor plus authoring evidence: a temporal tag (`\t`, `\move`, `\k`/`\kf`/`\ko`/`\K`, or anything wrapped in `\t(...)`), an animated `Effect` column (`Karaoke`, `Banner`, `Scroll`), or override values that change across the run. Static tags shared by every event (`\pos`, an identical `\clip`) are not evidence. SRT/VTT carry no such metadata, so there collapsing needs at least five contiguous events all under 0.1s — the frame timing left behind by ASS-to-SRT conversion. The parser keeps this authoring metadata (style, actor, layer, `Effect`, parsed override commands, source order) private; `parseSubtitleCues()` returns only `SubtitleCue`.
|
||||||
|
|
||||||
|
ASS scripts can also redraw one complete lyric for two or more long color/highlight phases. Those flush-timed phases collapse separately from short animation frames when they share text, style, actor, and layer and carry direct animation evidence, such as temporal tags or changing non-spatial overrides. Spatial command changes do not prove a phase, so separately positioned signs remain distinct.
|
||||||
|
|
||||||
|
**Canonical animation recovery.** Some ASS producers keep the readable lyric or sign as a timed `Comment:` and generate hundreds of `Dialogue:` frames containing repeated glyphs or changing clip regions. Others retain the complete line as brief `Dialogue:` events around the generated fragments. A complete event is promoted only when nearby dialogue from the same style and actor forms a proven animation cluster and reconstructs its entire text in source order. The generated frames are then replaced by one cue marked `source: 'canonical-ass'`. This source marker lets the live primary-subtitle path prefer the clean authored text and timing for display, sidebar history, immersion recording, and mining, while unmatched editor notes and alternative translations remain ignored.
|
||||||
|
|
||||||
#### Prefetch Service Lifecycle
|
#### Prefetch Service Lifecycle
|
||||||
|
|
||||||
1. **Activation trigger:** When a subtitle track is activated (or changes), check if it's external via MPV's `track-list` property. If `external === true`, read the file via `external-filename` using the existing `loadSubtitleSourceText` infrastructure.
|
1. **Activation trigger:** When a subtitle track is activated (or changes), check if it's external via MPV's `track-list` property. If `external === true`, read the file via `external-filename` using the existing `loadSubtitleSourceText` infrastructure.
|
||||||
|
|||||||
@@ -23,7 +23,9 @@ Trend charts now consume one chart-oriented backend payload from `/api/stats/tre
|
|||||||
- lookup rate trends
|
- lookup rate trends
|
||||||
- watch-time by day-of-week/hour
|
- watch-time by day-of-week/hour
|
||||||
- vocabulary-backed:
|
- vocabulary-backed:
|
||||||
- new-words trend
|
- new-words trend reads permanent daily lexical rollups
|
||||||
|
- rollup rows count only vocabulary-visible tokens and normalize mixed legacy timestamp units
|
||||||
|
- a persisted rollup version invalidates stale materializations and triggers an atomic background rebuild
|
||||||
|
|
||||||
## Metric Semantics
|
## Metric Semantics
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
# Subtitle Overlay Priming
|
# Subtitle Overlay Priming
|
||||||
|
|
||||||
Status: active
|
Status: active
|
||||||
Last verified: 2026-08-04
|
Last verified: 2026-08-18
|
||||||
Owner: Kyle Yasuda
|
Owner: Kyle Yasuda
|
||||||
Read when: debugging subtitle state or blank Linux/X11 overlay windows when the visible overlay is shown or recreated
|
Read when: debugging subtitle state or blank Linux/X11 overlay windows when the visible overlay is shown or recreated
|
||||||
|
|
||||||
@@ -69,6 +69,11 @@ coming and prefetching would otherwise idle for the rest of the cue.
|
|||||||
|
|
||||||
## Live Cue Delivery
|
## Live Cue Delivery
|
||||||
|
|
||||||
|
- Primary live text first resolves recovered canonical ASS animations. Otherwise, when
|
||||||
|
every live mpv line matches an active parsed cue, it uses the parsed cue text so exact
|
||||||
|
full-span style layers appear once instead of repeating for fill, border, blur, and
|
||||||
|
shadow events. Any unmatched live line keeps the complete live stack, preserving
|
||||||
|
dialogue or signs that overlap a lyric.
|
||||||
- A tokenization cache miss emits the plain cue synchronously. Tokenization remains serialized so
|
- A tokenization cache miss emits the plain cue synchronously. Tokenization remains serialized so
|
||||||
live work does not contend for Yomitan state.
|
live work does not contend for Yomitan state.
|
||||||
- If a newer cue arrives while an older line is still tokenizing, the newer plain cue or empty
|
- If a newer cue arrives while an older line is still tokenizing, the newer plain cue or empty
|
||||||
@@ -77,6 +82,25 @@ coming and prefetching would otherwise idle for the rest of the cue.
|
|||||||
- The current cue upgrades in place when its tokens and annotations are ready. This can reflow text
|
- The current cue upgrades in place when its tokens and annotations are ready. This can reflow text
|
||||||
or character images, but cue visibility does not wait for that work.
|
or character images, but cue visibility does not wait for that work.
|
||||||
|
|
||||||
|
## Secondary Subtitle Flow
|
||||||
|
|
||||||
|
- `secondary-sub-text` remains the immediate fallback, so unreadable and remote subtitle sources
|
||||||
|
still appear without waiting for file resolution.
|
||||||
|
- `secondary-subtitle-track.ts` resolves `secondary-sid` against mpv's track list. External tracks
|
||||||
|
are read directly; supported embedded text tracks are extracted through the same ffmpeg-backed
|
||||||
|
source resolver used by primary subtitle prefetching.
|
||||||
|
- The selected source is parsed with `parseSubtitleCues()`, including metadata-aware ASS duplicate
|
||||||
|
and animation collapse. Playback `time-pos` selects the active parsed cue after applying
|
||||||
|
`secondary-sub-delay`.
|
||||||
|
- The resolved text is stored in `mpvClient.currentSecondarySubText` before it is broadcast. The
|
||||||
|
overlay, mining, timing tracker, and immersion statistics therefore consume the same secondary
|
||||||
|
text when a readable source is available.
|
||||||
|
- Media and `secondary-sid` changes clear the previous parsed state before refreshing the source;
|
||||||
|
track-list changes refresh without discarding an unchanged source. Observed
|
||||||
|
`secondary-sub-delay` changes retime the active parsed cue without rereading the file. If loading,
|
||||||
|
extraction, or parsing fails, the controller returns to live mpv text and the renderer's
|
||||||
|
conservative short stack heuristic remains the final display fallback.
|
||||||
|
|
||||||
## Emitted State
|
## Emitted State
|
||||||
|
|
||||||
- `emitSubtitle(payload)` maps to `emitSubtitlePayload(payload)`. Overlay windows and annotation
|
- `emitSubtitle(payload)` maps to `emitSubtitlePayload(payload)`. Overlay windows and annotation
|
||||||
@@ -84,8 +108,8 @@ coming and prefetching would otherwise idle for the rest of the cue.
|
|||||||
- The basic subtitle websocket receives the immediate plain cue only. Because its serialized
|
- The basic subtitle websocket receives the immediate plain cue only. Because its serialized
|
||||||
payload discards annotations, the later upgrade would be an identical duplicate and is skipped
|
payload discards annotations, the later upgrade would be an identical duplicate and is skipped
|
||||||
when text and cue timing match.
|
when text and cue timing match.
|
||||||
- Secondary priming reads mpv `secondary-sub-text`, stores it in
|
- Secondary priming reads mpv `secondary-sub-text` and routes it through the secondary track
|
||||||
`mpvClient.currentSecondarySubText`, and broadcasts `secondary-subtitle:set` to overlay windows.
|
controller. A parsed active cue replaces the live text when the selected source is readable.
|
||||||
- If secondary `requestProperty` fails, the primary flow stays complete and only a debug line is
|
- If secondary `requestProperty` fails, the primary flow stays complete and only a debug line is
|
||||||
written.
|
written.
|
||||||
|
|
||||||
@@ -129,7 +153,11 @@ coming and prefetching would otherwise idle for the rest of the cue.
|
|||||||
path, empty or stale bounding shapes produced invisible or clipped subtitles even though the
|
path, empty or stale bounding shapes produced invisible or clipped subtitles even though the
|
||||||
overlay window remained mapped above mpv.
|
overlay window remained mapped above mpv.
|
||||||
- Pointer pass-through should continue to use `setIgnoreMouseEvents(true, { forward: true })` and
|
- Pointer pass-through should continue to use `setIgnoreMouseEvents(true, { forward: true })` and
|
||||||
the Linux cursor-poll fallback, not bounding-shape clipping.
|
the Linux cursor-poll fallback, not bounding-shape clipping. Note that on Windows click-through
|
||||||
|
must go through `applyOverlayClickThrough()` (`src/core/services/overlay-click-through.ts`),
|
||||||
|
which omits `forward: true` there: Electron implements forwarding with a global low-level mouse
|
||||||
|
hook that lags mouse input system-wide whenever the main thread stalls; the Windows cursor poll
|
||||||
|
handles overlay wake-up instead.
|
||||||
- Visible-overlay show/reset marks Linux pointer passthrough state dirty even when the logical
|
- Visible-overlay show/reset marks Linux pointer passthrough state dirty even when the logical
|
||||||
interaction state is already inactive. The next cursor-poll tick must still reapply
|
interaction state is already inactive. The next cursor-poll tick must still reapply
|
||||||
`setIgnoreMouseEvents(true, { forward: true })`; otherwise a newly shown Electron overlay can keep
|
`setIgnoreMouseEvents(true, { forward: true })`; otherwise a newly shown Electron overlay can keep
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import {
|
|||||||
type HistorySeriesEntry,
|
type HistorySeriesEntry,
|
||||||
} from '../history.js';
|
} from '../history.js';
|
||||||
import type { Args } from '../types.js';
|
import type { Args } from '../types.js';
|
||||||
|
import { ensureLinuxRuntimePluginAvailable } from '../runtime-plugin-preflight.js';
|
||||||
import type { LauncherCommandContext } from './context.js';
|
import type { LauncherCommandContext } from './context.js';
|
||||||
|
|
||||||
export type HistorySessionAction = 'previous' | 'replay' | 'next' | 'browse' | 'quit';
|
export type HistorySessionAction = 'previous' | 'replay' | 'next' | 'browse' | 'quit';
|
||||||
@@ -333,6 +334,13 @@ export async function runHistoryCommand(
|
|||||||
const { args, scriptPath } = context;
|
const { args, scriptPath } = context;
|
||||||
|
|
||||||
checkPickerDependencies(args);
|
checkPickerDependencies(args);
|
||||||
|
if (args.useRofi) {
|
||||||
|
await ensureLinuxRuntimePluginAvailable({
|
||||||
|
appPath: context.appPath ?? undefined,
|
||||||
|
scriptPath,
|
||||||
|
logLevel: args.logLevel,
|
||||||
|
});
|
||||||
|
}
|
||||||
const themePath = args.useRofi ? findRofiTheme(scriptPath) : null;
|
const themePath = args.useRofi ? findRofiTheme(scriptPath) : null;
|
||||||
|
|
||||||
const dbPath = resolveImmersionDbPath();
|
const dbPath = resolveImmersionDbPath();
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { fail } from '../log.js';
|
|||||||
import { runAppCommandWithInherit } from '../mpv.js';
|
import { runAppCommandWithInherit } from '../mpv.js';
|
||||||
import { commandExists } from '../util.js';
|
import { commandExists } from '../util.js';
|
||||||
import { runJellyfinPlayMenu } from '../jellyfin.js';
|
import { runJellyfinPlayMenu } from '../jellyfin.js';
|
||||||
|
import { ensureLinuxRuntimePluginAvailable } from '../runtime-plugin-preflight.js';
|
||||||
import { shouldForwardLogLevel } from '../types.js';
|
import { shouldForwardLogLevel } from '../types.js';
|
||||||
import type { LauncherCommandContext } from './context.js';
|
import type { LauncherCommandContext } from './context.js';
|
||||||
|
|
||||||
@@ -64,6 +65,13 @@ export async function runJellyfinCommand(context: LauncherCommandContext): Promi
|
|||||||
if (args.useRofi && !commandExists('rofi')) {
|
if (args.useRofi && !commandExists('rofi')) {
|
||||||
fail('rofi not found. Install rofi or omit -R for fzf.');
|
fail('rofi not found. Install rofi or omit -R for fzf.');
|
||||||
}
|
}
|
||||||
|
if (args.useRofi) {
|
||||||
|
await ensureLinuxRuntimePluginAvailable({
|
||||||
|
appPath,
|
||||||
|
scriptPath,
|
||||||
|
logLevel: args.logLevel,
|
||||||
|
});
|
||||||
|
}
|
||||||
await runJellyfinPlayMenu(appPath, args, scriptPath, mpvSocketPath);
|
await runJellyfinPlayMenu(appPath, args, scriptPath, mpvSocketPath);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -496,3 +496,39 @@ test('playback command ensures Linux runtime plugin before mpv launch', async ()
|
|||||||
|
|
||||||
assert.deepEqual(calls, ['plugin', 'startMpv']);
|
assert.deepEqual(calls, ['plugin', 'startMpv']);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('rofi playback repairs support assets before opening the picker', async () => {
|
||||||
|
const context = createContext();
|
||||||
|
context.args = {
|
||||||
|
...context.args,
|
||||||
|
target: '',
|
||||||
|
targetKind: '',
|
||||||
|
useRofi: true,
|
||||||
|
};
|
||||||
|
const calls: string[] = [];
|
||||||
|
|
||||||
|
await runPlaybackCommandWithDeps(context, {
|
||||||
|
ensurePlaybackSetupReady: async () => {},
|
||||||
|
ensureRuntimePluginReady: async () => {
|
||||||
|
calls.push('assets');
|
||||||
|
},
|
||||||
|
chooseTarget: async () => {
|
||||||
|
calls.push('picker');
|
||||||
|
return { target: '/tmp/movie.mkv', kind: 'file' };
|
||||||
|
},
|
||||||
|
checkPickerDependencies: () => {},
|
||||||
|
checkDependencies: () => {},
|
||||||
|
registerCleanup: () => {},
|
||||||
|
startMpv: async () => {
|
||||||
|
calls.push('startMpv');
|
||||||
|
},
|
||||||
|
waitForUnixSocketReady: async () => true,
|
||||||
|
startOverlay: async () => {},
|
||||||
|
launchAppCommandDetached: () => {},
|
||||||
|
log: () => {},
|
||||||
|
cleanupPlaybackSession: async () => {},
|
||||||
|
getMpvProc: () => null,
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.deepEqual(calls, ['assets', 'picker', 'startMpv']);
|
||||||
|
});
|
||||||
|
|||||||
@@ -157,6 +157,7 @@ export async function runPlaybackCommand(context: LauncherCommandContext): Promi
|
|||||||
});
|
});
|
||||||
},
|
},
|
||||||
chooseTarget,
|
chooseTarget,
|
||||||
|
checkPickerDependencies,
|
||||||
checkDependencies,
|
checkDependencies,
|
||||||
registerCleanup,
|
registerCleanup,
|
||||||
startMpv,
|
startMpv,
|
||||||
@@ -177,6 +178,7 @@ type PlaybackCommandDeps = {
|
|||||||
args: Args,
|
args: Args,
|
||||||
scriptPath: string,
|
scriptPath: string,
|
||||||
) => Promise<{ target: string; kind: 'file' | 'url' } | null>;
|
) => Promise<{ target: string; kind: 'file' | 'url' } | null>;
|
||||||
|
checkPickerDependencies?: (args: Args) => void;
|
||||||
checkDependencies: (args: Args) => void;
|
checkDependencies: (args: Args) => void;
|
||||||
registerCleanup: (context: LauncherCommandContext) => void;
|
registerCleanup: (context: LauncherCommandContext) => void;
|
||||||
startMpv: typeof startMpv;
|
startMpv: typeof startMpv;
|
||||||
@@ -201,7 +203,18 @@ export async function runPlaybackCommandWithDeps(
|
|||||||
await deps.ensurePlaybackSetupReady(context);
|
await deps.ensurePlaybackSetupReady(context);
|
||||||
|
|
||||||
if (!args.target) {
|
if (!args.target) {
|
||||||
checkPickerDependencies(args);
|
(deps.checkPickerDependencies ?? checkPickerDependencies)(args);
|
||||||
|
}
|
||||||
|
|
||||||
|
let runtimeAssetsReady = false;
|
||||||
|
const ensureRuntimeAssetsReady = async (): Promise<void> => {
|
||||||
|
if (runtimeAssetsReady) return;
|
||||||
|
await deps.ensureRuntimePluginReady(context);
|
||||||
|
runtimeAssetsReady = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!args.target && args.useRofi) {
|
||||||
|
await ensureRuntimeAssetsReady();
|
||||||
}
|
}
|
||||||
|
|
||||||
const targetChoice = await deps.chooseTarget(args, scriptPath);
|
const targetChoice = await deps.chooseTarget(args, scriptPath);
|
||||||
@@ -266,7 +279,7 @@ export async function runPlaybackCommandWithDeps(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
await deps.ensureRuntimePluginReady(context);
|
await ensureRuntimeAssetsReady();
|
||||||
|
|
||||||
await deps.startMpv(
|
await deps.startMpv(
|
||||||
selectedTarget.target,
|
selectedTarget.target,
|
||||||
|
|||||||
@@ -36,6 +36,11 @@ test('runUpdateCommand updates directly on Linux without launching Electron', as
|
|||||||
launcher: { status: 'updated' },
|
launcher: { status: 'updated' },
|
||||||
supportAssets: [
|
supportAssets: [
|
||||||
{ status: 'updated', component: 'theme', message: 'Installed theme.' },
|
{ status: 'updated', component: 'theme', message: 'Installed theme.' },
|
||||||
|
{
|
||||||
|
status: 'updated',
|
||||||
|
component: 'thumbnailer',
|
||||||
|
message: 'Installed rofi thumbnailer.',
|
||||||
|
},
|
||||||
{ status: 'skipped', component: 'plugin', message: 'Plugin already up to date.' },
|
{ status: 'skipped', component: 'plugin', message: 'Plugin already up to date.' },
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
@@ -52,6 +57,7 @@ test('runUpdateCommand updates directly on Linux without launching Electron', as
|
|||||||
'info:AppImage update: updated',
|
'info:AppImage update: updated',
|
||||||
'info:Launcher update: updated',
|
'info:Launcher update: updated',
|
||||||
'info:Support assets (theme) update: updated - Installed theme.',
|
'info:Support assets (theme) update: updated - Installed theme.',
|
||||||
|
'info:Support assets (thumbnailer) update: updated - Installed rofi thumbnailer.',
|
||||||
'info:Support assets (plugin) update: skipped - Plugin already up to date.',
|
'info:Support assets (plugin) update: skipped - Plugin already up to date.',
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -21,7 +21,10 @@ import {
|
|||||||
parseSha256Sums,
|
parseSha256Sums,
|
||||||
type FetchLike,
|
type FetchLike,
|
||||||
} from '../../src/main/runtime/update/release-assets.js';
|
} from '../../src/main/runtime/update/release-assets.js';
|
||||||
import { updateSupportAssetsFromRelease } from '../../src/main/runtime/update/support-assets.js';
|
import {
|
||||||
|
updateSupportAssetsFromRelease,
|
||||||
|
type SupportAssetsUpdateResult,
|
||||||
|
} from '../../src/main/runtime/update/support-assets.js';
|
||||||
|
|
||||||
type UpdateCommandResponse = {
|
type UpdateCommandResponse = {
|
||||||
ok: boolean;
|
ok: boolean;
|
||||||
@@ -36,15 +39,14 @@ type DirectReleaseUpdateRequest = {
|
|||||||
channel: UpdateChannel;
|
channel: UpdateChannel;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type DirectSupportAssetsUpdateResult = Omit<SupportAssetsUpdateResult, 'status'> & {
|
||||||
|
status: string;
|
||||||
|
};
|
||||||
|
|
||||||
type DirectReleaseUpdateResult = {
|
type DirectReleaseUpdateResult = {
|
||||||
appImage: { status: string; command?: string; message?: string };
|
appImage: { status: string; command?: string; message?: string };
|
||||||
launcher: { status: string; command?: string; message?: string };
|
launcher: { status: string; command?: string; message?: string };
|
||||||
supportAssets: Array<{
|
supportAssets: DirectSupportAssetsUpdateResult[];
|
||||||
status: string;
|
|
||||||
component?: 'theme' | 'plugin';
|
|
||||||
command?: string;
|
|
||||||
message?: string;
|
|
||||||
}>;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
type UpdateCommandDeps = {
|
type UpdateCommandDeps = {
|
||||||
@@ -129,12 +131,7 @@ function readUpdateChannel(root: Record<string, unknown> | null): UpdateChannel
|
|||||||
|
|
||||||
function logUpdateResult(
|
function logUpdateResult(
|
||||||
label: string,
|
label: string,
|
||||||
result: {
|
result: DirectSupportAssetsUpdateResult,
|
||||||
status: string;
|
|
||||||
component?: 'theme' | 'plugin';
|
|
||||||
command?: string;
|
|
||||||
message?: string;
|
|
||||||
},
|
|
||||||
configuredLogLevel: NonNullable<LauncherCommandContext['args']['logLevel']>,
|
configuredLogLevel: NonNullable<LauncherCommandContext['args']['logLevel']>,
|
||||||
deps: Pick<UpdateCommandDeps, 'log'>,
|
deps: Pick<UpdateCommandDeps, 'log'>,
|
||||||
): void {
|
): void {
|
||||||
|
|||||||
@@ -73,20 +73,21 @@ function makeTestEnv(homeDir: string, xdgConfigHome: string): NodeJS.ProcessEnv
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// On Linux the playback path runs `ensureLinuxRuntimePluginAvailable`, which —
|
// On Linux the playback path runs `ensureLinuxRuntimePluginAvailable`, which
|
||||||
// when the runtime plugin/theme are missing — spawns the app with
|
// spawns the app with `--ensure-linux-runtime-plugin-assets` when managed
|
||||||
// `--ensure-linux-runtime-plugin-assets` and polls up to 30s
|
// support assets are missing and polls up to 30s
|
||||||
// (RESPONSE_TIMEOUT_MS) for an install response. A fake app that just exits
|
// (RESPONSE_TIMEOUT_MS) for an install response. A fake app that just exits
|
||||||
// never writes that response, so the launcher hangs and the test times out on
|
// never writes that response, so the launcher hangs and the test times out on
|
||||||
// Linux CI (the preflight is a no-op on macOS/Windows). This shell prelude makes
|
// Linux CI (the preflight is a no-op on macOS/Windows). This shell prelude makes
|
||||||
// the fake app install the managed plugin/theme and write the response, matching
|
// the fake app install the managed support assets and write the response, matching
|
||||||
// launcher/smoke.e2e.test.ts. Prepend it to each fake app that reaches playback.
|
// launcher/smoke.e2e.test.ts. Prepend it to each fake app that reaches playback.
|
||||||
const RUNTIME_PLUGIN_PREFLIGHT_SH = `if [ "$1" = "--ensure-linux-runtime-plugin-assets" ]; then
|
const RUNTIME_PLUGIN_PREFLIGHT_SH = `if [ "$1" = "--ensure-linux-runtime-plugin-assets" ]; then
|
||||||
data="\${XDG_DATA_HOME:-$HOME/.local/share}/SubMiner"
|
data="\${XDG_DATA_HOME:-$HOME/.local/share}/SubMiner"
|
||||||
mkdir -p "$data/plugin/subminer" "$data/themes"
|
mkdir -p "$data/plugin/subminer" "$data/themes" "$data/thumbnailers"
|
||||||
printf -- '-- test plugin\\n' > "$data/plugin/subminer/main.lua"
|
printf -- '-- test plugin\\n' > "$data/plugin/subminer/main.lua"
|
||||||
printf 'test=true\\n' > "$data/plugin/subminer.conf"
|
printf 'test=true\\n' > "$data/plugin/subminer.conf"
|
||||||
printf '/* test theme */\\n' > "$data/themes/subminer.rasi"
|
printf '/* test theme */\\n' > "$data/themes/subminer.rasi"
|
||||||
|
printf '[Thumbnailer Entry]\\n' > "$data/thumbnailers/subminer-ffmpegthumbnailer.thumbnailer"
|
||||||
if [ "$2" = "--ensure-linux-runtime-plugin-assets-response-path" ] && [ -n "$3" ]; then
|
if [ "$2" = "--ensure-linux-runtime-plugin-assets-response-path" ] && [ -n "$3" ]; then
|
||||||
mkdir -p "$(dirname "$3")"
|
mkdir -p "$(dirname "$3")"
|
||||||
printf '{"ok":true,"status":"installed","path":"%s"}' "$data/plugin/subminer/main.lua" > "$3"
|
printf '{"ok":true,"status":"installed","path":"%s"}' "$data/plugin/subminer/main.lua" > "$3"
|
||||||
|
|||||||
@@ -3,7 +3,12 @@ import assert from 'node:assert/strict';
|
|||||||
import fs from 'node:fs';
|
import fs from 'node:fs';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import os from 'node:os';
|
import os from 'node:os';
|
||||||
import { findRofiTheme, formatRofiPrompt } from './picker';
|
import {
|
||||||
|
findRofiTheme,
|
||||||
|
findRofiThumbnailerDataRoot,
|
||||||
|
formatRofiPrompt,
|
||||||
|
prependXdgDataDir,
|
||||||
|
} from './picker';
|
||||||
|
|
||||||
// ── formatRofiPrompt: spacing between prompt and input field ──────────────────
|
// ── formatRofiPrompt: spacing between prompt and input field ──────────────────
|
||||||
|
|
||||||
@@ -23,6 +28,7 @@ test('formatRofiPrompt leaves an empty prompt empty', () => {
|
|||||||
// ── findRofiTheme: Linux packaged path discovery ──────────────────────────────
|
// ── findRofiTheme: Linux packaged path discovery ──────────────────────────────
|
||||||
|
|
||||||
const ROFI_THEME_FILE = 'subminer.rasi';
|
const ROFI_THEME_FILE = 'subminer.rasi';
|
||||||
|
const ROFI_THUMBNAILER_FILE = 'subminer-ffmpegthumbnailer.thumbnailer';
|
||||||
|
|
||||||
function makeFile(filePath: string): void {
|
function makeFile(filePath: string): void {
|
||||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||||
@@ -121,3 +127,42 @@ test('findRofiTheme resolves ~/.local/share/SubMiner/themes/subminer.rasi when X
|
|||||||
fs.rmSync(baseDir, { recursive: true, force: true });
|
fs.rmSync(baseDir, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('findRofiThumbnailerDataRoot resolves the managed XDG data root', () => {
|
||||||
|
const xdgDataHome = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-test-xdg-'));
|
||||||
|
const originalXdgDataHome = process.env.XDG_DATA_HOME;
|
||||||
|
try {
|
||||||
|
process.env.XDG_DATA_HOME = xdgDataHome;
|
||||||
|
const dataRoot = path.join(xdgDataHome, 'SubMiner');
|
||||||
|
makeFile(path.join(dataRoot, 'thumbnailers', ROFI_THUMBNAILER_FILE));
|
||||||
|
|
||||||
|
const result = withPlatform('linux', () => findRofiThumbnailerDataRoot('/usr/bin/subminer'));
|
||||||
|
assert.equal(result, dataRoot);
|
||||||
|
} finally {
|
||||||
|
if (originalXdgDataHome === undefined) {
|
||||||
|
delete process.env.XDG_DATA_HOME;
|
||||||
|
} else {
|
||||||
|
process.env.XDG_DATA_HOME = originalXdgDataHome;
|
||||||
|
}
|
||||||
|
fs.rmSync(xdgDataHome, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('findRofiThumbnailerDataRoot is Linux-only', () => {
|
||||||
|
assert.equal(
|
||||||
|
withPlatform('darwin', () => findRofiThumbnailerDataRoot('/usr/bin/subminer')),
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('prependXdgDataDir preserves existing roots and avoids duplicates', () => {
|
||||||
|
const root = '/tmp/subminer-data';
|
||||||
|
assert.equal(
|
||||||
|
prependXdgDataDir(root, `/opt/share${path.delimiter}${root}${path.delimiter}/usr/share`),
|
||||||
|
`${root}${path.delimiter}/opt/share${path.delimiter}/usr/share`,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
prependXdgDataDir(root),
|
||||||
|
`${root}${path.delimiter}/usr/local/share${path.delimiter}/usr/share`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|||||||
@@ -159,6 +159,9 @@ interface RofiIconEntry {
|
|||||||
iconPath?: string;
|
iconPath?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ROFI_THUMBNAILER_FILE = 'subminer-ffmpegthumbnailer.thumbnailer';
|
||||||
|
const DEFAULT_XDG_DATA_DIRS = ['/usr/local/share', '/usr/share'];
|
||||||
|
|
||||||
function showRofiIconMenu(
|
function showRofiIconMenu(
|
||||||
entries: RofiIconEntry[],
|
entries: RofiIconEntry[],
|
||||||
prompt: string,
|
prompt: string,
|
||||||
@@ -389,6 +392,47 @@ export function findRofiTheme(scriptPath: string): string | null {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function findRofiThumbnailerDataRoot(scriptPath: string): string | null {
|
||||||
|
if (process.platform !== 'linux') return null;
|
||||||
|
|
||||||
|
const scriptDir = path.dirname(realpathMaybe(scriptPath));
|
||||||
|
const xdgDataHome = process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local/share');
|
||||||
|
const roots = [
|
||||||
|
path.join(xdgDataHome, 'SubMiner'),
|
||||||
|
path.posix.join('/usr/local/share/SubMiner'),
|
||||||
|
path.posix.join('/usr/share/SubMiner'),
|
||||||
|
path.join(scriptDir, 'assets'),
|
||||||
|
path.join(scriptDir, '..', 'assets'),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const root of roots) {
|
||||||
|
if (fs.existsSync(path.join(root, 'thumbnailers', ROFI_THUMBNAILER_FILE))) {
|
||||||
|
return root;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function prependXdgDataDir(dataRoot: string, currentValue?: string): string {
|
||||||
|
const currentDirs = currentValue
|
||||||
|
? currentValue.split(path.delimiter).filter(Boolean)
|
||||||
|
: DEFAULT_XDG_DATA_DIRS;
|
||||||
|
return [dataRoot, ...currentDirs.filter((candidate) => candidate !== dataRoot)].join(
|
||||||
|
path.delimiter,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildRofiThumbnailEnvironment(scriptPath: string): NodeJS.ProcessEnv {
|
||||||
|
if (!commandExists('ffmpegthumbnailer')) return process.env;
|
||||||
|
const dataRoot = findRofiThumbnailerDataRoot(scriptPath);
|
||||||
|
if (!dataRoot) return process.env;
|
||||||
|
return {
|
||||||
|
...process.env,
|
||||||
|
XDG_DATA_DIRS: prependXdgDataDir(dataRoot, process.env.XDG_DATA_DIRS),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function showRofiMenu(
|
export function showRofiMenu(
|
||||||
videos: string[],
|
videos: string[],
|
||||||
dir: string,
|
dir: string,
|
||||||
@@ -420,6 +464,7 @@ export function showRofiMenu(
|
|||||||
const result = spawnSync('rofi', args, {
|
const result = spawnSync('rofi', args, {
|
||||||
input: buildRofiMenu(videos, dir, recursive),
|
input: buildRofiMenu(videos, dir, recursive),
|
||||||
encoding: 'utf8',
|
encoding: 'utf8',
|
||||||
|
env: buildRofiThumbnailEnvironment(scriptPath),
|
||||||
stdio: ['pipe', 'pipe', 'ignore'],
|
stdio: ['pipe', 'pipe', 'ignore'],
|
||||||
});
|
});
|
||||||
if (result.error) {
|
if (result.error) {
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
import fs from 'node:fs';
|
import fs from 'node:fs';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
import {
|
import {
|
||||||
ensureLinuxRuntimePluginAvailable,
|
ensureLinuxRuntimePluginAvailable,
|
||||||
installManagedPluginAssetsViaApp,
|
installManagedPluginAssetsViaApp,
|
||||||
@@ -31,7 +33,7 @@ test('ensureLinuxRuntimePluginAvailable is a no-op on non-Linux platforms', asyn
|
|||||||
assert.deepEqual(calls, []);
|
assert.deepEqual(calls, []);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('ensureLinuxRuntimePluginAvailable skips install when installed global plugin and managed theme exist', async () => {
|
test('ensureLinuxRuntimePluginAvailable skips install when plugin, theme, and thumbnailer exist', async () => {
|
||||||
const calls: string[] = [];
|
const calls: string[] = [];
|
||||||
|
|
||||||
await ensureLinuxRuntimePluginAvailable({
|
await ensureLinuxRuntimePluginAvailable({
|
||||||
@@ -52,13 +54,17 @@ test('ensureLinuxRuntimePluginAvailable skips install when installed global plug
|
|||||||
calls.push('theme');
|
calls.push('theme');
|
||||||
return true;
|
return true;
|
||||||
},
|
},
|
||||||
|
isManagedThumbnailerAvailable: () => {
|
||||||
|
calls.push('thumbnailer');
|
||||||
|
return true;
|
||||||
|
},
|
||||||
log: () => {},
|
log: () => {},
|
||||||
});
|
});
|
||||||
|
|
||||||
assert.deepEqual(calls, ['detect', 'theme']);
|
assert.deepEqual(calls, ['detect', 'theme', 'thumbnailer']);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('ensureLinuxRuntimePluginAvailable skips install when managed runtime path and theme already resolve', async () => {
|
test('ensureLinuxRuntimePluginAvailable skips install when all managed assets resolve', async () => {
|
||||||
const calls: string[] = [];
|
const calls: string[] = [];
|
||||||
|
|
||||||
await ensureLinuxRuntimePluginAvailable({
|
await ensureLinuxRuntimePluginAvailable({
|
||||||
@@ -80,14 +86,19 @@ test('ensureLinuxRuntimePluginAvailable skips install when managed runtime path
|
|||||||
calls.push('theme');
|
calls.push('theme');
|
||||||
return true;
|
return true;
|
||||||
},
|
},
|
||||||
|
isManagedThumbnailerAvailable: () => {
|
||||||
|
calls.push('thumbnailer');
|
||||||
|
return true;
|
||||||
|
},
|
||||||
log: () => {},
|
log: () => {},
|
||||||
});
|
});
|
||||||
|
|
||||||
assert.deepEqual(calls, ['detect', 'resolve', 'theme']);
|
assert.deepEqual(calls, ['detect', 'resolve', 'theme', 'thumbnailer']);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('ensureLinuxRuntimePluginAvailable installs managed assets when rofi theme is missing', async () => {
|
test('ensureLinuxRuntimePluginAvailable installs managed assets when rofi theme is missing', async () => {
|
||||||
const calls: string[] = [];
|
const calls: string[] = [];
|
||||||
|
let themeAvailable = false;
|
||||||
|
|
||||||
await ensureLinuxRuntimePluginAvailable({
|
await ensureLinuxRuntimePluginAvailable({
|
||||||
platform: 'linux',
|
platform: 'linux',
|
||||||
@@ -102,10 +113,15 @@ test('ensureLinuxRuntimePluginAvailable installs managed assets when rofi theme
|
|||||||
},
|
},
|
||||||
isManagedThemeAvailable: () => {
|
isManagedThemeAvailable: () => {
|
||||||
calls.push('theme');
|
calls.push('theme');
|
||||||
return false;
|
return themeAvailable;
|
||||||
|
},
|
||||||
|
isManagedThumbnailerAvailable: () => {
|
||||||
|
calls.push('thumbnailer');
|
||||||
|
return true;
|
||||||
},
|
},
|
||||||
installManagedPluginAssets: async () => {
|
installManagedPluginAssets: async () => {
|
||||||
calls.push('install');
|
calls.push('install');
|
||||||
|
themeAvailable = true;
|
||||||
return { ok: true, status: 'installed', path: '/tmp/plugin/main.lua' };
|
return { ok: true, status: 'installed', path: '/tmp/plugin/main.lua' };
|
||||||
},
|
},
|
||||||
log: (level, _configured, message) => {
|
log: (level, _configured, message) => {
|
||||||
@@ -117,13 +133,68 @@ test('ensureLinuxRuntimePluginAvailable installs managed assets when rofi theme
|
|||||||
'detect',
|
'detect',
|
||||||
'resolve',
|
'resolve',
|
||||||
'theme',
|
'theme',
|
||||||
'info:Linux runtime support assets missing; installing managed plugin/theme assets.',
|
'info:Linux runtime support assets missing; installing managed plugin/theme/thumbnailer assets.',
|
||||||
'install',
|
'install',
|
||||||
'info:Managed Linux runtime support assets installed: plugin=/tmp/plugin/main.lua theme=/tmp/xdg-data/SubMiner/themes/subminer.rasi',
|
'info:Managed Linux runtime support assets installed: plugin=/tmp/plugin/main.lua theme=/tmp/xdg-data/SubMiner/themes/subminer.rasi thumbnailer=/tmp/xdg-data/SubMiner/thumbnailers/subminer-ffmpegthumbnailer.thumbnailer',
|
||||||
'resolve',
|
'resolve',
|
||||||
|
'theme',
|
||||||
|
'thumbnailer',
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('ensureLinuxRuntimePluginAvailable installs managed assets when thumbnailer is missing', async () => {
|
||||||
|
const calls: string[] = [];
|
||||||
|
let thumbnailerAvailable = false;
|
||||||
|
|
||||||
|
await ensureLinuxRuntimePluginAvailable({
|
||||||
|
platform: 'linux',
|
||||||
|
xdgDataHome: '/tmp/xdg-data',
|
||||||
|
detectInstalledPlugin: () => true,
|
||||||
|
resolveRuntimePluginPath: () => '/tmp/plugin/main.lua',
|
||||||
|
isManagedThemeAvailable: () => true,
|
||||||
|
isManagedThumbnailerAvailable: () => thumbnailerAvailable,
|
||||||
|
installManagedPluginAssets: async () => {
|
||||||
|
calls.push('install');
|
||||||
|
thumbnailerAvailable = true;
|
||||||
|
return { ok: true, status: 'installed', path: '/tmp/plugin/main.lua' };
|
||||||
|
},
|
||||||
|
log: (_level, _configured, message) => {
|
||||||
|
calls.push(message);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.deepEqual(calls, [
|
||||||
|
'Linux runtime support assets missing; installing managed plugin/theme/thumbnailer assets.',
|
||||||
|
'install',
|
||||||
|
'Managed Linux runtime support assets installed: plugin=/tmp/plugin/main.lua theme=/tmp/xdg-data/SubMiner/themes/subminer.rasi thumbnailer=/tmp/xdg-data/SubMiner/thumbnailers/subminer-ffmpegthumbnailer.thumbnailer',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ensureLinuxRuntimePluginAvailable retains an installed plugin after installing support assets', async () => {
|
||||||
|
const calls: string[] = [];
|
||||||
|
let thumbnailerAvailable = false;
|
||||||
|
|
||||||
|
await ensureLinuxRuntimePluginAvailable({
|
||||||
|
platform: 'linux',
|
||||||
|
xdgDataHome: '/tmp/xdg-data',
|
||||||
|
detectInstalledPlugin: () => true,
|
||||||
|
resolveRuntimePluginPath: () => {
|
||||||
|
calls.push('resolve');
|
||||||
|
return null;
|
||||||
|
},
|
||||||
|
isManagedThemeAvailable: () => true,
|
||||||
|
isManagedThumbnailerAvailable: () => thumbnailerAvailable,
|
||||||
|
installManagedPluginAssets: async () => {
|
||||||
|
calls.push('install');
|
||||||
|
thumbnailerAvailable = true;
|
||||||
|
return { ok: true, status: 'installed', path: '/tmp/plugin/main.lua' };
|
||||||
|
},
|
||||||
|
log: () => {},
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.deepEqual(calls, ['install']);
|
||||||
|
});
|
||||||
|
|
||||||
test('ensureLinuxRuntimePluginAvailable installs managed assets and re-resolves plugin path', async () => {
|
test('ensureLinuxRuntimePluginAvailable installs managed assets and re-resolves plugin path', async () => {
|
||||||
const calls: string[] = [];
|
const calls: string[] = [];
|
||||||
let resolveCount = 0;
|
let resolveCount = 0;
|
||||||
@@ -137,6 +208,8 @@ test('ensureLinuxRuntimePluginAvailable installs managed assets and re-resolves
|
|||||||
calls.push(`resolve:${resolveCount}`);
|
calls.push(`resolve:${resolveCount}`);
|
||||||
return resolveCount === 1 ? null : '/tmp/plugin/main.lua';
|
return resolveCount === 1 ? null : '/tmp/plugin/main.lua';
|
||||||
},
|
},
|
||||||
|
isManagedThemeAvailable: () => true,
|
||||||
|
isManagedThumbnailerAvailable: () => true,
|
||||||
installManagedPluginAssets: async () => {
|
installManagedPluginAssets: async () => {
|
||||||
calls.push('install');
|
calls.push('install');
|
||||||
return { ok: true, status: 'installed', path: '/tmp/plugin/main.lua' };
|
return { ok: true, status: 'installed', path: '/tmp/plugin/main.lua' };
|
||||||
@@ -148,9 +221,9 @@ test('ensureLinuxRuntimePluginAvailable installs managed assets and re-resolves
|
|||||||
|
|
||||||
assert.deepEqual(calls, [
|
assert.deepEqual(calls, [
|
||||||
'resolve:1',
|
'resolve:1',
|
||||||
'info:Linux runtime support assets missing; installing managed plugin/theme assets.',
|
'info:Linux runtime support assets missing; installing managed plugin/theme/thumbnailer assets.',
|
||||||
'install',
|
'install',
|
||||||
'info:Managed Linux runtime support assets installed: plugin=/tmp/plugin/main.lua theme=/tmp/xdg-data/SubMiner/themes/subminer.rasi',
|
'info:Managed Linux runtime support assets installed: plugin=/tmp/plugin/main.lua theme=/tmp/xdg-data/SubMiner/themes/subminer.rasi thumbnailer=/tmp/xdg-data/SubMiner/thumbnailers/subminer-ffmpegthumbnailer.thumbnailer',
|
||||||
'resolve:2',
|
'resolve:2',
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
@@ -191,6 +264,60 @@ test('ensureLinuxRuntimePluginAvailable fails when runtime path remains unresolv
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('ensureLinuxRuntimePluginAvailable fails when thumbnailer remains missing after install', async () => {
|
||||||
|
await assert.rejects(
|
||||||
|
() =>
|
||||||
|
ensureLinuxRuntimePluginAvailable({
|
||||||
|
platform: 'linux',
|
||||||
|
xdgDataHome: '/tmp/xdg-data',
|
||||||
|
detectInstalledPlugin: () => true,
|
||||||
|
resolveRuntimePluginPath: () => '/tmp/plugin/main.lua',
|
||||||
|
isManagedThemeAvailable: () => true,
|
||||||
|
isManagedThumbnailerAvailable: () => false,
|
||||||
|
installManagedPluginAssets: async () => ({
|
||||||
|
ok: true,
|
||||||
|
status: 'installed',
|
||||||
|
path: '/tmp/plugin/main.lua',
|
||||||
|
}),
|
||||||
|
log: () => {},
|
||||||
|
}),
|
||||||
|
/thumbnailer=.*subminer-ffmpegthumbnailer\.thumbnailer/i,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ensureLinuxRuntimePluginAvailable rejects a thumbnailer directory before and after install', async () => {
|
||||||
|
const xdgDataHome = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-thumbnailer-directory-'));
|
||||||
|
const thumbnailerPath = path.join(
|
||||||
|
xdgDataHome,
|
||||||
|
'SubMiner',
|
||||||
|
'thumbnailers',
|
||||||
|
'subminer-ffmpegthumbnailer.thumbnailer',
|
||||||
|
);
|
||||||
|
fs.mkdirSync(thumbnailerPath, { recursive: true });
|
||||||
|
const calls: string[] = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
await assert.rejects(
|
||||||
|
() =>
|
||||||
|
ensureLinuxRuntimePluginAvailable({
|
||||||
|
platform: 'linux',
|
||||||
|
xdgDataHome,
|
||||||
|
detectInstalledPlugin: () => true,
|
||||||
|
isManagedThemeAvailable: () => true,
|
||||||
|
installManagedPluginAssets: async () => {
|
||||||
|
calls.push('install');
|
||||||
|
return { ok: true, status: 'installed', path: '/tmp/plugin/main.lua' };
|
||||||
|
},
|
||||||
|
log: () => {},
|
||||||
|
}),
|
||||||
|
/thumbnailer=.*subminer-ffmpegthumbnailer\.thumbnailer/i,
|
||||||
|
);
|
||||||
|
assert.deepEqual(calls, ['install']);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(xdgDataHome, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('installManagedPluginAssetsViaApp returns launch errors without waiting for a response file', async () => {
|
test('installManagedPluginAssetsViaApp returns launch errors without waiting for a response file', async () => {
|
||||||
let waited = false;
|
let waited = false;
|
||||||
|
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ type EnsureLinuxRuntimePluginAvailableOptions = {
|
|||||||
detectInstalledPlugin?: () => boolean;
|
detectInstalledPlugin?: () => boolean;
|
||||||
resolveRuntimePluginPath?: () => string | null;
|
resolveRuntimePluginPath?: () => string | null;
|
||||||
isManagedThemeAvailable?: () => boolean;
|
isManagedThemeAvailable?: () => boolean;
|
||||||
|
isManagedThumbnailerAvailable?: () => boolean;
|
||||||
installManagedPluginAssets?: () => Promise<EnsureLinuxRuntimePluginAssetsResult>;
|
installManagedPluginAssets?: () => Promise<EnsureLinuxRuntimePluginAssetsResult>;
|
||||||
log?: PreflightLog;
|
log?: PreflightLog;
|
||||||
};
|
};
|
||||||
@@ -48,6 +49,14 @@ function resolveConfiguredLogLevel(
|
|||||||
return logLevel ?? 'warn';
|
return logLevel ?? 'warn';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isRegularFile(filePath: string): boolean {
|
||||||
|
try {
|
||||||
|
return fs.statSync(filePath).isFile();
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function waitForInstallResponse(
|
async function waitForInstallResponse(
|
||||||
responsePath: string,
|
responsePath: string,
|
||||||
): Promise<RuntimePluginPreflightResponse | null> {
|
): Promise<RuntimePluginPreflightResponse | null> {
|
||||||
@@ -170,15 +179,17 @@ export async function ensureLinuxRuntimePluginAvailable(
|
|||||||
});
|
});
|
||||||
const isManagedThemeAvailable =
|
const isManagedThemeAvailable =
|
||||||
options.isManagedThemeAvailable ?? (() => fs.existsSync(managedPaths.themePath));
|
options.isManagedThemeAvailable ?? (() => fs.existsSync(managedPaths.themePath));
|
||||||
|
const isManagedThumbnailerAvailable =
|
||||||
|
options.isManagedThumbnailerAvailable ?? (() => isRegularFile(managedPaths.thumbnailerPath));
|
||||||
const runtimePluginAvailable = installedPluginAvailable || Boolean(resolveRuntimePluginPath());
|
const runtimePluginAvailable = installedPluginAvailable || Boolean(resolveRuntimePluginPath());
|
||||||
if (runtimePluginAvailable && isManagedThemeAvailable()) {
|
if (runtimePluginAvailable && isManagedThemeAvailable() && isManagedThumbnailerAvailable()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
log(
|
log(
|
||||||
'info',
|
'info',
|
||||||
configuredLogLevel,
|
configuredLogLevel,
|
||||||
'Linux runtime support assets missing; installing managed plugin/theme assets.',
|
'Linux runtime support assets missing; installing managed plugin/theme/thumbnailer assets.',
|
||||||
);
|
);
|
||||||
const installManagedPluginAssets =
|
const installManagedPluginAssets =
|
||||||
options.installManagedPluginAssets ??
|
options.installManagedPluginAssets ??
|
||||||
@@ -207,16 +218,21 @@ export async function ensureLinuxRuntimePluginAvailable(
|
|||||||
log(
|
log(
|
||||||
'info',
|
'info',
|
||||||
configuredLogLevel,
|
configuredLogLevel,
|
||||||
`Managed Linux runtime support assets installed: plugin=${installResult.path ?? 'unknown path'} theme=${managedPaths.themePath}`,
|
`Managed Linux runtime support assets installed: plugin=${installResult.path ?? 'unknown path'} theme=${managedPaths.themePath} thumbnailer=${managedPaths.thumbnailerPath}`,
|
||||||
);
|
);
|
||||||
const runtimePluginPath = resolveRuntimePluginPath();
|
const runtimePluginAvailableAfterInstall =
|
||||||
if (runtimePluginPath) {
|
installedPluginAvailable || Boolean(resolveRuntimePluginPath());
|
||||||
|
if (
|
||||||
|
runtimePluginAvailableAfterInstall &&
|
||||||
|
isManagedThemeAvailable() &&
|
||||||
|
isManagedThumbnailerAvailable()
|
||||||
|
) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const message =
|
const message =
|
||||||
`Linux managed runtime plugin assets could not be installed. ` +
|
`Linux managed runtime plugin assets could not be installed. ` +
|
||||||
`Checked path: ${managedPaths.pluginEntrypointPath}. ` +
|
`Checked paths: plugin=${managedPaths.pluginEntrypointPath} theme=${managedPaths.themePath} thumbnailer=${managedPaths.thumbnailerPath}. ` +
|
||||||
'Launch aborted before starting mpv.';
|
'Launch aborted before starting mpv.';
|
||||||
log('warn', configuredLogLevel, message);
|
log('warn', configuredLogLevel, message);
|
||||||
throw new Error(message);
|
throw new Error(message);
|
||||||
|
|||||||
@@ -165,11 +165,14 @@ if (entry.argv.includes('--ensure-linux-runtime-plugin-assets')) {
|
|||||||
const pluginDir = path.join(dataDir, 'plugin', 'subminer');
|
const pluginDir = path.join(dataDir, 'plugin', 'subminer');
|
||||||
const pluginConfigPath = path.join(dataDir, 'plugin', 'subminer.conf');
|
const pluginConfigPath = path.join(dataDir, 'plugin', 'subminer.conf');
|
||||||
const themePath = path.join(dataDir, 'themes', 'subminer.rasi');
|
const themePath = path.join(dataDir, 'themes', 'subminer.rasi');
|
||||||
|
const thumbnailerPath = path.join(dataDir, 'thumbnailers', 'subminer-ffmpegthumbnailer.thumbnailer');
|
||||||
fs.mkdirSync(pluginDir, { recursive: true });
|
fs.mkdirSync(pluginDir, { recursive: true });
|
||||||
fs.mkdirSync(path.dirname(themePath), { recursive: true });
|
fs.mkdirSync(path.dirname(themePath), { recursive: true });
|
||||||
|
fs.mkdirSync(path.dirname(thumbnailerPath), { recursive: true });
|
||||||
fs.writeFileSync(path.join(pluginDir, 'main.lua'), '-- smoke plugin\\n');
|
fs.writeFileSync(path.join(pluginDir, 'main.lua'), '-- smoke plugin\\n');
|
||||||
fs.writeFileSync(pluginConfigPath, 'smoke=true\\n');
|
fs.writeFileSync(pluginConfigPath, 'smoke=true\\n');
|
||||||
fs.writeFileSync(themePath, '/* smoke theme */\\n');
|
fs.writeFileSync(themePath, '/* smoke theme */\\n');
|
||||||
|
fs.writeFileSync(thumbnailerPath, '[Thumbnailer Entry]\\n');
|
||||||
if (responsePath) {
|
if (responsePath) {
|
||||||
fs.mkdirSync(path.dirname(responsePath), { recursive: true });
|
fs.mkdirSync(path.dirname(responsePath), { recursive: true });
|
||||||
fs.writeFileSync(responsePath, JSON.stringify({ ok: true, status: 'installed', path: path.join(pluginDir, 'main.lua') }));
|
fs.writeFileSync(responsePath, JSON.stringify({ ok: true, status: 'installed', path: path.join(pluginDir, 'main.lua') }));
|
||||||
@@ -620,11 +623,22 @@ test(
|
|||||||
);
|
);
|
||||||
assert.match(result.stdout, /pause mpv until overlay and tokenization are ready/i);
|
assert.match(result.stdout, /pause mpv until overlay and tokenization are ready/i);
|
||||||
if (process.platform === 'linux') {
|
if (process.platform === 'linux') {
|
||||||
assert.match(result.stdout, /managed plugin\/theme assets/i);
|
assert.match(result.stdout, /managed plugin\/theme\/thumbnailer assets/i);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
fs.existsSync(path.join(smokeCase.xdgDataHome, 'SubMiner', 'themes', 'subminer.rasi')),
|
fs.existsSync(path.join(smokeCase.xdgDataHome, 'SubMiner', 'themes', 'subminer.rasi')),
|
||||||
true,
|
true,
|
||||||
);
|
);
|
||||||
|
assert.equal(
|
||||||
|
fs.existsSync(
|
||||||
|
path.join(
|
||||||
|
smokeCase.xdgDataHome,
|
||||||
|
'SubMiner',
|
||||||
|
'thumbnailers',
|
||||||
|
'subminer-ffmpegthumbnailer.thumbnailer',
|
||||||
|
),
|
||||||
|
),
|
||||||
|
true,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -18,6 +18,9 @@ export function createImmersionDbFixture(dbPath: string): void {
|
|||||||
db.prepare(
|
db.prepare(
|
||||||
`INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('last_rollup_sample_ms', 0)`,
|
`INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('last_rollup_sample_ms', 0)`,
|
||||||
).run();
|
).run();
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('lexical_daily_rollups_version', 0)`,
|
||||||
|
).run();
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`INSERT INTO imm_lifetime_global(global_id, CREATED_DATE, LAST_UPDATE_DATE) VALUES (1, ?, ?)`,
|
`INSERT INTO imm_lifetime_global(global_id, CREATED_DATE, LAST_UPDATE_DATE) VALUES (1, ?, ?)`,
|
||||||
).run(String(Date.now()), String(Date.now()));
|
).run(String(Date.now()), String(Date.now()));
|
||||||
|
|||||||
@@ -108,6 +108,36 @@ test('fixture schema stays aligned with production sync-touched tables and index
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('fixture leaves lexical rollups pending when their table is absent', () => {
|
||||||
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-fixture-rollup-state-'));
|
||||||
|
const fixturePath = path.join(dir, 'fixture.sqlite');
|
||||||
|
try {
|
||||||
|
createImmersionDbFixture(fixturePath);
|
||||||
|
const db = new BunDatabase(fixturePath, { readonly: true });
|
||||||
|
try {
|
||||||
|
const state = db
|
||||||
|
.query<{ state_value: string }>(
|
||||||
|
`SELECT state_value FROM imm_rollup_state
|
||||||
|
WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||||
|
)
|
||||||
|
.get();
|
||||||
|
const rollupTable = db
|
||||||
|
.query<{ name: string }>(
|
||||||
|
`SELECT name FROM sqlite_schema
|
||||||
|
WHERE type = 'table' AND name = 'imm_lexical_daily_rollups'`,
|
||||||
|
)
|
||||||
|
.get();
|
||||||
|
|
||||||
|
assert.equal(state?.state_value, '0');
|
||||||
|
assert.equal(rollupTable, null);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('fixture session inserts enforce foreign keys', () => {
|
test('fixture session inserts enforce foreign keys', () => {
|
||||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-fixture-foreign-keys-'));
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-fixture-foreign-keys-'));
|
||||||
const fixturePath = path.join(dir, 'fixture.sqlite');
|
const fixturePath = path.join(dir, 'fixture.sqlite');
|
||||||
|
|||||||
@@ -154,6 +154,7 @@ export const IMMERSION_DB_FIXTURE_DDL = `
|
|||||||
last_seen REAL,
|
last_seen REAL,
|
||||||
frequency INTEGER,
|
frequency INTEGER,
|
||||||
frequency_rank INTEGER,
|
frequency_rank INTEGER,
|
||||||
|
vocabulary_visible INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1)),
|
||||||
UNIQUE(headword, word, reading)
|
UNIQUE(headword, word, reading)
|
||||||
);
|
);
|
||||||
CREATE TABLE imm_kanji(
|
CREATE TABLE imm_kanji(
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
"name": "subminer",
|
"name": "subminer",
|
||||||
"productName": "SubMiner",
|
"productName": "SubMiner",
|
||||||
"desktopName": "SubMiner.desktop",
|
"desktopName": "SubMiner.desktop",
|
||||||
"version": "0.19.3",
|
"version": "0.19.4-beta.2",
|
||||||
"description": "All-in-one sentence mining overlay with AnkiConnect and dictionary integration",
|
"description": "All-in-one sentence mining overlay with AnkiConnect and dictionary integration",
|
||||||
"packageManager": "bun@1.3.5",
|
"packageManager": "bun@1.3.5",
|
||||||
"main": "dist/main-entry.js",
|
"main": "dist/main-entry.js",
|
||||||
|
|||||||
@@ -58,6 +58,8 @@ package() {
|
|||||||
"${pkgdir}/usr/share/SubMiner/plugin/subminer.conf"
|
"${pkgdir}/usr/share/SubMiner/plugin/subminer.conf"
|
||||||
install -Dm644 "${srcdir}/assets/themes/subminer.rasi" \
|
install -Dm644 "${srcdir}/assets/themes/subminer.rasi" \
|
||||||
"${pkgdir}/usr/share/SubMiner/themes/subminer.rasi"
|
"${pkgdir}/usr/share/SubMiner/themes/subminer.rasi"
|
||||||
|
install -Dm644 "${srcdir}/assets/thumbnailers/subminer-ffmpegthumbnailer.thumbnailer" \
|
||||||
|
"${pkgdir}/usr/share/SubMiner/thumbnailers/subminer-ffmpegthumbnailer.thumbnailer"
|
||||||
|
|
||||||
install -dm755 "${pkgdir}/usr/share/SubMiner/plugin/subminer"
|
install -dm755 "${pkgdir}/usr/share/SubMiner/plugin/subminer"
|
||||||
cp -a "${srcdir}/plugin/subminer/." "${pkgdir}/usr/share/SubMiner/plugin/subminer/"
|
cp -a "${srcdir}/plugin/subminer/." "${pkgdir}/usr/share/SubMiner/plugin/subminer/"
|
||||||
|
|||||||
@@ -106,8 +106,11 @@ function M.create(ctx)
|
|||||||
|
|
||||||
local function get_subtitle_ass_property()
|
local function get_subtitle_ass_property()
|
||||||
local ass_text = mp.get_property("sub-text/ass")
|
local ass_text = mp.get_property("sub-text/ass")
|
||||||
if type(ass_text) == "string" and ass_text ~= "" then
|
if ass_text ~= nil then
|
||||||
return ass_text
|
if type(ass_text) == "string" and ass_text ~= "" then
|
||||||
|
return ass_text
|
||||||
|
end
|
||||||
|
return nil
|
||||||
end
|
end
|
||||||
ass_text = mp.get_property("sub-text-ass")
|
ass_text = mp.get_property("sub-text-ass")
|
||||||
if type(ass_text) == "string" and ass_text ~= "" then
|
if type(ass_text) == "string" and ass_text ~= "" then
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ local OVERLAY_RESTART_PING_MAX_ATTEMPTS = 20
|
|||||||
local OVERLAY_LOADING_OSD_PREFIX = "Overlay loading "
|
local OVERLAY_LOADING_OSD_PREFIX = "Overlay loading "
|
||||||
local OVERLAY_LOADING_OSD_FRAMES = { "|", "/", "-", "\\" }
|
local OVERLAY_LOADING_OSD_FRAMES = { "|", "/", "-", "\\" }
|
||||||
local OVERLAY_LOADING_OSD_REFRESH_SECONDS = 0.18
|
local OVERLAY_LOADING_OSD_REFRESH_SECONDS = 0.18
|
||||||
|
local OVERLAY_LOADING_OSD_DEADLINE_SECONDS = 30
|
||||||
|
local OVERLAY_LOADING_OSD_TIMEOUT_MESSAGE = "Overlay did not become ready; check SubMiner logs"
|
||||||
local AUTO_PLAY_READY_LOADING_OSD = "Loading subtitle tokenization..."
|
local AUTO_PLAY_READY_LOADING_OSD = "Loading subtitle tokenization..."
|
||||||
local AUTO_PLAY_READY_READY_OSD = "Subtitle tokenization ready"
|
local AUTO_PLAY_READY_READY_OSD = "Subtitle tokenization ready"
|
||||||
local DEFAULT_AUTO_PLAY_READY_TIMEOUT_SECONDS = 30
|
local DEFAULT_AUTO_PLAY_READY_TIMEOUT_SECONDS = 30
|
||||||
@@ -265,10 +267,19 @@ function M.create(ctx)
|
|||||||
state.overlay_loading_osd_timer = nil
|
state.overlay_loading_osd_timer = nil
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function clear_overlay_loading_osd_deadline()
|
||||||
|
local timeout = state.overlay_loading_osd_deadline
|
||||||
|
if timeout and timeout.kill then
|
||||||
|
timeout:kill()
|
||||||
|
end
|
||||||
|
state.overlay_loading_osd_deadline = nil
|
||||||
|
end
|
||||||
|
|
||||||
local function stop_overlay_loading_osd()
|
local function stop_overlay_loading_osd()
|
||||||
state.overlay_loading_osd_active = false
|
state.overlay_loading_osd_active = false
|
||||||
state.overlay_loading_osd_frame = 1
|
state.overlay_loading_osd_frame = 1
|
||||||
clear_overlay_loading_osd_timer()
|
clear_overlay_loading_osd_timer()
|
||||||
|
clear_overlay_loading_osd_deadline()
|
||||||
end
|
end
|
||||||
|
|
||||||
local function start_overlay_loading_osd()
|
local function start_overlay_loading_osd()
|
||||||
@@ -291,6 +302,21 @@ function M.create(ctx)
|
|||||||
end
|
end
|
||||||
end)
|
end)
|
||||||
end
|
end
|
||||||
|
if type(mp.add_timeout) == "function" then
|
||||||
|
state.overlay_loading_osd_deadline = mp.add_timeout(OVERLAY_LOADING_OSD_DEADLINE_SECONDS, function()
|
||||||
|
if not state.overlay_loading_osd_active then
|
||||||
|
return
|
||||||
|
end
|
||||||
|
state.overlay_loading_osd_deadline = nil
|
||||||
|
stop_overlay_loading_osd()
|
||||||
|
subminer_log(
|
||||||
|
"warn",
|
||||||
|
"process",
|
||||||
|
"Overlay loading deadline expired before the app reported content ready"
|
||||||
|
)
|
||||||
|
show_osd(OVERLAY_LOADING_OSD_TIMEOUT_MESSAGE, { force = true })
|
||||||
|
end)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
local function disarm_auto_play_ready_gate(options)
|
local function disarm_auto_play_ready_gate(options)
|
||||||
|
|||||||
@@ -232,7 +232,7 @@ function M.create(ctx)
|
|||||||
elseif action_id == "triggerFieldGrouping" then
|
elseif action_id == "triggerFieldGrouping" then
|
||||||
return { "--trigger-field-grouping" }
|
return { "--trigger-field-grouping" }
|
||||||
elseif action_id == "triggerSubsync" then
|
elseif action_id == "triggerSubsync" then
|
||||||
return { "--trigger-subsync" }
|
return { "--session-action", '{"actionId":"triggerSubsync"}' }
|
||||||
elseif action_id == "mineSentence" then
|
elseif action_id == "mineSentence" then
|
||||||
return { "--mine-sentence" }
|
return { "--mine-sentence" }
|
||||||
elseif action_id == "mineSentenceMultiple" then
|
elseif action_id == "mineSentenceMultiple" then
|
||||||
@@ -251,7 +251,7 @@ function M.create(ctx)
|
|||||||
elseif action_id == "markWatched" then
|
elseif action_id == "markWatched" then
|
||||||
return { "--mark-watched" }
|
return { "--mark-watched" }
|
||||||
elseif action_id == "openRuntimeOptions" then
|
elseif action_id == "openRuntimeOptions" then
|
||||||
return { "--open-runtime-options" }
|
return { "--session-action", '{"actionId":"openRuntimeOptions"}' }
|
||||||
elseif action_id == "openJimaku" then
|
elseif action_id == "openJimaku" then
|
||||||
return { "--open-jimaku" }
|
return { "--open-jimaku" }
|
||||||
elseif action_id == "openTsukihime" or action_id == "openAnimetosho" then
|
elseif action_id == "openTsukihime" or action_id == "openAnimetosho" then
|
||||||
@@ -259,7 +259,7 @@ function M.create(ctx)
|
|||||||
elseif action_id == "openYoutubePicker" then
|
elseif action_id == "openYoutubePicker" then
|
||||||
return { "--open-youtube-picker" }
|
return { "--open-youtube-picker" }
|
||||||
elseif action_id == "openSessionHelp" then
|
elseif action_id == "openSessionHelp" then
|
||||||
return { "--open-session-help" }
|
return { "--session-action", '{"actionId":"openSessionHelp"}' }
|
||||||
elseif action_id == "openCharacterDictionaryManager" then
|
elseif action_id == "openCharacterDictionaryManager" then
|
||||||
return { "--session-action", '{"actionId":"openCharacterDictionaryManager"}' }
|
return { "--session-action", '{"actionId":"openCharacterDictionaryManager"}' }
|
||||||
elseif action_id == "openControllerSelect" then
|
elseif action_id == "openControllerSelect" then
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ function M.new()
|
|||||||
auto_play_ready_initial_pause_ownership_consumed = false,
|
auto_play_ready_initial_pause_ownership_consumed = false,
|
||||||
overlay_loading_osd_active = false,
|
overlay_loading_osd_active = false,
|
||||||
overlay_loading_osd_timer = nil,
|
overlay_loading_osd_timer = nil,
|
||||||
|
overlay_loading_osd_deadline = nil,
|
||||||
overlay_loading_osd_frame = 1,
|
overlay_loading_osd_frame = 1,
|
||||||
pending_visible_overlay_hide_timer = nil,
|
pending_visible_overlay_hide_timer = nil,
|
||||||
pending_visible_overlay_hide_generation = 0,
|
pending_visible_overlay_hide_generation = 0,
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ function M.create(ctx)
|
|||||||
local mp = ctx.mp
|
local mp = ctx.mp
|
||||||
local input = ctx.input
|
local input = ctx.input
|
||||||
local process = ctx.process
|
local process = ctx.process
|
||||||
|
local state = ctx.state
|
||||||
local subminer_log = ctx.log.subminer_log
|
local subminer_log = ctx.log.subminer_log
|
||||||
local show_osd = ctx.log.show_osd
|
local show_osd = ctx.log.show_osd
|
||||||
|
|
||||||
@@ -93,7 +94,18 @@ function M.create(ctx)
|
|||||||
if not ensure_binary_for_menu() then
|
if not ensure_binary_for_menu() then
|
||||||
return
|
return
|
||||||
end
|
end
|
||||||
process.run_control_command_async("open-session-help")
|
process.run_binary_command_async({
|
||||||
|
state.binary_path,
|
||||||
|
"--session-action",
|
||||||
|
'{"actionId":"openSessionHelp"}',
|
||||||
|
}, function(ok, result, error)
|
||||||
|
if ok then
|
||||||
|
return
|
||||||
|
end
|
||||||
|
local reason = error or (result and result.stderr) or "unknown error"
|
||||||
|
subminer_log("warn", "session-bindings", "Session action failed: " .. tostring(reason))
|
||||||
|
show_osd("Session action failed")
|
||||||
|
end)
|
||||||
end)
|
end)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|||||||
@@ -1,80 +1,65 @@
|
|||||||
> This is a prerelease build for testing. Stable changelog and docs-site updates remain pending until the final stable release.
|
> This is a prerelease build for testing. Stable changelog and docs-site updates remain pending until the final stable release.
|
||||||
|
|
||||||
<!-- prerelease-base-version: 0.19.0 -->
|
<!-- prerelease-base-version: 0.19.4 -->
|
||||||
|
|
||||||
## Highlights
|
## Highlights
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
- **Sync Stats & History**
|
- Library Merge & Reassignment
|
||||||
- New **Sync Stats & History** window (tray menu) and `subminer sync <host>` command keep mining stats and watch history in sync between machines over SSH, with saved devices, per-host sync direction, and live stage-by-stage progress.
|
- Duplicate library entries for the same show can now be merged: pick entries in "Select" mode and use "Merge Selected" to combine sessions, mined cards, and watch time onto one card.
|
||||||
- Merges are safe to repeat: data combines without duplicates, and hosts with auto-sync enabled sync automatically in the background on a schedule, reporting results as overlay notifications.
|
- Episodes can be moved to a different library entry with a per-episode "→" button, fixing cases where a stray filename split off its own entry; manual assignments now survive later filename parsing, Jellyfin refreshes, and season repair.
|
||||||
- Manual snapshot tools (create, merge, reveal, delete) and connection testing cover one-off transfers; Windows machines running the built-in OpenSSH Server work as sync remotes too, with no setup needed beyond SSH access. Power users can script transfers directly with `--push`/`--pull`, `--check`, `--snapshot`/`--merge`, and `--json` flags.
|
- Exact AniList matches with compatible seasons now merge automatically, and likely (fuzzy) matches surface as a dismissible "Possible duplicate" suggestion instead of merging silently.
|
||||||
|
|
||||||
- **TsukiHime Subtitle Downloads**
|
- Duplicate Line Cleanup
|
||||||
- Download Japanese and secondary-language subtitles for the current video directly from TsukiHime, mirroring the existing Jimaku flow.
|
- The Vocabulary tab's new **Duplicates** button scans a chosen time window for the repeated-line bursts described under Fixed below and collapses each burst to a single line once you confirm it; a matching `subminer stats cleanup --duplicate-lines` command (with `--dry-run` and `--lookback-days <n>`) is available from the terminal.
|
||||||
- Press `Ctrl+Shift+T` to search by tabs for the primary and secondary languages; the matching release is found automatically from the video filename and loads straight into mpv, no API key required.
|
- Only the affected subtitle lines and the vocabulary counts they inflated are touched; watch time and lines-seen totals are left as recorded.
|
||||||
|
|
||||||
- **Post-Playback History Menu**
|
|
||||||
- After a watch-history episode ends or mpv closes, the fzf/rofi launcher returns to that series with options to play the previous or next episode, rewatch, pick another episode, or quit SubMiner.
|
|
||||||
- Previous/Next continue across season directories, so you can binge a show without manually browsing folders.
|
|
||||||
- The menu shown right after picking a series from `subminer -H` now offers the previous episode too, matching the post-playback menu.
|
|
||||||
|
|
||||||
- **Known-Word Highlighting by Anki Maturity**
|
|
||||||
- Subtitle highlights for known words can now be colored by Anki card maturity (new, learning, young, mature), similar to asbplayer. Enable it with `ankiConnect.knownWords.maturityEnabled`, or toggle it live during a session.
|
|
||||||
- The mature-interval threshold and the four tier colors are configurable, and the in-session help legend shows the active tier colors while maturity highlighting is on.
|
|
||||||
- Tiers follow Anki's own card state: a lapsed card correctly shows as learning rather than young, and a note is treated as mature if any of its cards are mature. Stats and other known-word tools stay accurate with this new data.
|
|
||||||
|
|
||||||
- **Stats Library Entry Deletion**
|
|
||||||
- Added a "Delete Entry" action in the stats Library detail view that removes an entire title in one step: every episode, session, subtitle line, rollup, cover, and vocabulary count derived from it. Previously a mistaken entry had to be cleared episode by episode and still lingered in the Library.
|
|
||||||
- Delete progress (session, session group, episode, or full entry) now shows app-wide as a progress bar plus a status toast, staying visible across tabs and windows instead of disappearing when you switch away.
|
|
||||||
- Deletes are dramatically faster on large libraries, and opening the Vocabulary tab no longer stalls; the first launch after upgrading migrates the stats database in place to support this.
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
|
|
||||||
- **Clipboard-Video Shortcut**
|
|
||||||
- The "append clipboard video to queue" shortcut is now configurable via `shortcuts.appendClipboardVideoToQueue` instead of being fixed.
|
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
- **Word Highlighting Accuracy**
|
- Subtitle Duplication from Karaoke & Animated Signs
|
||||||
- Fixed several incorrect word highlighting and annotation cases: inconsistent part-of-speech exclusions on merged quote-particle tokens, missing annotations for rare kanji, katakana punctuation wrongly treated as non-kana noise, and certain kanji vocabulary skipped for next-level ("N+1") highlighting.
|
- Typeset ASS karaoke and animated signs no longer flood the overlay, subtitle sidebar, immersion history, mined cards, or stats with repeated glyph fragments or per-frame duplicates; the complete authored line is recovered instead, without merging genuinely repeated dialogue or separately positioned signs.
|
||||||
|
- The secondary overlay now shares the same deduplication logic as the primary overlay, so layered animation text no longer appears multiple times there or in what gets mined.
|
||||||
|
- Vocabulary stats no longer count every animation frame of a karaoke opening as a separate line, which previously could push an OP lyric to the top of "Top Repeated Words."
|
||||||
|
|
||||||
- **AniList Season Resolution**
|
- Anki Media Generation
|
||||||
- Season 2 and later episodes now resolve to the correct AniList entry by walking sequel relations instead of guessing from the title, so watch progress, the character dictionary, and cover art for later seasons no longer silently fall back to season 1.
|
- Sentence-audio generation no longer times out on slow network-mounted video files with many subtitle and font streams, and a failed extraction now reports a clear error instead of a raw `ENOENT`.
|
||||||
- Manual AniList overrides now stay in effect for every episode in the same season (by folder and detected season), and setting an override now fixes both the character dictionary and AniList watch progress together instead of needing separate corrections.
|
- Mined audio and animated AVIF clips now capture the subtitle line you actually mined, instead of whatever line happened to be on screen once slow audio extraction finished.
|
||||||
|
|
||||||
- **Startup Playback Pausing Too Early**
|
- Character Dictionary Performance & Notifications
|
||||||
- Fixed playback resuming before subtitle processing finished warming up, which could briefly show untranslated subtitles right after opening a video.
|
- Character dictionary generation, merged rebuilds, and imports no longer freeze the app on large dictionaries, and cached results are reused across launches instead of regenerating character data and portraits every time.
|
||||||
- Most noticeable when resuming mid-episode or when a subtitle cue starts within the first couple of seconds.
|
- Desktop progress notifications, including on Linux AppImage installs, now update in place instead of flickering closed and reopening.
|
||||||
|
|
||||||
- **Linux AppImage Crash Notification on Quit**
|
- Overlay Reliability
|
||||||
- Fixed a spurious "Service Crash" desktop notification appearing after closing a video when running the Linux AppImage.
|
- Overlay modals (settings, stats, etc.) now open promptly on the first shortcut press and appear above fullscreen mpv on macOS instead of switching Spaces or opening off-screen.
|
||||||
- If needed, the mount-keepalive behavior behind this fix can be disabled with `SUBMINER_NO_APPIMAGE_MOUNT_KEEPALIVE=1`.
|
- The overlay no longer gets stuck on "Overlay loading" indefinitely if mpv's connection stalls; it now retries and shows an actionable error after 30 seconds.
|
||||||
|
- Fixed native Wayland drag-and-drop from file managers like Thunar, so subtitle and video files dropped on the overlay reach mpv.
|
||||||
|
- Fixed system-wide mouse lag on Windows caused by the overlay's click-through handling and repeated mpv window lookups.
|
||||||
|
|
||||||
- **AnkiConnect Proxy Port Conflict**
|
- Stats Dashboard
|
||||||
- Fixed video playback failing to start when another process already held the configured AnkiConnect proxy port; SubMiner now shows a notification explaining how to resolve the conflict instead of crashing.
|
- Deletes, library merges, video moves, and AniList reassignments no longer freeze the stats dashboard or rebuild lifetime totals from scratch; large deletes that used to take minutes now finish in milliseconds.
|
||||||
|
- Vocabulary totals and charts now count all tracked vocabulary instead of just the first page, new-word history uses corrected daily rollups, calendar labels respect time zones west of UTC, and vocabulary cards refresh automatically after editing the word exclusion list.
|
||||||
|
|
||||||
- **Stats & Settings Reliability**
|
- Linux Launcher Thumbnails
|
||||||
- Fixed session stats reporting zero known words after the known-word cache gained maturity tiers.
|
- Fixed missing MKV thumbnails in the Linux rofi picker when the system thumbnailer only registers legacy Matroska MIME aliases.
|
||||||
- Hardened the stats server against malformed requests, stalled AniList lookups, media mismatches during word mining, and missing Yomitan connections.
|
|
||||||
- AnkiConnect settings validation now preserves valid custom configurations while safely falling back on invalid values instead of failing.
|
|
||||||
|
|
||||||
- **Stats Library Cover After Relink**
|
|
||||||
- Relinking a title to a different AniList entry now updates its cover art in the stats Library grid, not just the detail view, so unrelated titles no longer end up sharing the wrong cover.
|
|
||||||
|
|
||||||
- **Rofi Menu Prompt Spacing**
|
|
||||||
- Rofi menu prompts now keep a space between the prompt label and the input field instead of crowding the search placeholder text.
|
|
||||||
|
|
||||||
## What's Changed
|
## What's Changed
|
||||||
|
|
||||||
- feat(shortcuts): make clipboard-video-append shortcut configurable by @ksyasuda in #158
|
- feat(stats): add library entry merge and episode move by @ksyasuda in #190
|
||||||
- refactor(tokenizer): extract subtitle annotation filter into rule table by @ksyasuda in #162
|
- fix(stats): stop counting duplicate typeset subtitle lines by @ksyasuda in #191
|
||||||
- refactor(tsukihime): swap Animetosho backend for TsukiHime API by @ksyasuda in #165
|
- fix(media): tolerate slow MKV audio extraction by @ksyasuda in #195
|
||||||
- refactor: split anki-connect and stats-server resolvers into modules by @ksyasuda in #169
|
- fix(stats): subtract lifetime totals incrementally on delete by @ksyasuda in #196
|
||||||
- feat(launcher): add post-playback history menu with previous episode by @ksyasuda in #170
|
- fix(anki): snapshot mining media clip timing by @ksyasuda in #197
|
||||||
- Anki maturity-based known-word highlighting by @ksyasuda in #172
|
- fix(notifications): replace Linux progress updates in place by @ksyasuda in #198
|
||||||
- fix(anilist): resolve later seasons via sequel relations, not title guessing by @ksyasuda in #173
|
- fix(overlay): support native Wayland file drag-and-drop by @ksyasuda in #199
|
||||||
- feat(stats): add library entry deletion and app-wide delete progress by @ksyasuda in #174
|
- fix(overlay): keep macOS modal windows on fullscreen Spaces by @ksyasuda in #200
|
||||||
|
- fix(overlay): prevent Windows mouse lag during click-through tracking by @ksyasuda in #201
|
||||||
|
- fix(stats): report complete vocabulary totals and new-word history by @ksyasuda in #202
|
||||||
|
- fix(mpv): recover from stalled IPC connects by @ksyasuda in #204
|
||||||
|
- fix(dictionary): prevent freezes and restore AppImage notifications by @ksyasuda in #205
|
||||||
|
- fix(subtitles): recover canonical lines from ASS animation by @ksyasuda in #207
|
||||||
|
- fix(overlay): deduplicate secondary subtitle rendering by @ksyasuda in #208
|
||||||
|
- fix(launcher): restore Matroska thumbnails in Linux rofi picker by @ksyasuda in #210
|
||||||
|
- fix(character-dictionary): cache completed MeCab refreshes by @ksyasuda in #212
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
|
|||||||
@@ -35,6 +35,17 @@ const archiveCacheRoot = join(repoRoot, '.tmp/docs-versioned-archive-cache');
|
|||||||
const maxCloudflareFiles = 20_000;
|
const maxCloudflareFiles = 20_000;
|
||||||
const maxCloudflareFileBytes = 25 * 1024 * 1024;
|
const maxCloudflareFileBytes = 25 * 1024 * 1024;
|
||||||
|
|
||||||
|
// Cloudflare Pages header rules for the whole deployment. Mirrors the `noindex,follow`
|
||||||
|
// meta tag the non-root channels emit, so the duplicate trees stay out of the index
|
||||||
|
// even for responses a crawler takes without parsing the HTML.
|
||||||
|
const deployHeaders = `# Generated by scripts/build-versioned-docs.ts. Do not edit by hand.
|
||||||
|
/main/*
|
||||||
|
X-Robots-Tag: noindex, follow
|
||||||
|
|
||||||
|
/v/*
|
||||||
|
X-Robots-Tag: noindex, follow
|
||||||
|
`;
|
||||||
|
|
||||||
function run(
|
function run(
|
||||||
command: string,
|
command: string,
|
||||||
args: string[],
|
args: string[],
|
||||||
@@ -173,6 +184,7 @@ function buildDocs(options: {
|
|||||||
SUBMINER_DOCS_BASE: options.base,
|
SUBMINER_DOCS_BASE: options.base,
|
||||||
SUBMINER_DOCS_OUT_DIR: options.outDir,
|
SUBMINER_DOCS_OUT_DIR: options.outDir,
|
||||||
SUBMINER_DOCS_SOURCE_DIR: options.snapshotDocsSite,
|
SUBMINER_DOCS_SOURCE_DIR: options.snapshotDocsSite,
|
||||||
|
SUBMINER_DOCS_REPO_DIR: currentDocsSite,
|
||||||
SUBMINER_DOCS_CHANNEL: options.channel,
|
SUBMINER_DOCS_CHANNEL: options.channel,
|
||||||
SUBMINER_DOCS_VERSION: options.version ?? '',
|
SUBMINER_DOCS_VERSION: options.version ?? '',
|
||||||
SUBMINER_DOCS_LATEST_STABLE: options.latestStable,
|
SUBMINER_DOCS_LATEST_STABLE: options.latestStable,
|
||||||
@@ -378,6 +390,7 @@ function main() {
|
|||||||
});
|
});
|
||||||
|
|
||||||
writeFileSync(join(aggregateOutDir, 'versions.json'), `${JSON.stringify(manifest, null, 2)}\n`);
|
writeFileSync(join(aggregateOutDir, 'versions.json'), `${JSON.stringify(manifest, null, 2)}\n`);
|
||||||
|
writeFileSync(join(aggregateOutDir, '_headers'), deployHeaders);
|
||||||
assertCloudflarePagesLimits(aggregateOutDir);
|
assertCloudflarePagesLimits(aggregateOutDir);
|
||||||
const prunedArchives = pruneArchiveCacheGenerations({
|
const prunedArchives = pruneArchiveCacheGenerations({
|
||||||
cacheRoot: archiveCacheRoot,
|
cacheRoot: archiveCacheRoot,
|
||||||
|
|||||||
@@ -19,7 +19,8 @@ Options:
|
|||||||
-w, --webp Generate animated WebP preview
|
-w, --webp Generate animated WebP preview
|
||||||
|
|
||||||
Encoding profile:
|
Encoding profile:
|
||||||
- Crop: 1920x1080 at x=760 y=200
|
- Crop: mpv region at 1920x1080, x=760 y=205 on a 3440x1440 canvas
|
||||||
|
- Output size: 1920x1080
|
||||||
- MP4: H.264 + AAC
|
- MP4: H.264 + AAC
|
||||||
- WebM: AV1/VP9 + Opus at 30 fps
|
- WebM: AV1/VP9 + Opus at 30 fps
|
||||||
USAGE
|
USAGE
|
||||||
@@ -148,7 +149,8 @@ pick_webp_encoder() {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
crop_vf="crop=1920:1080:760:205"
|
# OBS may resize the 3440x1440 canvas, so scale the mpv bounds with the input.
|
||||||
|
crop_vf="crop=1920*iw/3440:1080*ih/1440:760*iw/3440:205*ih/1440,scale=1920:1080:flags=lanczos"
|
||||||
webm_vf="${crop_vf},fps=30"
|
webm_vf="${crop_vf},fps=30"
|
||||||
|
|
||||||
echo "Generating MP4: $mp4_out"
|
echo "Generating MP4: $mp4_out"
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ function toBashPath(filePath: string): string {
|
|||||||
return `${drive.toUpperCase()}:/${rest}`;
|
return `${drive.toUpperCase()}:/${rest}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
test('mkv-to-readme-video accepts libwebp_anim when libwebp is unavailable', () => {
|
test('mkv-to-readme-video builds every output with the scaled mpv crop', () => {
|
||||||
withTempDir((root) => {
|
withTempDir((root) => {
|
||||||
const binDir = path.join(root, 'bin');
|
const binDir = path.join(root, 'bin');
|
||||||
const inputPath = path.join(root, 'sample.mkv');
|
const inputPath = path.join(root, 'sample.mkv');
|
||||||
@@ -104,5 +104,9 @@ touch "$output"
|
|||||||
|
|
||||||
const ffmpegLog = fs.readFileSync(ffmpegLogPath, 'utf8');
|
const ffmpegLog = fs.readFileSync(ffmpegLogPath, 'utf8');
|
||||||
assert.match(ffmpegLog, /-c:v libwebp_anim/);
|
assert.match(ffmpegLog, /-c:v libwebp_anim/);
|
||||||
|
const scaledCropUses = ffmpegLog.match(
|
||||||
|
/-vf crop=1920\*iw\/3440:1080\*ih\/1440:760\*iw\/3440:205\*ih\/1440,scale=1920:1080:flags=lanczos/g,
|
||||||
|
);
|
||||||
|
assert.equal(scaledCropUses?.length, 4);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -130,7 +130,9 @@ local function run_plugin_scenario(config)
|
|||||||
|
|
||||||
function mp.add_timeout(seconds, callback)
|
function mp.add_timeout(seconds, callback)
|
||||||
recorded.timeouts[#recorded.timeouts + 1] = seconds
|
recorded.timeouts[#recorded.timeouts + 1] = seconds
|
||||||
|
local delay = tonumber(seconds) or 0
|
||||||
local timeout = {
|
local timeout = {
|
||||||
|
seconds = delay,
|
||||||
killed = false,
|
killed = false,
|
||||||
callback = callback,
|
callback = callback,
|
||||||
}
|
}
|
||||||
@@ -138,7 +140,6 @@ local function run_plugin_scenario(config)
|
|||||||
self.killed = true
|
self.killed = true
|
||||||
end
|
end
|
||||||
|
|
||||||
local delay = tonumber(seconds) or 0
|
|
||||||
if callback and delay < 5 and not config.defer_timeouts then
|
if callback and delay < 5 and not config.defer_timeouts then
|
||||||
callback()
|
callback()
|
||||||
end
|
end
|
||||||
@@ -514,6 +515,15 @@ local function has_timeout(timeouts, target)
|
|||||||
return false
|
return false
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function find_timeout_handle(recorded, target)
|
||||||
|
for _, timeout in ipairs(recorded.timeout_handles) do
|
||||||
|
if math.abs(timeout.seconds - target) < 0.0001 then
|
||||||
|
return timeout
|
||||||
|
end
|
||||||
|
end
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
|
||||||
local function env_has(call, target)
|
local function env_has(call, target)
|
||||||
local env = (call and call.env) or {}
|
local env = (call and call.env) or {}
|
||||||
for _, value in ipairs(env) do
|
for _, value in ipairs(env) do
|
||||||
@@ -1636,6 +1646,8 @@ do
|
|||||||
#recorded.periodic_timers == 1,
|
#recorded.periodic_timers == 1,
|
||||||
"auto-start visible overlay should refresh the early overlay loading OSD"
|
"auto-start visible overlay should refresh the early overlay loading OSD"
|
||||||
)
|
)
|
||||||
|
local overlay_loading_deadline = find_timeout_handle(recorded, 30)
|
||||||
|
assert_true(overlay_loading_deadline ~= nil, "overlay loading OSD should have a bounded deadline")
|
||||||
local overlay_loading_timer = recorded.periodic_timers[1]
|
local overlay_loading_timer = recorded.periodic_timers[1]
|
||||||
recorded.periodic_timers[1].callback()
|
recorded.periodic_timers[1].callback()
|
||||||
assert_true(
|
assert_true(
|
||||||
@@ -1670,6 +1682,46 @@ do
|
|||||||
recorded.periodic_timers[1].killed == true,
|
recorded.periodic_timers[1].killed == true,
|
||||||
"overlay loading ready should stop the early overlay loading OSD refresher"
|
"overlay loading ready should stop the early overlay loading OSD refresher"
|
||||||
)
|
)
|
||||||
|
assert_true(
|
||||||
|
overlay_loading_deadline.killed == true,
|
||||||
|
"overlay loading ready should cancel the bounded loading deadline"
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
do
|
||||||
|
local recorded, err = run_plugin_scenario({
|
||||||
|
defer_timeouts = true,
|
||||||
|
process_list = "",
|
||||||
|
option_overrides = {
|
||||||
|
binary_path = binary_path,
|
||||||
|
auto_start = "yes",
|
||||||
|
auto_start_visible_overlay = "yes",
|
||||||
|
osd_messages = false,
|
||||||
|
socket_path = "/tmp/subminer-socket",
|
||||||
|
},
|
||||||
|
input_ipc_server = "/tmp/subminer-socket",
|
||||||
|
media_title = "Random Movie",
|
||||||
|
files = {
|
||||||
|
[binary_path] = true,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
assert_true(recorded ~= nil, "plugin failed to load for overlay loading deadline scenario: " .. tostring(err))
|
||||||
|
fire_event(recorded, "start-file")
|
||||||
|
local overlay_loading_deadline = find_timeout_handle(recorded, 30)
|
||||||
|
assert_true(overlay_loading_deadline ~= nil, "overlay loading deadline should be scheduled")
|
||||||
|
overlay_loading_deadline.callback()
|
||||||
|
assert_true(
|
||||||
|
recorded.periodic_timers[1].killed == true,
|
||||||
|
"overlay loading deadline should stop the loading spinner"
|
||||||
|
)
|
||||||
|
assert_true(
|
||||||
|
has_osd_message(recorded.osd, "SubMiner: Overlay did not become ready; check SubMiner logs"),
|
||||||
|
"overlay loading deadline should replace the spinner with actionable feedback"
|
||||||
|
)
|
||||||
|
assert_true(
|
||||||
|
has_log_containing(recorded.logs, "Overlay loading deadline expired"),
|
||||||
|
"overlay loading deadline should leave a diagnostic log entry"
|
||||||
|
)
|
||||||
end
|
end
|
||||||
|
|
||||||
do
|
do
|
||||||
|
|||||||
@@ -82,6 +82,7 @@ test('update-aur-package updates PKGBUILD and .SRCINFO without makepkg', () => {
|
|||||||
pkgbuild,
|
pkgbuild,
|
||||||
/^\s*install -Dm755 "\$\{srcdir\}\/subminer-\$\{pkgver\}" "\$\{pkgdir\}\/usr\/bin\/subminer"$/m,
|
/^\s*install -Dm755 "\$\{srcdir\}\/subminer-\$\{pkgver\}" "\$\{pkgdir\}\/usr\/bin\/subminer"$/m,
|
||||||
);
|
);
|
||||||
|
assert.match(pkgbuild, /assets\/thumbnailers\/subminer-ffmpegthumbnailer\.thumbnailer/);
|
||||||
assert.match(srcinfo, /^\tpkgver = 0\.6\.3$/m);
|
assert.match(srcinfo, /^\tpkgver = 0\.6\.3$/m);
|
||||||
assert.match(srcinfo, /^\tprovides = subminer=0\.6\.3$/m);
|
assert.match(srcinfo, /^\tprovides = subminer=0\.6\.3$/m);
|
||||||
assert.match(
|
assert.match(
|
||||||
|
|||||||
@@ -284,6 +284,22 @@ function createMockTracker(
|
|||||||
getSessionTimeline: async () => [],
|
getSessionTimeline: async () => [],
|
||||||
getSessionEvents: async () => [],
|
getSessionEvents: async () => [],
|
||||||
getVocabularyStats: async () => VOCABULARY_STATS,
|
getVocabularyStats: async () => VOCABULARY_STATS,
|
||||||
|
getVocabularySummary: async () => ({
|
||||||
|
uniqueWords: 501,
|
||||||
|
uniqueWordsWithoutNames: 500,
|
||||||
|
uniqueKanji: 201,
|
||||||
|
newThisWeek: 7,
|
||||||
|
newThisWeekWithoutNames: 6,
|
||||||
|
knownWordCount: 250,
|
||||||
|
knownWordCountWithoutNames: 249,
|
||||||
|
}),
|
||||||
|
getVocabularyChartData: async () => ({
|
||||||
|
ready: true,
|
||||||
|
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||||
|
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||||
|
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
|
||||||
|
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
|
||||||
|
}),
|
||||||
getStatsExcludedWords: async () => [],
|
getStatsExcludedWords: async () => [],
|
||||||
replaceStatsExcludedWords: async () => {},
|
replaceStatsExcludedWords: async () => {},
|
||||||
getKanjiStats: async () => KANJI_STATS,
|
getKanjiStats: async () => KANJI_STATS,
|
||||||
@@ -711,6 +727,38 @@ describe('stats server API routes', () => {
|
|||||||
assert.equal(body[0].headword, 'する');
|
assert.equal(body[0].headword, 'する');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('GET /api/stats/vocabulary/summary returns database-wide card totals', async () => {
|
||||||
|
const app = createStatsApp(createMockTracker());
|
||||||
|
|
||||||
|
const res = await app.request('/api/stats/vocabulary/summary');
|
||||||
|
|
||||||
|
assert.equal(res.status, 200);
|
||||||
|
assert.deepEqual(await res.json(), {
|
||||||
|
uniqueWords: 501,
|
||||||
|
uniqueWordsWithoutNames: 500,
|
||||||
|
uniqueKanji: 201,
|
||||||
|
newThisWeek: 7,
|
||||||
|
newThisWeekWithoutNames: 6,
|
||||||
|
knownWordCount: 250,
|
||||||
|
knownWordCountWithoutNames: 249,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('GET /api/stats/vocabulary/charts returns complete chart datasets', async () => {
|
||||||
|
const app = createStatsApp(createMockTracker());
|
||||||
|
|
||||||
|
const res = await app.request('/api/stats/vocabulary/charts');
|
||||||
|
|
||||||
|
assert.equal(res.status, 200);
|
||||||
|
assert.deepEqual(await res.json(), {
|
||||||
|
ready: true,
|
||||||
|
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||||
|
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||||
|
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
|
||||||
|
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('GET /api/stats/kanji returns kanji frequency data', async () => {
|
it('GET /api/stats/kanji returns kanji frequency data', async () => {
|
||||||
const app = createStatsApp(createMockTracker());
|
const app = createStatsApp(createMockTracker());
|
||||||
const res = await app.request('/api/stats/kanji');
|
const res = await app.request('/api/stats/kanji');
|
||||||
|
|||||||
@@ -559,6 +559,241 @@ test('fresh tracker DB creates lifetime summary tables', async () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('fresh tracker DB skips lexical rollup backfill work', async () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
let tracker: ImmersionTrackerService | null = null;
|
||||||
|
let backfillRuns = 0;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const Ctor = await loadTrackerCtor();
|
||||||
|
tracker = new Ctor({ dbPath }, {
|
||||||
|
runLexicalRollupBackfillTask: async () => {
|
||||||
|
backfillRuns += 1;
|
||||||
|
},
|
||||||
|
} as never);
|
||||||
|
|
||||||
|
assert.equal(backfillRuns, 0);
|
||||||
|
} finally {
|
||||||
|
tracker?.destroy();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('tracker starts the injected lexical rollup backfill when it is pending', async () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
let tracker: ImmersionTrackerService | null = null;
|
||||||
|
let backfillRuns = 0;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const setupDb = new Database(dbPath);
|
||||||
|
const { ensureSchema } = await import('./immersion-tracker/storage');
|
||||||
|
ensureSchema(setupDb);
|
||||||
|
setupDb
|
||||||
|
.prepare(
|
||||||
|
`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||||
|
)
|
||||||
|
.run();
|
||||||
|
setupDb.close();
|
||||||
|
|
||||||
|
const Ctor = await loadTrackerCtor();
|
||||||
|
tracker = new Ctor({ dbPath }, {
|
||||||
|
runLexicalRollupBackfillTask: async () => {
|
||||||
|
backfillRuns += 1;
|
||||||
|
},
|
||||||
|
} as never);
|
||||||
|
|
||||||
|
assert.equal(backfillRuns, 1);
|
||||||
|
await waitForCondition(
|
||||||
|
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
(tracker as unknown as { preserveWriteQueueUntilDrained: boolean })
|
||||||
|
.preserveWriteQueueUntilDrained,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
tracker?.destroy();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('tracker runs startup session-rollup maintenance before lexical backfill locks writes', async () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
let tracker: ImmersionTrackerService | null = null;
|
||||||
|
let releaseBackfill = (): void => {};
|
||||||
|
const heldBackfill = new Promise<void>((resolve) => {
|
||||||
|
releaseBackfill = resolve;
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const startedAtMs = trackerNowMs() - 60_000;
|
||||||
|
const endedAtMs = trackerNowMs();
|
||||||
|
const setupDb = new Database(dbPath);
|
||||||
|
const { ensureSchema } = await import('./immersion-tracker/storage');
|
||||||
|
ensureSchema(setupDb);
|
||||||
|
setupDb.exec(`
|
||||||
|
INSERT INTO imm_videos (
|
||||||
|
video_id, video_key, canonical_title, source_type, duration_ms, CREATED_DATE, LAST_UPDATE_DATE
|
||||||
|
) VALUES (1, 'local:/tmp/rollup-recovery.mkv', 'Rollup Recovery', 1, 0, '1', '1');
|
||||||
|
INSERT INTO imm_sessions (
|
||||||
|
session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status,
|
||||||
|
active_watched_ms, lines_seen, tokens_seen, cards_mined, CREATED_DATE, LAST_UPDATE_DATE
|
||||||
|
) VALUES (
|
||||||
|
1, 'rollup-recovery', 1, '${startedAtMs}', '${endedAtMs}', 2,
|
||||||
|
60000, 10, 20, 2, '${startedAtMs}', '${endedAtMs}'
|
||||||
|
);
|
||||||
|
INSERT INTO imm_session_telemetry (
|
||||||
|
session_id, sample_ms, total_watched_ms, active_watched_ms, lines_seen,
|
||||||
|
tokens_seen, cards_mined, lookup_count, lookup_hits, CREATED_DATE, LAST_UPDATE_DATE
|
||||||
|
) VALUES (
|
||||||
|
1, '${endedAtMs}', 60000, 60000, 10, 20, 2, 0, 0,
|
||||||
|
'${endedAtMs}', '${endedAtMs}'
|
||||||
|
);
|
||||||
|
DELETE FROM imm_daily_rollups;
|
||||||
|
DELETE FROM imm_monthly_rollups;
|
||||||
|
UPDATE imm_rollup_state SET state_value = '0';
|
||||||
|
`);
|
||||||
|
setupDb.close();
|
||||||
|
|
||||||
|
const Ctor = await loadTrackerCtor();
|
||||||
|
tracker = new Ctor({ dbPath }, {
|
||||||
|
runLexicalRollupBackfillTask: async () => heldBackfill,
|
||||||
|
} as never);
|
||||||
|
|
||||||
|
const privateApi = tracker as unknown as {
|
||||||
|
db: DatabaseSync;
|
||||||
|
writeLock: { locked: boolean };
|
||||||
|
};
|
||||||
|
assert.equal(privateApi.writeLock.locked, true);
|
||||||
|
assert.equal(
|
||||||
|
(
|
||||||
|
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_daily_rollups').get() as {
|
||||||
|
total: number;
|
||||||
|
}
|
||||||
|
).total,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
(
|
||||||
|
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_monthly_rollups').get() as {
|
||||||
|
total: number;
|
||||||
|
}
|
||||||
|
).total,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
releaseBackfill();
|
||||||
|
if (tracker) {
|
||||||
|
await waitForCondition(
|
||||||
|
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
tracker?.destroy();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('tracker queues playback writes until lexical rollup backfill settles', async () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
let tracker: ImmersionTrackerService | null = null;
|
||||||
|
let startBackfill = (): void => {};
|
||||||
|
let releaseBackfill = (): void => {};
|
||||||
|
let markBackfillStarted = (): void => {};
|
||||||
|
const backfillStartGate = new Promise<void>((resolve) => {
|
||||||
|
startBackfill = resolve;
|
||||||
|
});
|
||||||
|
const heldBackfill = new Promise<void>((resolve) => {
|
||||||
|
releaseBackfill = resolve;
|
||||||
|
});
|
||||||
|
const backfillStarted = new Promise<void>((resolve) => {
|
||||||
|
markBackfillStarted = resolve;
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const setupDb = new Database(dbPath);
|
||||||
|
const { ensureSchema } = await import('./immersion-tracker/storage');
|
||||||
|
ensureSchema(setupDb);
|
||||||
|
setupDb
|
||||||
|
.prepare(
|
||||||
|
`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||||
|
)
|
||||||
|
.run();
|
||||||
|
setupDb.close();
|
||||||
|
|
||||||
|
const Ctor = await loadTrackerCtor();
|
||||||
|
tracker = new Ctor(
|
||||||
|
{ dbPath, policy: { queueCap: 100 } },
|
||||||
|
{
|
||||||
|
runLexicalRollupBackfillTask: async (workerDbPath) => {
|
||||||
|
await backfillStartGate;
|
||||||
|
const workerDb = new Database(workerDbPath);
|
||||||
|
try {
|
||||||
|
workerDb.exec('BEGIN IMMEDIATE');
|
||||||
|
markBackfillStarted();
|
||||||
|
await heldBackfill;
|
||||||
|
workerDb.exec('COMMIT');
|
||||||
|
} catch (error) {
|
||||||
|
try {
|
||||||
|
workerDb.exec('ROLLBACK');
|
||||||
|
} catch {
|
||||||
|
// Preserve the original worker failure.
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
} finally {
|
||||||
|
workerDb.close();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
tracker.handleMediaChange('https://example.com/backfill-test.mp4', 'Backfill Test');
|
||||||
|
startBackfill();
|
||||||
|
await backfillStarted;
|
||||||
|
for (let index = 0; index < 125; index += 1) tracker.recordCardsMined(1);
|
||||||
|
|
||||||
|
const privateApi = tracker as unknown as {
|
||||||
|
db: DatabaseSync;
|
||||||
|
queue: unknown[];
|
||||||
|
droppedWriteCount: number;
|
||||||
|
flushNow: () => void;
|
||||||
|
writeLock: { locked: boolean };
|
||||||
|
};
|
||||||
|
assert.equal(privateApi.writeLock.locked, true);
|
||||||
|
privateApi.flushNow();
|
||||||
|
|
||||||
|
assert.ok(privateApi.queue.length > 100, 'the protected queue may grow past its normal cap');
|
||||||
|
assert.equal(privateApi.droppedWriteCount, 0, 'backfill must not discard playback writes');
|
||||||
|
assert.equal(
|
||||||
|
(
|
||||||
|
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
|
||||||
|
total: number;
|
||||||
|
}
|
||||||
|
).total,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
|
||||||
|
releaseBackfill();
|
||||||
|
await waitForCondition(() => privateApi.queue.length === 0, 5_000);
|
||||||
|
assert.equal(
|
||||||
|
(
|
||||||
|
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
|
||||||
|
total: number;
|
||||||
|
}
|
||||||
|
).total,
|
||||||
|
125,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
releaseBackfill();
|
||||||
|
if (tracker) {
|
||||||
|
await waitForCondition(
|
||||||
|
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
|
||||||
|
5_000,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
tracker?.destroy();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('startup backfills lifetime summaries when retained sessions exist but summary tables are empty', async () => {
|
test('startup backfills lifetime summaries when retained sessions exist but summary tables are empty', async () => {
|
||||||
const dbPath = makeDbPath();
|
const dbPath = makeDbPath();
|
||||||
let tracker: ImmersionTrackerService | null = null;
|
let tracker: ImmersionTrackerService | null = null;
|
||||||
@@ -4909,3 +5144,149 @@ test('ensureAnimeCoverArt fetches art via the latest video of the anime', async
|
|||||||
cleanupDbPath(dbPath);
|
cleanupDbPath(dbPath);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('getVocabularySummary coalesces concurrent requests into one worker task', async () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
let tracker: ImmersionTrackerService | null = null;
|
||||||
|
let taskRuns = 0;
|
||||||
|
let releaseTask: (() => void) | null = null;
|
||||||
|
const seenKnownWords: Array<ReadonlySet<string> | null> = [];
|
||||||
|
const summary = {
|
||||||
|
uniqueWords: 1,
|
||||||
|
uniqueWordsWithoutNames: 1,
|
||||||
|
uniqueKanji: 0,
|
||||||
|
newThisWeek: 0,
|
||||||
|
newThisWeekWithoutNames: 0,
|
||||||
|
knownWordCount: null,
|
||||||
|
knownWordCountWithoutNames: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const Ctor = await loadTrackerCtor();
|
||||||
|
tracker = new Ctor(
|
||||||
|
{ dbPath },
|
||||||
|
{
|
||||||
|
runVocabularySummaryTask: async (_dbPath, knownWords) => {
|
||||||
|
taskRuns += 1;
|
||||||
|
seenKnownWords.push(knownWords);
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
releaseTask = resolve;
|
||||||
|
});
|
||||||
|
return summary;
|
||||||
|
},
|
||||||
|
destroyVocabularySummaryRunner: () => {},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const knownWordsSnapshot = new Set(['猫']);
|
||||||
|
const first = tracker.getVocabularySummary(knownWordsSnapshot);
|
||||||
|
const second = tracker.getVocabularySummary(knownWordsSnapshot);
|
||||||
|
await waitForCondition(() => releaseTask !== null);
|
||||||
|
let release = releaseTask as (() => void) | null;
|
||||||
|
assert.ok(release);
|
||||||
|
release();
|
||||||
|
assert.deepEqual(await first, summary);
|
||||||
|
assert.equal(await second, await first);
|
||||||
|
assert.equal(taskRuns, 1);
|
||||||
|
assert.deepEqual(seenKnownWords, [knownWordsSnapshot]);
|
||||||
|
|
||||||
|
releaseTask = null;
|
||||||
|
const third = tracker.getVocabularySummary(null);
|
||||||
|
await waitForCondition(() => releaseTask !== null);
|
||||||
|
release = releaseTask as (() => void) | null;
|
||||||
|
assert.ok(release);
|
||||||
|
release();
|
||||||
|
assert.deepEqual(await third, summary);
|
||||||
|
assert.equal(taskRuns, 2);
|
||||||
|
} finally {
|
||||||
|
tracker?.destroy();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('getVocabularySummary coalesces equivalent known-word snapshots by value', async () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
let tracker: ImmersionTrackerService | null = null;
|
||||||
|
let taskRuns = 0;
|
||||||
|
const releases: Array<() => void> = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
const Ctor = await loadTrackerCtor();
|
||||||
|
tracker = new Ctor(
|
||||||
|
{ dbPath },
|
||||||
|
{
|
||||||
|
runVocabularySummaryTask: async () => {
|
||||||
|
taskRuns += 1;
|
||||||
|
await new Promise<void>((resolve) => releases.push(resolve));
|
||||||
|
return {
|
||||||
|
uniqueWords: 2,
|
||||||
|
uniqueWordsWithoutNames: 2,
|
||||||
|
uniqueKanji: 2,
|
||||||
|
newThisWeek: 0,
|
||||||
|
newThisWeekWithoutNames: 0,
|
||||||
|
knownWordCount: 2,
|
||||||
|
knownWordCountWithoutNames: 2,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
destroyVocabularySummaryRunner: () => {},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const first = tracker.getVocabularySummary(new Set(['猫', '犬']));
|
||||||
|
const second = tracker.getVocabularySummary(new Set(['犬', '猫']));
|
||||||
|
await waitForCondition(() => releases.length > 0);
|
||||||
|
const observedTaskRuns = taskRuns;
|
||||||
|
for (const release of releases) release();
|
||||||
|
await Promise.all([first, second]);
|
||||||
|
|
||||||
|
assert.equal(observedTaskRuns, 1);
|
||||||
|
|
||||||
|
const third = tracker.getVocabularySummary(new Set(['猫', '犬']));
|
||||||
|
await waitForCondition(() => releases.length === 2);
|
||||||
|
releases[1]!();
|
||||||
|
await third;
|
||||||
|
assert.equal(taskRuns, 2, 'a settled snapshot must be evicted from the in-flight map');
|
||||||
|
} finally {
|
||||||
|
tracker?.destroy();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('getVocabularySummary keeps different known-word snapshots independent', async () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
let tracker: ImmersionTrackerService | null = null;
|
||||||
|
const releases: Array<() => void> = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
const Ctor = await loadTrackerCtor();
|
||||||
|
tracker = new Ctor(
|
||||||
|
{ dbPath },
|
||||||
|
{
|
||||||
|
runVocabularySummaryTask: async (_dbPath, knownWords) => {
|
||||||
|
await new Promise<void>((resolve) => releases.push(resolve));
|
||||||
|
return {
|
||||||
|
uniqueWords: 1,
|
||||||
|
uniqueWordsWithoutNames: 1,
|
||||||
|
uniqueKanji: 0,
|
||||||
|
newThisWeek: 0,
|
||||||
|
newThisWeekWithoutNames: 0,
|
||||||
|
knownWordCount: knownWords?.size ?? null,
|
||||||
|
knownWordCountWithoutNames: knownWords?.size ?? null,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
destroyVocabularySummaryRunner: () => {},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const withoutKnownWords = tracker.getVocabularySummary(null);
|
||||||
|
const withKnownWords = tracker.getVocabularySummary(new Set(['猫']));
|
||||||
|
await waitForCondition(() => releases.length === 2);
|
||||||
|
for (const release of releases) release();
|
||||||
|
|
||||||
|
assert.equal((await withoutKnownWords).knownWordCount, null);
|
||||||
|
assert.equal((await withKnownWords).knownWordCount, 1);
|
||||||
|
} finally {
|
||||||
|
tracker?.destroy();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ import {
|
|||||||
getSessionEvents,
|
getSessionEvents,
|
||||||
getSimilarWords,
|
getSimilarWords,
|
||||||
getStatsExcludedWords,
|
getStatsExcludedWords,
|
||||||
|
getVocabularyChartData,
|
||||||
getVocabularyStats,
|
getVocabularyStats,
|
||||||
replaceStatsExcludedWords,
|
replaceStatsExcludedWords,
|
||||||
searchSubtitleSentences,
|
searchSubtitleSentences,
|
||||||
@@ -96,6 +97,12 @@ import {
|
|||||||
DeleteMaintenanceWorkerRuntime,
|
DeleteMaintenanceWorkerRuntime,
|
||||||
type RunDeleteMaintenanceTask,
|
type RunDeleteMaintenanceTask,
|
||||||
} from './immersion-tracker/delete-maintenance-worker-runtime';
|
} from './immersion-tracker/delete-maintenance-worker-runtime';
|
||||||
|
import {
|
||||||
|
VocabularySummaryWorkerRuntime,
|
||||||
|
type RunVocabularySummaryTask,
|
||||||
|
} from './immersion-tracker/vocabulary-summary-worker-runtime';
|
||||||
|
import { LexicalRollupWorkerRuntime } from './immersion-tracker/lexical-rollup-worker-runtime';
|
||||||
|
import { areLexicalDailyRollupsReady } from './immersion-tracker/lexical-rollups';
|
||||||
import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
|
import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
|
||||||
import {
|
import {
|
||||||
cleanupDuplicateSubtitleLines,
|
cleanupDuplicateSubtitleLines,
|
||||||
@@ -185,6 +192,7 @@ import {
|
|||||||
type StatsExcludedWordRow,
|
type StatsExcludedWordRow,
|
||||||
type StreakCalendarRow,
|
type StreakCalendarRow,
|
||||||
type VocabularyCleanupSummary,
|
type VocabularyCleanupSummary,
|
||||||
|
type VocabularyStatsSummary,
|
||||||
type WatchTimePerAnimeRow,
|
type WatchTimePerAnimeRow,
|
||||||
type WordAnimeAppearanceRow,
|
type WordAnimeAppearanceRow,
|
||||||
type WordDetailRow,
|
type WordDetailRow,
|
||||||
@@ -405,13 +413,24 @@ export class ImmersionTrackerService {
|
|||||||
private readonly monthlyRollupRetentionMs: number;
|
private readonly monthlyRollupRetentionMs: number;
|
||||||
private readonly vacuumIntervalMs: number;
|
private readonly vacuumIntervalMs: number;
|
||||||
private readonly dbPath: string;
|
private readonly dbPath: string;
|
||||||
private readonly writeLock = { locked: false };
|
private readonly writeLock = {
|
||||||
|
locked: false,
|
||||||
|
reasons: new Set<'flush' | 'delete-maintenance' | 'lexical-rollup-backfill'>(),
|
||||||
|
};
|
||||||
private readonly destroyDeleteMaintenanceRunner: () => void;
|
private readonly destroyDeleteMaintenanceRunner: () => void;
|
||||||
|
private readonly runVocabularySummaryTask: (
|
||||||
|
knownWords: ReadonlySet<string> | null,
|
||||||
|
) => Promise<VocabularyStatsSummary>;
|
||||||
|
private readonly vocabularySummariesInFlight = new Map<string, Promise<VocabularyStatsSummary>>();
|
||||||
|
private readonly destroyVocabularySummaryRunner: () => void;
|
||||||
|
private readonly runLexicalRollupBackfillTask: () => Promise<void>;
|
||||||
|
private readonly destroyLexicalRollupBackfillRunner: () => void;
|
||||||
private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
|
private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
|
||||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
|
private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
private flushScheduled = false;
|
private flushScheduled = false;
|
||||||
private droppedWriteCount = 0;
|
private droppedWriteCount = 0;
|
||||||
|
private preserveWriteQueueUntilDrained = false;
|
||||||
private lastVacuumMs = 0;
|
private lastVacuumMs = 0;
|
||||||
private isDestroyed = false;
|
private isDestroyed = false;
|
||||||
private sessionState: SessionState | null = null;
|
private sessionState: SessionState | null = null;
|
||||||
@@ -434,6 +453,10 @@ export class ImmersionTrackerService {
|
|||||||
dependencies: {
|
dependencies: {
|
||||||
runDeleteMaintenanceTask?: RunDeleteMaintenanceTask;
|
runDeleteMaintenanceTask?: RunDeleteMaintenanceTask;
|
||||||
destroyDeleteMaintenanceRunner?: () => void;
|
destroyDeleteMaintenanceRunner?: () => void;
|
||||||
|
runVocabularySummaryTask?: RunVocabularySummaryTask;
|
||||||
|
destroyVocabularySummaryRunner?: () => void;
|
||||||
|
runLexicalRollupBackfillTask?: (dbPath: string) => Promise<void>;
|
||||||
|
destroyLexicalRollupBackfillRunner?: () => void;
|
||||||
} = {},
|
} = {},
|
||||||
) {
|
) {
|
||||||
this.dbPath = options.dbPath;
|
this.dbPath = options.dbPath;
|
||||||
@@ -453,13 +476,34 @@ export class ImmersionTrackerService {
|
|||||||
runTask: (task) => runDeleteMaintenanceTask(this.dbPath, task),
|
runTask: (task) => runDeleteMaintenanceTask(this.dbPath, task),
|
||||||
onBusy: () => {
|
onBusy: () => {
|
||||||
this.requireWriteQueueDrained('delete maintenance');
|
this.requireWriteQueueDrained('delete maintenance');
|
||||||
this.writeLock.locked = true;
|
this.setWriteLock('delete-maintenance', true);
|
||||||
},
|
},
|
||||||
onIdle: () => {
|
onIdle: () => {
|
||||||
this.writeLock.locked = false;
|
this.setWriteLock('delete-maintenance', false);
|
||||||
if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
|
if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
if (dependencies.runVocabularySummaryTask) {
|
||||||
|
this.runVocabularySummaryTask = (knownWords) =>
|
||||||
|
dependencies.runVocabularySummaryTask!(this.dbPath, knownWords);
|
||||||
|
this.destroyVocabularySummaryRunner =
|
||||||
|
dependencies.destroyVocabularySummaryRunner ?? (() => {});
|
||||||
|
} else {
|
||||||
|
const vocabularySummaryRuntime = new VocabularySummaryWorkerRuntime();
|
||||||
|
this.runVocabularySummaryTask = (knownWords) =>
|
||||||
|
vocabularySummaryRuntime.run(this.dbPath, knownWords);
|
||||||
|
this.destroyVocabularySummaryRunner = () => vocabularySummaryRuntime.destroy();
|
||||||
|
}
|
||||||
|
if (dependencies.runLexicalRollupBackfillTask) {
|
||||||
|
this.runLexicalRollupBackfillTask = () =>
|
||||||
|
dependencies.runLexicalRollupBackfillTask!(this.dbPath);
|
||||||
|
this.destroyLexicalRollupBackfillRunner =
|
||||||
|
dependencies.destroyLexicalRollupBackfillRunner ?? (() => {});
|
||||||
|
} else {
|
||||||
|
const lexicalRollupRuntime = new LexicalRollupWorkerRuntime();
|
||||||
|
this.runLexicalRollupBackfillTask = () => lexicalRollupRuntime.run(this.dbPath);
|
||||||
|
this.destroyLexicalRollupBackfillRunner = () => lexicalRollupRuntime.destroy();
|
||||||
|
}
|
||||||
const parentDir = path.dirname(this.dbPath);
|
const parentDir = path.dirname(this.dbPath);
|
||||||
if (!fs.existsSync(parentDir)) {
|
if (!fs.existsSync(parentDir)) {
|
||||||
fs.mkdirSync(parentDir, { recursive: true });
|
fs.mkdirSync(parentDir, { recursive: true });
|
||||||
@@ -548,6 +592,7 @@ export class ImmersionTrackerService {
|
|||||||
}
|
}
|
||||||
this.preparedStatements = createTrackerPreparedStatements(this.db);
|
this.preparedStatements = createTrackerPreparedStatements(this.db);
|
||||||
this.scheduleMaintenance();
|
this.scheduleMaintenance();
|
||||||
|
if (!areLexicalDailyRollupsReady(this.db)) this.startLexicalRollupBackfill();
|
||||||
this.scheduleFlush();
|
this.scheduleFlush();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -565,6 +610,8 @@ export class ImmersionTrackerService {
|
|||||||
this.isDestroyed = true;
|
this.isDestroyed = true;
|
||||||
this.deleteMaintenanceScheduler.destroy();
|
this.deleteMaintenanceScheduler.destroy();
|
||||||
this.destroyDeleteMaintenanceRunner();
|
this.destroyDeleteMaintenanceRunner();
|
||||||
|
this.destroyVocabularySummaryRunner();
|
||||||
|
this.destroyLexicalRollupBackfillRunner();
|
||||||
this.db.close();
|
this.db.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -634,6 +681,25 @@ export class ImmersionTrackerService {
|
|||||||
return getVocabularyStats(this.db, limit, excludePos);
|
return getVocabularyStats(this.db, limit, excludePos);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async getVocabularySummary(knownWords: ReadonlySet<string> | null) {
|
||||||
|
const key = knownWords ? JSON.stringify([...knownWords].sort()) : 'null';
|
||||||
|
const inFlight = this.vocabularySummariesInFlight.get(key);
|
||||||
|
if (inFlight) return inFlight;
|
||||||
|
const task = this.runVocabularySummaryTask(knownWords);
|
||||||
|
this.vocabularySummariesInFlight.set(key, task);
|
||||||
|
try {
|
||||||
|
return await task;
|
||||||
|
} finally {
|
||||||
|
if (this.vocabularySummariesInFlight.get(key) === task) {
|
||||||
|
this.vocabularySummariesInFlight.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async getVocabularyChartData() {
|
||||||
|
return getVocabularyChartData(this.db);
|
||||||
|
}
|
||||||
|
|
||||||
async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {
|
async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {
|
||||||
return getStatsExcludedWords(this.db);
|
return getStatsExcludedWords(this.db);
|
||||||
}
|
}
|
||||||
@@ -910,6 +976,33 @@ export class ImmersionTrackerService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private setWriteLock(
|
||||||
|
reason: 'flush' | 'delete-maintenance' | 'lexical-rollup-backfill',
|
||||||
|
active: boolean,
|
||||||
|
): void {
|
||||||
|
if (active) this.writeLock.reasons.add(reason);
|
||||||
|
else this.writeLock.reasons.delete(reason);
|
||||||
|
this.writeLock.locked = this.writeLock.reasons.size > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private startLexicalRollupBackfill(): void {
|
||||||
|
this.requireWriteQueueDrained('lexical rollup backfill');
|
||||||
|
this.preserveWriteQueueUntilDrained = true;
|
||||||
|
this.setWriteLock('lexical-rollup-backfill', true);
|
||||||
|
void this.runLexicalRollupBackfillTask()
|
||||||
|
.catch((error: unknown) => {
|
||||||
|
this.logger.warn(
|
||||||
|
'Lexical daily rollup backfill failed; it will retry on next startup',
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
this.setWriteLock('lexical-rollup-backfill', false);
|
||||||
|
if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
|
||||||
|
else if (!this.isDestroyed) this.scheduleFlush(0);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async reassignAnimeAnilist(
|
async reassignAnimeAnilist(
|
||||||
animeId: number,
|
animeId: number,
|
||||||
info: {
|
info: {
|
||||||
@@ -1906,7 +1999,12 @@ export class ImmersionTrackerService {
|
|||||||
|
|
||||||
private recordWrite(write: QueuedWrite): void {
|
private recordWrite(write: QueuedWrite): void {
|
||||||
if (this.isDestroyed) return;
|
if (this.isDestroyed) return;
|
||||||
const { dropped } = enqueueWrite(this.queue, write, this.queueCap);
|
// A lexical migration owns the database write lock, so dropping the oldest
|
||||||
|
// entry cannot relieve pressure: nothing can flush until the worker exits.
|
||||||
|
// Preserve that finite startup burst and drain it as soon as the lock lifts.
|
||||||
|
const { dropped } = this.preserveWriteQueueUntilDrained
|
||||||
|
? (this.queue.push(write), { dropped: 0 })
|
||||||
|
: enqueueWrite(this.queue, write, this.queueCap);
|
||||||
if (dropped > 0) {
|
if (dropped > 0) {
|
||||||
this.droppedWriteCount += dropped;
|
this.droppedWriteCount += dropped;
|
||||||
this.logger.warn(`Immersion tracker queue overflow; dropped ${dropped} oldest writes`);
|
this.logger.warn(`Immersion tracker queue overflow; dropped ${dropped} oldest writes`);
|
||||||
@@ -1954,6 +2052,7 @@ export class ImmersionTrackerService {
|
|||||||
private flushNow(): void {
|
private flushNow(): void {
|
||||||
if (this.writeLock.locked || this.isDestroyed) return;
|
if (this.writeLock.locked || this.isDestroyed) return;
|
||||||
if (this.queue.length === 0) {
|
if (this.queue.length === 0) {
|
||||||
|
this.preserveWriteQueueUntilDrained = false;
|
||||||
this.flushScheduled = false;
|
this.flushScheduled = false;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1965,7 +2064,7 @@ export class ImmersionTrackerService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const batch = this.queue.splice(0, Math.min(this.batchSize, this.queue.length));
|
const batch = this.queue.splice(0, Math.min(this.batchSize, this.queue.length));
|
||||||
this.writeLock.locked = true;
|
this.setWriteLock('flush', true);
|
||||||
try {
|
try {
|
||||||
this.db.exec('BEGIN IMMEDIATE');
|
this.db.exec('BEGIN IMMEDIATE');
|
||||||
for (const write of batch) {
|
for (const write of batch) {
|
||||||
@@ -1977,8 +2076,9 @@ export class ImmersionTrackerService {
|
|||||||
this.queue.unshift(...batch);
|
this.queue.unshift(...batch);
|
||||||
this.logger.warn('Immersion tracker flush failed, retrying later', error as Error);
|
this.logger.warn('Immersion tracker flush failed, retrying later', error as Error);
|
||||||
} finally {
|
} finally {
|
||||||
this.writeLock.locked = false;
|
this.setWriteLock('flush', false);
|
||||||
this.flushScheduled = false;
|
this.flushScheduled = false;
|
||||||
|
if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
|
||||||
if (this.queue.length > 0) {
|
if (this.queue.length > 0) {
|
||||||
this.scheduleFlush(this.flushIntervalMs);
|
this.scheduleFlush(this.flushIntervalMs);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,4 @@
|
|||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
import fs from 'node:fs';
|
|
||||||
import os from 'node:os';
|
|
||||||
import path from 'node:path';
|
|
||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
import { Database } from '../sqlite.js';
|
import { Database } from '../sqlite.js';
|
||||||
import type { DatabaseSync } from '../sqlite.js';
|
import type { DatabaseSync } from '../sqlite.js';
|
||||||
@@ -21,17 +18,6 @@ interface SeedLine {
|
|||||||
createdMs?: number;
|
createdMs?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
function makeDbPath(): string {
|
|
||||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-duplicate-line-test-'));
|
|
||||||
return path.join(dir, 'immersion.sqlite');
|
|
||||||
}
|
|
||||||
|
|
||||||
function cleanupDbPath(dbPath: string): void {
|
|
||||||
const dir = path.dirname(dbPath);
|
|
||||||
if (!fs.existsSync(dir)) return;
|
|
||||||
fs.rmSync(dir, { recursive: true, force: true });
|
|
||||||
}
|
|
||||||
|
|
||||||
/** One episode, two sessions of it, and one word occurrence per seeded line. */
|
/** One episode, two sessions of it, and one word occurrence per seeded line. */
|
||||||
function seed(db: DatabaseSync, lines: SeedLine[]): void {
|
function seed(db: DatabaseSync, lines: SeedLine[]): void {
|
||||||
db.exec(`
|
db.exec(`
|
||||||
@@ -82,12 +68,16 @@ function seed(db: DatabaseSync, lines: SeedLine[]): void {
|
|||||||
`);
|
`);
|
||||||
}
|
}
|
||||||
|
|
||||||
function createDb(lines: SeedLine[]): { db: DatabaseSync; dbPath: string } {
|
/**
|
||||||
const dbPath = makeDbPath();
|
* These tests exercise the cleanup SQL, not durability. A fresh on-disk database per
|
||||||
const db = new Database(dbPath);
|
* test pays a schema-creation fsync that is cheap on a local NVMe but slow enough on CI
|
||||||
|
* runners to blow the 5s per-test timeout, so the database stays in memory.
|
||||||
|
*/
|
||||||
|
function createDb(lines: SeedLine[]): { db: DatabaseSync } {
|
||||||
|
const db = new Database(':memory:');
|
||||||
ensureSchema(db);
|
ensureSchema(db);
|
||||||
seed(db, lines);
|
seed(db, lines);
|
||||||
return { db, dbPath };
|
return { db };
|
||||||
}
|
}
|
||||||
|
|
||||||
/** A typeset line mpv reported once per animation frame. */
|
/** A typeset line mpv reported once per animation frame. */
|
||||||
@@ -119,7 +109,7 @@ function wordFrequency(db: DatabaseSync): number {
|
|||||||
}
|
}
|
||||||
|
|
||||||
test('a karaoke burst collapses to one line and gives back its word counts', () => {
|
test('a karaoke burst collapses to one line and gives back its word counts', () => {
|
||||||
const { db, dbPath } = createDb([
|
const { db } = createDb([
|
||||||
...karaokeFrames(1, '飛び上がる', 10_000, 40, 40),
|
...karaokeFrames(1, '飛び上がる', 10_000, 40, 40),
|
||||||
{ session: 1, text: 'おはよう', startMs: 20_000, endMs: 22_000 },
|
{ session: 1, text: 'おはよう', startMs: 20_000, endMs: 22_000 },
|
||||||
]);
|
]);
|
||||||
@@ -148,7 +138,6 @@ test('a karaoke burst collapses to one line and gives back its word counts', ()
|
|||||||
assert.equal(summary.samples[0]!.videoTitle, 'Ep 1');
|
assert.equal(summary.samples[0]!.videoTitle, 'Ep 1');
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -160,7 +149,7 @@ test('ordinary repeated dialogue survives', () => {
|
|||||||
startMs: 5_000 + index * 800,
|
startMs: 5_000 + index * 800,
|
||||||
endMs: 5_000 + (index + 1) * 800,
|
endMs: 5_000 + (index + 1) * 800,
|
||||||
}));
|
}));
|
||||||
const { db, dbPath } = createDb(lines);
|
const { db } = createDb(lines);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const summary = cleanupDuplicateSubtitleLines(db);
|
const summary = cleanupDuplicateSubtitleLines(db);
|
||||||
@@ -171,14 +160,13 @@ test('ordinary repeated dialogue survives', () => {
|
|||||||
assert.equal(wordFrequency(db), 6);
|
assert.equal(wordFrequency(db), 6);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('a long run of quarter-second frames is still a burst', () => {
|
test('a long run of quarter-second frames is still a burst', () => {
|
||||||
// Between the timing-only bound (0.1s) and the animation-frame bound (0.3s): heavier
|
// Between the timing-only bound (0.1s) and the animation-frame bound (0.3s): heavier
|
||||||
// typesetting lands here, and the run length is what makes it conclusive.
|
// typesetting lands here, and the run length is what makes it conclusive.
|
||||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
|
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const summary = cleanupDuplicateSubtitleLines(db);
|
const summary = cleanupDuplicateSubtitleLines(db);
|
||||||
@@ -189,12 +177,11 @@ test('a long run of quarter-second frames is still a burst', () => {
|
|||||||
assert.equal(wordFrequency(db), 1);
|
assert.equal(wordFrequency(db), 1);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('a qualifying short-frame burst may end with one long hold frame', () => {
|
test('a qualifying short-frame burst may end with one long hold frame', () => {
|
||||||
const { db, dbPath } = createDb([
|
const { db } = createDb([
|
||||||
...karaokeFrames(1, '飛び上がる', 10_000, 8, 40),
|
...karaokeFrames(1, '飛び上がる', 10_000, 8, 40),
|
||||||
{ session: 1, text: '飛び上がる', startMs: 10_320, endMs: 12_320 },
|
{ session: 1, text: '飛び上がる', startMs: 10_320, endMs: 12_320 },
|
||||||
]);
|
]);
|
||||||
@@ -208,12 +195,11 @@ test('a qualifying short-frame burst may end with one long hold frame', () => {
|
|||||||
assert.equal(wordFrequency(db), 1);
|
assert.equal(wordFrequency(db), 1);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('a long event before the final frame prevents burst cleanup', () => {
|
test('a long event before the final frame prevents burst cleanup', () => {
|
||||||
const { db, dbPath } = createDb([
|
const { db } = createDb([
|
||||||
...karaokeFrames(1, '飛び上がる', 10_000, 5, 40),
|
...karaokeFrames(1, '飛び上がる', 10_000, 5, 40),
|
||||||
{ session: 1, text: '飛び上がる', startMs: 10_200, endMs: 12_200 },
|
{ session: 1, text: '飛び上がる', startMs: 10_200, endMs: 12_200 },
|
||||||
{ session: 1, text: '飛び上がる', startMs: 12_200, endMs: 12_240 },
|
{ session: 1, text: '飛び上がる', startMs: 12_200, endMs: 12_240 },
|
||||||
@@ -226,12 +212,11 @@ test('a long event before the final frame prevents burst cleanup', () => {
|
|||||||
assert.equal(countLines(db), 7);
|
assert.equal(countLines(db), 7);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('a run of frames longer than the animation bound survives', () => {
|
test('a run of frames longer than the animation bound survives', () => {
|
||||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 400));
|
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 400));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const summary = cleanupDuplicateSubtitleLines(db);
|
const summary = cleanupDuplicateSubtitleLines(db);
|
||||||
@@ -240,7 +225,6 @@ test('a run of frames longer than the animation bound survives', () => {
|
|||||||
assert.equal(countLines(db), 6);
|
assert.equal(countLines(db), 6);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -248,7 +232,7 @@ test('the four-frame residue the live gate stores is cleaned up', () => {
|
|||||||
// The streaming gate records the first four frames of a burst before the run is long
|
// The streaming gate records the first four frames of a burst before the run is long
|
||||||
// enough to recognise. Four contiguous identical events under the strict timing-only
|
// enough to recognise. Four contiguous identical events under the strict timing-only
|
||||||
// bound are that residue, and no real dialogue.
|
// bound are that residue, and no real dialogue.
|
||||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 40));
|
const { db } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 40));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const summary = cleanupDuplicateSubtitleLines(db);
|
const summary = cleanupDuplicateSubtitleLines(db);
|
||||||
@@ -259,14 +243,13 @@ test('the four-frame residue the live gate stores is cleaned up', () => {
|
|||||||
assert.equal(wordFrequency(db), 1);
|
assert.equal(wordFrequency(db), 1);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('a four-frame run above the strict frame bound survives', () => {
|
test('a four-frame run above the strict frame bound survives', () => {
|
||||||
// Long enough per event to be plausible dialogue; only a five-event run may use the
|
// Long enough per event to be plausible dialogue; only a five-event run may use the
|
||||||
// looser animation-frame bound.
|
// looser animation-frame bound.
|
||||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 250));
|
const { db } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 250));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const summary = cleanupDuplicateSubtitleLines(db);
|
const summary = cleanupDuplicateSubtitleLines(db);
|
||||||
@@ -275,14 +258,13 @@ test('a four-frame run above the strict frame bound survives', () => {
|
|||||||
assert.equal(countLines(db), 4);
|
assert.equal(countLines(db), 4);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('an explicit minRunLength raises the bar', () => {
|
test('an explicit minRunLength raises the bar', () => {
|
||||||
// Five quarter-second frames qualify under the defaults; a cautious run asking for six
|
// Five quarter-second frames qualify under the defaults; a cautious run asking for six
|
||||||
// leaves them alone. Above the strict bound, so the residue rule stays out of it.
|
// leaves them alone. Above the strict bound, so the residue rule stays out of it.
|
||||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 5, 250));
|
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 5, 250));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
|
const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
|
||||||
@@ -293,12 +275,11 @@ test('an explicit minRunLength raises the bar', () => {
|
|||||||
assert.equal(countLines(db), 5);
|
assert.equal(countLines(db), 5);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('an explicit maxFrameSeconds tightens the frame bound', () => {
|
test('an explicit maxFrameSeconds tightens the frame bound', () => {
|
||||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
|
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const summary = cleanupDuplicateSubtitleLines(db, { maxFrameSeconds: 0.2 });
|
const summary = cleanupDuplicateSubtitleLines(db, { maxFrameSeconds: 0.2 });
|
||||||
@@ -307,13 +288,12 @@ test('an explicit maxFrameSeconds tightens the frame bound', () => {
|
|||||||
assert.equal(countLines(db), 6);
|
assert.equal(countLines(db), 6);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('a non-finite maxFrameSeconds falls back to the default bound', () => {
|
test('a non-finite maxFrameSeconds falls back to the default bound', () => {
|
||||||
// Six normal-beat lines: Infinity must not turn every event into a "short frame".
|
// Six normal-beat lines: Infinity must not turn every event into a "short frame".
|
||||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 800));
|
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 800));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const summary = cleanupDuplicateSubtitleLines(db, { maxFrameSeconds: Infinity });
|
const summary = cleanupDuplicateSubtitleLines(db, { maxFrameSeconds: Infinity });
|
||||||
@@ -322,12 +302,11 @@ test('a non-finite maxFrameSeconds falls back to the default bound', () => {
|
|||||||
assert.equal(countLines(db), 6);
|
assert.equal(countLines(db), 6);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('sampleLimit zero removes bursts but reports no samples', () => {
|
test('sampleLimit zero removes bursts but reports no samples', () => {
|
||||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
|
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const summary = cleanupDuplicateSubtitleLines(db, { sampleLimit: 0 });
|
const summary = cleanupDuplicateSubtitleLines(db, { sampleLimit: 0 });
|
||||||
@@ -337,12 +316,11 @@ test('sampleLimit zero removes bursts but reports no samples', () => {
|
|||||||
assert.equal(countLines(db), 1);
|
assert.equal(countLines(db), 1);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('a short run below every threshold survives', () => {
|
test('a short run below every threshold survives', () => {
|
||||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 3, 40));
|
const { db } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 3, 40));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const summary = cleanupDuplicateSubtitleLines(db);
|
const summary = cleanupDuplicateSubtitleLines(db);
|
||||||
@@ -351,7 +329,6 @@ test('a short run below every threshold survives', () => {
|
|||||||
assert.equal(countLines(db), 3);
|
assert.equal(countLines(db), 3);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -361,7 +338,7 @@ test('interleaved dual-line karaoke collapses each line to one row', () => {
|
|||||||
const kanji = karaokeFrames(1, '飛び上がる', 10_000, 20, 60);
|
const kanji = karaokeFrames(1, '飛び上がる', 10_000, 20, 60);
|
||||||
const romaji = karaokeFrames(1, 'tobiagaru', 10_001, 20, 60);
|
const romaji = karaokeFrames(1, 'tobiagaru', 10_001, 20, 60);
|
||||||
const interleaved = [...kanji, ...romaji].sort((a, b) => a.startMs - b.startMs);
|
const interleaved = [...kanji, ...romaji].sort((a, b) => a.startMs - b.startMs);
|
||||||
const { db, dbPath } = createDb(interleaved);
|
const { db } = createDb(interleaved);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const summary = cleanupDuplicateSubtitleLines(db);
|
const summary = cleanupDuplicateSubtitleLines(db);
|
||||||
@@ -372,12 +349,11 @@ test('interleaved dual-line karaoke collapses each line to one row', () => {
|
|||||||
assert.equal(wordFrequency(db), 2);
|
assert.equal(wordFrequency(db), 2);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('the same line in a rewatch session is never merged into the first watch', () => {
|
test('the same line in a rewatch session is never merged into the first watch', () => {
|
||||||
const { db, dbPath } = createDb([
|
const { db } = createDb([
|
||||||
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
|
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
|
||||||
...karaokeFrames(2, '飛び上がる', 10_000, 6, 40),
|
...karaokeFrames(2, '飛び上がる', 10_000, 6, 40),
|
||||||
]);
|
]);
|
||||||
@@ -392,12 +368,11 @@ test('the same line in a rewatch session is never merged into the first watch',
|
|||||||
assert.equal(wordFrequency(db), 2);
|
assert.equal(wordFrequency(db), 2);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('a gap between runs splits them', () => {
|
test('a gap between runs splits them', () => {
|
||||||
const { db, dbPath } = createDb([
|
const { db } = createDb([
|
||||||
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
|
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
|
||||||
...karaokeFrames(1, '飛び上がる', 60_000, 6, 40),
|
...karaokeFrames(1, '飛び上がる', 60_000, 6, 40),
|
||||||
]);
|
]);
|
||||||
@@ -409,12 +384,11 @@ test('a gap between runs splits them', () => {
|
|||||||
assert.equal(countLines(db), 2);
|
assert.equal(countLines(db), 2);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('a dry run reports what an apply would do and writes nothing', () => {
|
test('a dry run reports what an apply would do and writes nothing', () => {
|
||||||
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
|
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
|
const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
|
||||||
@@ -430,14 +404,13 @@ test('a dry run reports what an apply would do and writes nothing', () => {
|
|||||||
assert.equal(countLines(db), 1);
|
assert.equal(countLines(db), 1);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('the lookback window leaves older bursts alone', () => {
|
test('the lookback window leaves older bursts alone', () => {
|
||||||
const recentMs = BASE_MS;
|
const recentMs = BASE_MS;
|
||||||
const oldMs = BASE_MS - 40 * DAY_MS;
|
const oldMs = BASE_MS - 40 * DAY_MS;
|
||||||
const { db, dbPath } = createDb([
|
const { db } = createDb([
|
||||||
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40).map((line) => ({
|
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40).map((line) => ({
|
||||||
...line,
|
...line,
|
||||||
createdMs: oldMs,
|
createdMs: oldMs,
|
||||||
@@ -462,6 +435,5 @@ test('the lookback window leaves older bursts alone', () => {
|
|||||||
} finally {
|
} finally {
|
||||||
globalThis.__subminerTestNowMs = undefined;
|
globalThis.__subminerTestNowMs = undefined;
|
||||||
db.close();
|
db.close();
|
||||||
cleanupDbPath(dbPath);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ import {
|
|||||||
getKanjiOccurrences,
|
getKanjiOccurrences,
|
||||||
getSessionSummaries,
|
getSessionSummaries,
|
||||||
getVocabularyStats,
|
getVocabularyStats,
|
||||||
|
getVocabularySummary,
|
||||||
getKanjiStats,
|
getKanjiStats,
|
||||||
getSessionEvents,
|
getSessionEvents,
|
||||||
getSessionTimeline,
|
getSessionTimeline,
|
||||||
@@ -1875,6 +1876,115 @@ test('getVocabularyStats returns rows ordered by frequency descending', () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('getVocabularySummary counts every tracked vocabulary row instead of a display page', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = openTestDb(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const nowSec = Math.floor(Date.now() / 1000);
|
||||||
|
const insertWord = db.prepare(`
|
||||||
|
INSERT INTO imm_words (
|
||||||
|
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||||
|
first_seen, last_seen, frequency
|
||||||
|
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', ?, ?, 1)
|
||||||
|
`);
|
||||||
|
const insertKanji = db.prepare(`
|
||||||
|
INSERT INTO imm_kanji (kanji, first_seen, last_seen, frequency)
|
||||||
|
VALUES (?, ?, ?, 1)
|
||||||
|
`);
|
||||||
|
|
||||||
|
for (let index = 0; index < 501; index += 1) {
|
||||||
|
insertWord.run(`単語${index}`, `単語${index}`, nowSec - 8 * 86_400, nowSec - 8 * 86_400);
|
||||||
|
}
|
||||||
|
for (let index = 0; index < 201; index += 1) {
|
||||||
|
insertKanji.run(
|
||||||
|
String.fromCodePoint(0x4e00 + index),
|
||||||
|
nowSec - 8 * 86_400,
|
||||||
|
nowSec - 8 * 86_400,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
insertWord.run('今週', '今週', nowSec - 86_400, nowSec - 86_400);
|
||||||
|
|
||||||
|
assert.deepEqual(getVocabularySummary(db, new Set(['単語0', '今週']), nowSec * 1000), {
|
||||||
|
uniqueWords: 502,
|
||||||
|
uniqueWordsWithoutNames: 502,
|
||||||
|
uniqueKanji: 201,
|
||||||
|
newThisWeek: 1,
|
||||||
|
newThisWeekWithoutNames: 1,
|
||||||
|
knownWordCount: 2,
|
||||||
|
knownWordCountWithoutNames: 2,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('getVocabularySummary applies vocabulary exclusions and Hide Names totals', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = openTestDb(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const insertWord = db.prepare(`
|
||||||
|
INSERT INTO imm_words (
|
||||||
|
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||||
|
first_seen, last_seen, frequency
|
||||||
|
) VALUES (?, ?, '', 'noun', '名詞', ?, '', 1, 1, 1)
|
||||||
|
`);
|
||||||
|
insertWord.run('猫', '猫', '一般');
|
||||||
|
insertWord.run('太郎', '太郎', '固有名詞');
|
||||||
|
insertWord.run('東京', '東京都', '一般');
|
||||||
|
db.prepare(
|
||||||
|
`
|
||||||
|
INSERT INTO imm_stats_excluded_words (headword, word, reading)
|
||||||
|
VALUES ('東京', '東京', '')
|
||||||
|
`,
|
||||||
|
).run();
|
||||||
|
|
||||||
|
assert.deepEqual(getVocabularySummary(db, new Set(['猫', '太郎', '東京']), 9 * 86_400_000), {
|
||||||
|
uniqueWords: 2,
|
||||||
|
uniqueWordsWithoutNames: 1,
|
||||||
|
uniqueKanji: 0,
|
||||||
|
newThisWeek: 0,
|
||||||
|
newThisWeekWithoutNames: 0,
|
||||||
|
knownWordCount: 2,
|
||||||
|
knownWordCountWithoutNames: 1,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('getVocabularySummary counts identically across id-keyed scan batches', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = openTestDb(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const insertWord = db.prepare(`
|
||||||
|
INSERT INTO imm_words (
|
||||||
|
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||||
|
first_seen, last_seen, frequency
|
||||||
|
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', 1, 1, 1)
|
||||||
|
`);
|
||||||
|
for (let index = 0; index < 5; index += 1) {
|
||||||
|
insertWord.run(`単語${index}`, `単語${index}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const fullScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000);
|
||||||
|
const batchedScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000, 2);
|
||||||
|
|
||||||
|
assert.equal(fullScan.uniqueWords, 5);
|
||||||
|
assert.deepEqual(batchedScan, fullScan);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () => {
|
test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () => {
|
||||||
const dbPath = makeDbPath();
|
const dbPath = makeDbPath();
|
||||||
const db = openTestDb(dbPath);
|
const db = openTestDb(dbPath);
|
||||||
|
|||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import test from 'node:test';
|
||||||
|
import {
|
||||||
|
LexicalRollupWorkerRuntime,
|
||||||
|
resolveLexicalRollupWorkerPath,
|
||||||
|
} from './lexical-rollup-worker-runtime';
|
||||||
|
import { areLexicalDailyRollupsReady } from './lexical-rollups';
|
||||||
|
import { Database } from './sqlite';
|
||||||
|
import { applyPragmas, ensureSchema } from './storage';
|
||||||
|
|
||||||
|
test('lexical rollup worker backfills without using the tracker connection', async () => {
|
||||||
|
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-runtime-'));
|
||||||
|
const dbPath = path.join(directory, 'immersion.sqlite');
|
||||||
|
const runtime = new LexicalRollupWorkerRuntime();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
applyPragmas(db);
|
||||||
|
ensureSchema(db);
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||||
|
VALUES ('鳥', '鳥', 'とり', 1700000000, 1700000000, 1)`,
|
||||||
|
).run();
|
||||||
|
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||||
|
db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
|
||||||
|
'lexical_daily_rollups_version',
|
||||||
|
);
|
||||||
|
db.close();
|
||||||
|
|
||||||
|
await runtime.run(dbPath);
|
||||||
|
|
||||||
|
const checkDb = new Database(dbPath);
|
||||||
|
try {
|
||||||
|
assert.equal(areLexicalDailyRollupsReady(checkDb), true);
|
||||||
|
} finally {
|
||||||
|
checkDb.close();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
runtime.destroy();
|
||||||
|
try {
|
||||||
|
db.close();
|
||||||
|
} catch {
|
||||||
|
// Closed before the worker starts.
|
||||||
|
}
|
||||||
|
fs.rmSync(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('lexical rollup worker module resolves in the current layout', () => {
|
||||||
|
const workerPath = resolveLexicalRollupWorkerPath();
|
||||||
|
assert.ok(workerPath, 'expected the lexical rollup worker module to resolve');
|
||||||
|
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('lexical rollup worker leaves a backfill pending when no worker can start', async () => {
|
||||||
|
const runtime = new LexicalRollupWorkerRuntime({
|
||||||
|
resolveWorkerPath: () => null,
|
||||||
|
warn: () => {},
|
||||||
|
} as never);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await assert.doesNotReject(runtime.run('/tmp/not-used.sqlite'));
|
||||||
|
} finally {
|
||||||
|
runtime.destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('lexical rollup worker absorbs termination failures after settling', async () => {
|
||||||
|
let sendMessage: ((message: { ok: boolean }) => void) | null = null;
|
||||||
|
const runtime = new LexicalRollupWorkerRuntime({
|
||||||
|
resolveWorkerPath: () => '/tmp/fake-worker.js',
|
||||||
|
createWorker: async () => ({
|
||||||
|
once(event: string, listener: (value: never) => void) {
|
||||||
|
if (event === 'message') sendMessage = listener as (message: { ok: boolean }) => void;
|
||||||
|
return this;
|
||||||
|
},
|
||||||
|
terminate: async () => {
|
||||||
|
throw new Error('termination failed');
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
warn: () => {},
|
||||||
|
} as never);
|
||||||
|
|
||||||
|
const unhandled: unknown[] = [];
|
||||||
|
const captureUnhandled = (reason: unknown) => unhandled.push(reason);
|
||||||
|
process.on('unhandledRejection', captureUnhandled);
|
||||||
|
try {
|
||||||
|
const task = runtime.run('/tmp/not-used.sqlite');
|
||||||
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
|
const notify = sendMessage as ((message: { ok: boolean }) => void) | null;
|
||||||
|
assert.ok(notify);
|
||||||
|
notify({ ok: true });
|
||||||
|
await task;
|
||||||
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
|
assert.deepEqual(unhandled, []);
|
||||||
|
} finally {
|
||||||
|
process.off('unhandledRejection', captureUnhandled);
|
||||||
|
runtime.destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('lexical rollup worker times out when it never responds', async () => {
|
||||||
|
let terminated = false;
|
||||||
|
const runtime = new LexicalRollupWorkerRuntime({
|
||||||
|
resolveWorkerPath: () => '/tmp/fake-worker.js',
|
||||||
|
createWorker: async () => ({
|
||||||
|
once() {
|
||||||
|
return this;
|
||||||
|
},
|
||||||
|
terminate: async () => {
|
||||||
|
terminated = true;
|
||||||
|
return 0;
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
timeoutMs: 1,
|
||||||
|
warn: () => {},
|
||||||
|
} as never);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const outcome = await Promise.race([
|
||||||
|
runtime.run('/tmp/not-used.sqlite').then(
|
||||||
|
() => 'resolved',
|
||||||
|
(error: unknown) => String(error),
|
||||||
|
),
|
||||||
|
new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.match(outcome, /timed out/);
|
||||||
|
assert.equal(terminated, true);
|
||||||
|
} finally {
|
||||||
|
runtime.destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { createLogger } from '../../../logger';
|
||||||
|
|
||||||
|
interface WorkerResponse {
|
||||||
|
ok?: boolean;
|
||||||
|
error?: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface WorkerHandle {
|
||||||
|
once(event: 'message', listener: (message: WorkerResponse) => void): this;
|
||||||
|
once(event: 'error', listener: (error: Error) => void): this;
|
||||||
|
once(event: 'exit', listener: (code: number) => void): this;
|
||||||
|
terminate(): Promise<number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface LexicalRollupWorkerRuntimeOptions {
|
||||||
|
resolveWorkerPath?: () => string | null;
|
||||||
|
createWorker?: (workerPath: string, workerData: { dbPath: string }) => Promise<WorkerHandle>;
|
||||||
|
timeoutMs?: number;
|
||||||
|
warn?: (message: string, ...meta: unknown[]) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const logger = createLogger('main:immersion-tracker:lexical-rollup-worker');
|
||||||
|
const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
|
||||||
|
|
||||||
|
export function resolveLexicalRollupWorkerPath(): string | null {
|
||||||
|
const fileName = __filename.endsWith('.ts')
|
||||||
|
? 'lexical-rollup-worker-thread.ts'
|
||||||
|
: 'lexical-rollup-worker-thread.js';
|
||||||
|
const workerPath = path.join(__dirname, fileName);
|
||||||
|
return fs.existsSync(workerPath) ? workerPath : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class LexicalRollupWorkerRuntime {
|
||||||
|
private readonly activeWorkers = new Set<WorkerHandle>();
|
||||||
|
private destroyed = false;
|
||||||
|
|
||||||
|
constructor(private readonly options: LexicalRollupWorkerRuntimeOptions = {}) {}
|
||||||
|
|
||||||
|
async run(dbPath: string): Promise<void> {
|
||||||
|
if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
|
||||||
|
let worker: WorkerHandle;
|
||||||
|
try {
|
||||||
|
const workerPath = (this.options.resolveWorkerPath ?? resolveLexicalRollupWorkerPath)();
|
||||||
|
if (!workerPath) throw new Error('Emitted lexical rollup worker module was not found');
|
||||||
|
const createWorker =
|
||||||
|
this.options.createWorker ??
|
||||||
|
(async (resolvedPath, workerData) => {
|
||||||
|
const { Worker } = await import('node:worker_threads');
|
||||||
|
return new Worker(resolvedPath, { workerData });
|
||||||
|
});
|
||||||
|
worker = await createWorker(workerPath, { dbPath });
|
||||||
|
} catch (error) {
|
||||||
|
if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
|
||||||
|
(this.options.warn ?? logger.warn)(
|
||||||
|
'Lexical rollup worker unavailable; leaving backfill pending for a later startup',
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.destroyed) {
|
||||||
|
await worker.terminate().catch(() => undefined);
|
||||||
|
throw new Error('Lexical rollup worker is shut down');
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Promise<void>((resolve, reject) => {
|
||||||
|
let settled = false;
|
||||||
|
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
this.activeWorkers.add(worker);
|
||||||
|
const settle = (error?: Error) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
if (timeout) clearTimeout(timeout);
|
||||||
|
this.activeWorkers.delete(worker);
|
||||||
|
void worker.terminate().catch(() => undefined);
|
||||||
|
if (error) reject(error);
|
||||||
|
else resolve();
|
||||||
|
};
|
||||||
|
timeout = setTimeout(
|
||||||
|
() => settle(new Error('Lexical rollup worker timed out')),
|
||||||
|
this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
|
||||||
|
);
|
||||||
|
worker.once('message', (message) => {
|
||||||
|
if (message.ok) settle();
|
||||||
|
else
|
||||||
|
settle(
|
||||||
|
new Error(
|
||||||
|
`Lexical rollup backfill failed: ${String(message.error ?? 'unknown error')}`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
worker.once('error', (error) => settle(error));
|
||||||
|
worker.once('exit', (code) => {
|
||||||
|
if (!settled) {
|
||||||
|
settle(
|
||||||
|
new Error(
|
||||||
|
code === 0
|
||||||
|
? 'Lexical rollup worker exited without a response'
|
||||||
|
: `Lexical rollup worker exited with code ${code}`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
destroy(): void {
|
||||||
|
if (this.destroyed) return;
|
||||||
|
this.destroyed = true;
|
||||||
|
for (const worker of this.activeWorkers) {
|
||||||
|
void worker.terminate().catch(() => undefined);
|
||||||
|
}
|
||||||
|
this.activeWorkers.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
import { parentPort, workerData } from 'node:worker_threads';
|
||||||
|
import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
|
||||||
|
|
||||||
|
if (!parentPort) throw new Error('lexical rollup worker missing parent port');
|
||||||
|
|
||||||
|
try {
|
||||||
|
executeLexicalRollupBackfillTask((workerData as { dbPath: string }).dbPath);
|
||||||
|
parentPort.postMessage({ ok: true });
|
||||||
|
} catch (error) {
|
||||||
|
parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import test from 'node:test';
|
||||||
|
import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
|
||||||
|
import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
|
||||||
|
import { Database } from './sqlite';
|
||||||
|
import { ensureSchema } from './storage';
|
||||||
|
|
||||||
|
test('lexical rollup backfill materializes pre-existing vocabulary off the caller DB connection', () => {
|
||||||
|
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-worker-'));
|
||||||
|
const dbPath = path.join(directory, 'immersion.sqlite');
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||||
|
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||||
|
).run('犬', '犬', 'いぬ', 1_700_000_000, 1_700_000_000);
|
||||||
|
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||||
|
db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
|
||||||
|
'lexical_daily_rollups_version',
|
||||||
|
);
|
||||||
|
|
||||||
|
executeLexicalRollupBackfillTask(dbPath);
|
||||||
|
|
||||||
|
assert.equal(areLexicalDailyRollupsReady(db), true);
|
||||||
|
assert.equal(getLexicalDailyRollups(db)[0]?.wordCount, 1);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { areLexicalDailyRollupsReady, rebuildLexicalDailyRollups } from './lexical-rollups';
|
||||||
|
import { Database } from './sqlite';
|
||||||
|
import { applyPragmas } from './storage';
|
||||||
|
|
||||||
|
export function executeLexicalRollupBackfillTask(dbPath: string): void {
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
try {
|
||||||
|
applyPragmas(db);
|
||||||
|
if (!areLexicalDailyRollupsReady(db)) {
|
||||||
|
rebuildLexicalDailyRollups(db);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,421 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import test from 'node:test';
|
||||||
|
import {
|
||||||
|
areLexicalDailyRollupsReady,
|
||||||
|
getLexicalDailyRollups,
|
||||||
|
rebuildLexicalDailyRollups,
|
||||||
|
} from './lexical-rollups';
|
||||||
|
import { getTrendsDashboard } from './query-trends';
|
||||||
|
import {
|
||||||
|
getVocabularyChartData,
|
||||||
|
getVocabularySummary,
|
||||||
|
replaceStatsExcludedWords,
|
||||||
|
} from './query-lexical';
|
||||||
|
import { Database } from './sqlite';
|
||||||
|
import type { DatabaseSync } from './sqlite';
|
||||||
|
import { ensureSchema } from './storage';
|
||||||
|
|
||||||
|
function makeDbPath(): string {
|
||||||
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollups-'));
|
||||||
|
return path.join(dir, 'immersion.sqlite');
|
||||||
|
}
|
||||||
|
|
||||||
|
test('lexical daily rollups follow first-seen corrections and deletions', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const firstDay = 19_500;
|
||||||
|
const correctedDay = firstDay + 2;
|
||||||
|
const firstSeen = firstDay * 86_400 + 43_200;
|
||||||
|
const correctedSeen = correctedDay * 86_400 + 43_200;
|
||||||
|
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||||
|
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||||
|
).run('猫', '猫', 'ねこ', firstSeen, firstSeen);
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
|
||||||
|
VALUES (?, ?, ?, 1)`,
|
||||||
|
).run('猫', firstSeen, firstSeen);
|
||||||
|
|
||||||
|
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||||
|
{ epochDay: firstDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 1 },
|
||||||
|
]);
|
||||||
|
|
||||||
|
db.prepare(`UPDATE imm_words SET first_seen = ? WHERE headword = ?`).run(correctedSeen, '猫');
|
||||||
|
db.prepare(`DELETE FROM imm_kanji WHERE kanji = ?`).run('猫');
|
||||||
|
|
||||||
|
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||||
|
{ epochDay: correctedDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
|
||||||
|
]);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('lexical daily rollups normalize second and millisecond timestamps', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const epochDay = 19_500;
|
||||||
|
const timestampSeconds = epochDay * 86_400 + 43_200;
|
||||||
|
const timestampMilliseconds = timestampSeconds * 1_000;
|
||||||
|
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||||
|
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||||
|
).run('猫', '猫', 'ねこ', timestampSeconds, timestampSeconds);
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||||
|
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||||
|
).run('犬', '犬', 'いぬ', timestampMilliseconds, timestampMilliseconds);
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
|
||||||
|
VALUES (?, ?, ?, 1)`,
|
||||||
|
).run('猫', timestampSeconds, timestampSeconds);
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
|
||||||
|
VALUES (?, ?, ?, 1)`,
|
||||||
|
).run('犬', timestampMilliseconds, timestampMilliseconds);
|
||||||
|
|
||||||
|
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||||
|
{ epochDay, wordCount: 2, wordCountWithoutNames: 2, kanjiCount: 2 },
|
||||||
|
]);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('lexical rollup rebuild excludes rows hidden by vocabulary persistence rules', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const epochDay = 19_500;
|
||||||
|
const firstSeen = epochDay * 86_400 + 43_200;
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(
|
||||||
|
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||||
|
).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(
|
||||||
|
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||||
|
).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
|
||||||
|
|
||||||
|
rebuildLexicalDailyRollups(db);
|
||||||
|
|
||||||
|
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||||
|
{ epochDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
|
||||||
|
]);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('lexical rollup rebuild tolerates nullable legacy vocabulary text', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||||
|
VALUES (NULL, NULL, NULL, 1700000000, 1700000000, 1)`,
|
||||||
|
).run();
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||||
|
VALUES (NULL, '猫', 'ねこ', 1700000000, 1700000000, 1)`,
|
||||||
|
).run();
|
||||||
|
|
||||||
|
assert.doesNotThrow(() => rebuildLexicalDailyRollups(db));
|
||||||
|
assert.equal(areLexicalDailyRollupsReady(db), true);
|
||||||
|
assert.equal(getVocabularySummary(db, null).uniqueWords, 1);
|
||||||
|
assert.equal(getVocabularySummary(db, new Set(['猫'])).knownWordCount, 1);
|
||||||
|
assert.equal(getVocabularyChartData(db).topWords[0]?.headword, '猫');
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('lexical rollup rebuild scans vocabulary visibility in bounded id batches', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
const expectedBatchSize = 5_000;
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const insertWord = db.prepare(
|
||||||
|
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||||
|
VALUES (?, ?, '', 1700000000, 1700000000, 1)`,
|
||||||
|
);
|
||||||
|
db.exec('BEGIN');
|
||||||
|
for (let index = 0; index <= expectedBatchSize; index += 1) {
|
||||||
|
insertWord.run(`語${index}`, `語${index}`);
|
||||||
|
}
|
||||||
|
db.exec('COMMIT');
|
||||||
|
|
||||||
|
const scanPageSizes: number[] = [];
|
||||||
|
const instrumentedDb: DatabaseSync = {
|
||||||
|
prepare(source) {
|
||||||
|
const statement = db.prepare(source);
|
||||||
|
if (!source.includes('WHERE id > ?') || !source.includes('ORDER BY id')) {
|
||||||
|
return statement;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
run: (...params) => statement.run(...params),
|
||||||
|
get: (...params) => statement.get(...params),
|
||||||
|
all: (...params) => {
|
||||||
|
const rows = statement.all(...params);
|
||||||
|
scanPageSizes.push(rows.length);
|
||||||
|
return rows;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
exec(source) {
|
||||||
|
db.exec(source);
|
||||||
|
return instrumentedDb;
|
||||||
|
},
|
||||||
|
close() {
|
||||||
|
return instrumentedDb;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
rebuildLexicalDailyRollups(instrumentedDb);
|
||||||
|
|
||||||
|
assert.deepEqual(scanPageSizes, [expectedBatchSize, 1]);
|
||||||
|
assert.equal(getVocabularySummary(db, null).uniqueWords, expectedBatchSize + 1);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('chart exclusions do not subtract vocabulary rows already hidden from the rollup', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const epochDay = 19_500;
|
||||||
|
const firstSeen = epochDay * 86_400 + 43_200;
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(
|
||||||
|
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||||
|
).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(
|
||||||
|
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||||
|
).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
|
||||||
|
rebuildLexicalDailyRollups(db);
|
||||||
|
replaceStatsExcludedWords(db, [{ headword: 'は', word: 'は', reading: 'は' }]);
|
||||||
|
|
||||||
|
const charts = getVocabularyChartData(db);
|
||||||
|
|
||||||
|
assert.deepEqual(charts.newWordsTimeline, [{ epochDay, wordCount: 1 }]);
|
||||||
|
assert.deepEqual(charts.newWordsTimelineWithoutNames, [{ epochDay, wordCount: 1 }]);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('legacy lexical rollup readiness does not satisfy the current rollup version', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_rollup_state(state_key, state_value)
|
||||||
|
VALUES ('lexical_daily_rollups_ready', '1')
|
||||||
|
ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
|
||||||
|
).run();
|
||||||
|
db.prepare(
|
||||||
|
`DELETE FROM imm_rollup_state WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||||
|
).run();
|
||||||
|
|
||||||
|
assert.equal(areLexicalDailyRollupsReady(db), false);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('current lexical rollup readiness accepts legacy integer state storage', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
db.exec(`
|
||||||
|
CREATE TABLE imm_rollup_state(
|
||||||
|
state_key TEXT PRIMARY KEY,
|
||||||
|
state_value INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
INSERT INTO imm_rollup_state(state_key, state_value)
|
||||||
|
VALUES ('lexical_daily_rollups_version', 2);
|
||||||
|
`);
|
||||||
|
|
||||||
|
assert.equal(areLexicalDailyRollupsReady(db), true);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('imm_words persists vocabulary visibility for rollup maintenance', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const columns = db.prepare(`PRAGMA table_info(imm_words)`).all() as Array<{ name: string }>;
|
||||||
|
|
||||||
|
assert.equal(
|
||||||
|
columns.some((column) => column.name === 'vocabulary_visible'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('vocabulary charts use complete top-word and lexical rollup data', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const insertWord = db.prepare(
|
||||||
|
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||||
|
VALUES (?, ?, '', 1700000000, 1700000000, ?)`,
|
||||||
|
);
|
||||||
|
for (let index = 0; index < 501; index += 1) {
|
||||||
|
insertWord.run(`語${index}`, `語${index}`, index === 500 ? 10_000 : 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const charts = getVocabularyChartData(db);
|
||||||
|
|
||||||
|
assert.equal(charts.topWords[0]?.headword, '語500');
|
||||||
|
assert.equal(charts.topWords[0]?.frequency, 10_000);
|
||||||
|
assert.equal(charts.newWordsTimeline[0]?.wordCount, 501);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('vocabulary charts find full top-word sets beyond excluded and name rows', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const insertWord = db.prepare(
|
||||||
|
`INSERT INTO imm_words(headword, word, reading, pos2, first_seen, last_seen, frequency)
|
||||||
|
VALUES (?, ?, '', ?, 1700000000, 1700000000, ?)`,
|
||||||
|
);
|
||||||
|
const exclusions = [];
|
||||||
|
for (let index = 0; index < 100; index += 1) {
|
||||||
|
const headword = `語${index}`;
|
||||||
|
insertWord.run(
|
||||||
|
headword,
|
||||||
|
headword,
|
||||||
|
index < 80 && index >= 60 ? '固有名詞' : '一般',
|
||||||
|
100 - index,
|
||||||
|
);
|
||||||
|
if (index < 60) exclusions.push({ headword, word: headword, reading: '' });
|
||||||
|
}
|
||||||
|
replaceStatsExcludedWords(db, exclusions);
|
||||||
|
|
||||||
|
const charts = getVocabularyChartData(db);
|
||||||
|
|
||||||
|
assert.equal(charts.topWords.length, 12);
|
||||||
|
assert.equal(charts.topWords[0]?.headword, '語60');
|
||||||
|
assert.equal(charts.topWordsWithoutNames.length, 12);
|
||||||
|
assert.equal(charts.topWordsWithoutNames[0]?.headword, '語80');
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('vocabulary charts handle exclusion lists above one SQLite variable batch', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||||
|
VALUES ('語0', '語0', '', 1700000000, 1700000000, 1)`,
|
||||||
|
).run();
|
||||||
|
const exclusions = Array.from({ length: 10_923 }, (_, index) => ({
|
||||||
|
headword: `語${index}`,
|
||||||
|
word: `語${index}`,
|
||||||
|
reading: '',
|
||||||
|
}));
|
||||||
|
replaceStatsExcludedWords(db, exclusions);
|
||||||
|
|
||||||
|
const charts = getVocabularyChartData(db);
|
||||||
|
|
||||||
|
assert.deepEqual(charts.topWords, []);
|
||||||
|
assert.deepEqual(charts.newWordsTimeline, []);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('lexical rollup rebuild preserves the original error when rollback also fails', () => {
|
||||||
|
const originalError = new Error('rebuild failed');
|
||||||
|
const db = {
|
||||||
|
exec(sql: string) {
|
||||||
|
if (sql === 'BEGIN IMMEDIATE') return;
|
||||||
|
if (sql === 'ROLLBACK') throw new Error('rollback failed');
|
||||||
|
throw originalError;
|
||||||
|
},
|
||||||
|
prepare() {
|
||||||
|
return { all: () => [], run: () => undefined };
|
||||||
|
},
|
||||||
|
} as unknown as DatabaseSync;
|
||||||
|
|
||||||
|
assert.throws(() => rebuildLexicalDailyRollups(db), originalError);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('trends read historical new-word buckets from lexical rollups', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||||
|
VALUES ('海', '海', 'うみ', 1700000000, 1700000000, 1)`,
|
||||||
|
).run();
|
||||||
|
db.prepare(`UPDATE imm_lexical_daily_rollups SET word_count = 9`).run();
|
||||||
|
|
||||||
|
const dashboard = getTrendsDashboard(db, 'all', 'day', false);
|
||||||
|
|
||||||
|
assert.equal(dashboard.progress.newWords[0]?.value, 9);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -0,0 +1,229 @@
|
|||||||
|
import type { DatabaseSync } from './sqlite';
|
||||||
|
import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility';
|
||||||
|
|
||||||
|
export interface LexicalDailyRollup {
|
||||||
|
epochDay: number;
|
||||||
|
wordCount: number;
|
||||||
|
wordCountWithoutNames: number;
|
||||||
|
kanjiCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const LOCAL_EPOCH_DAY_SQL = `
|
||||||
|
CAST(julianday(
|
||||||
|
CASE
|
||||||
|
WHEN ABS(CAST(%VALUE% AS REAL)) >= 10000000000 THEN CAST(%VALUE% AS REAL) / 1000
|
||||||
|
ELSE CAST(%VALUE% AS REAL)
|
||||||
|
END,
|
||||||
|
'unixepoch', 'localtime'
|
||||||
|
) - 2440587.5 AS INTEGER)
|
||||||
|
`;
|
||||||
|
|
||||||
|
const LEXICAL_DAILY_ROLLUP_VERSION = '2';
|
||||||
|
const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
|
||||||
|
const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
|
||||||
|
|
||||||
|
export function localEpochDaySql(value: string): string {
|
||||||
|
return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createWordRollupTriggers(db: DatabaseSync): void {
|
||||||
|
const dayForNew = localEpochDaySql('NEW.first_seen');
|
||||||
|
const dayForOld = localEpochDaySql('OLD.first_seen');
|
||||||
|
|
||||||
|
db.exec(`
|
||||||
|
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_insert;
|
||||||
|
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_delete;
|
||||||
|
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_first_seen_update;
|
||||||
|
|
||||||
|
CREATE TRIGGER imm_words_lexical_rollup_insert
|
||||||
|
AFTER INSERT ON imm_words
|
||||||
|
WHEN NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||||
|
VALUES (${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0)
|
||||||
|
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||||
|
word_count = word_count + 1,
|
||||||
|
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER imm_words_lexical_rollup_delete
|
||||||
|
AFTER DELETE ON imm_words
|
||||||
|
WHEN OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||||
|
VALUES (${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0)
|
||||||
|
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||||
|
word_count = word_count - 1,
|
||||||
|
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||||
|
DELETE FROM imm_lexical_daily_rollups
|
||||||
|
WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER imm_words_lexical_rollup_first_seen_update
|
||||||
|
AFTER UPDATE OF first_seen, pos2, vocabulary_visible ON imm_words
|
||||||
|
WHEN OLD.first_seen IS NOT NEW.first_seen
|
||||||
|
OR OLD.pos2 IS NOT NEW.pos2
|
||||||
|
OR OLD.vocabulary_visible IS NOT NEW.vocabulary_visible
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||||
|
SELECT ${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0
|
||||||
|
WHERE OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
|
||||||
|
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||||
|
word_count = word_count - 1,
|
||||||
|
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||||
|
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||||
|
SELECT ${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0
|
||||||
|
WHERE NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
|
||||||
|
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||||
|
word_count = word_count + 1,
|
||||||
|
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||||
|
DELETE FROM imm_lexical_daily_rollups
|
||||||
|
WHERE word_count = 0 AND kanji_count = 0;
|
||||||
|
END;
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createKanjiRollupTriggers(db: DatabaseSync): void {
|
||||||
|
const dayForNew = localEpochDaySql('NEW.first_seen');
|
||||||
|
const dayForOld = localEpochDaySql('OLD.first_seen');
|
||||||
|
db.exec(`
|
||||||
|
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_insert;
|
||||||
|
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_delete;
|
||||||
|
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_first_seen_update;
|
||||||
|
|
||||||
|
CREATE TRIGGER imm_kanji_lexical_rollup_insert
|
||||||
|
AFTER INSERT ON imm_kanji WHEN NEW.first_seen IS NOT NULL
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||||
|
VALUES (${dayForNew}, 0, 0, 1)
|
||||||
|
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
|
||||||
|
END;
|
||||||
|
CREATE TRIGGER imm_kanji_lexical_rollup_delete
|
||||||
|
AFTER DELETE ON imm_kanji WHEN OLD.first_seen IS NOT NULL
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||||
|
VALUES (${dayForOld}, 0, 0, -1)
|
||||||
|
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
|
||||||
|
DELETE FROM imm_lexical_daily_rollups
|
||||||
|
WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
|
||||||
|
END;
|
||||||
|
CREATE TRIGGER imm_kanji_lexical_rollup_first_seen_update
|
||||||
|
AFTER UPDATE OF first_seen ON imm_kanji WHEN OLD.first_seen IS NOT NEW.first_seen
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||||
|
SELECT ${dayForOld}, 0, 0, -1 WHERE OLD.first_seen IS NOT NULL
|
||||||
|
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
|
||||||
|
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||||
|
SELECT ${dayForNew}, 0, 0, 1 WHERE NEW.first_seen IS NOT NULL
|
||||||
|
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
|
||||||
|
DELETE FROM imm_lexical_daily_rollups WHERE word_count = 0 AND kanji_count = 0;
|
||||||
|
END;
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ensureLexicalDailyRollupTables(db: DatabaseSync): void {
|
||||||
|
db.exec(`
|
||||||
|
CREATE TABLE IF NOT EXISTS imm_lexical_daily_rollups(
|
||||||
|
epoch_day INTEGER PRIMARY KEY,
|
||||||
|
word_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
word_count_without_names INTEGER NOT NULL DEFAULT 0,
|
||||||
|
kanji_count INTEGER NOT NULL DEFAULT 0
|
||||||
|
);
|
||||||
|
INSERT INTO imm_rollup_state(state_key, state_value)
|
||||||
|
VALUES ('${LEXICAL_DAILY_ROLLUP_VERSION_KEY}', '0')
|
||||||
|
ON CONFLICT(state_key) DO NOTHING;
|
||||||
|
`);
|
||||||
|
createWordRollupTriggers(db);
|
||||||
|
createKanjiRollupTriggers(db);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function areLexicalDailyRollupsReady(db: DatabaseSync): boolean {
|
||||||
|
const row = db
|
||||||
|
.prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
|
||||||
|
.get(LEXICAL_DAILY_ROLLUP_VERSION_KEY) as { value: string | number } | undefined;
|
||||||
|
// Older databases created this column with INTEGER affinity, while current
|
||||||
|
// databases use TEXT. SQLite returns the same persisted version with a
|
||||||
|
// different JS type depending on that legacy schema.
|
||||||
|
return row !== undefined && String(row.value) === LEXICAL_DAILY_ROLLUP_VERSION;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function markLexicalDailyRollupsReady(db: DatabaseSync): void {
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO imm_rollup_state(state_key, state_value)
|
||||||
|
VALUES (?, ?)
|
||||||
|
ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
|
||||||
|
).run(LEXICAL_DAILY_ROLLUP_VERSION_KEY, LEXICAL_DAILY_ROLLUP_VERSION);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Rebuild from the first-seen source of truth; run off the UI/main DB thread. */
|
||||||
|
export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
|
||||||
|
let transactionStarted = false;
|
||||||
|
try {
|
||||||
|
db.exec('BEGIN IMMEDIATE');
|
||||||
|
transactionStarted = true;
|
||||||
|
const scanVocabulary = db.prepare(
|
||||||
|
`SELECT id, word, headword, reading, part_of_speech AS partOfSpeech,
|
||||||
|
pos1, pos2, pos3, frequency_rank AS frequencyRank
|
||||||
|
FROM imm_words
|
||||||
|
WHERE id > ?
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT ?`,
|
||||||
|
);
|
||||||
|
const updateVisibility = db.prepare(
|
||||||
|
`UPDATE imm_words SET vocabulary_visible = ? WHERE id = ? AND vocabulary_visible IS NOT ?`,
|
||||||
|
);
|
||||||
|
let lastId = Number.MIN_SAFE_INTEGER;
|
||||||
|
for (;;) {
|
||||||
|
const vocabularyRows = scanVocabulary.all(
|
||||||
|
lastId,
|
||||||
|
VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE,
|
||||||
|
) as Array<VocabularyVisibilityRow & { id: number }>;
|
||||||
|
if (vocabularyRows.length === 0) break;
|
||||||
|
for (const row of vocabularyRows) {
|
||||||
|
const visible = isVocabularyStatsRowVisible(row) ? 1 : 0;
|
||||||
|
updateVisibility.run(visible, row.id, visible);
|
||||||
|
}
|
||||||
|
lastId = vocabularyRows[vocabularyRows.length - 1]!.id;
|
||||||
|
if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break;
|
||||||
|
}
|
||||||
|
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||||
|
db.exec(`
|
||||||
|
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||||
|
SELECT ${localEpochDaySql('first_seen')}, COUNT(*),
|
||||||
|
SUM(CASE WHEN pos2 = '固有名詞' THEN 0 ELSE 1 END), 0
|
||||||
|
FROM imm_words
|
||||||
|
WHERE first_seen IS NOT NULL AND vocabulary_visible = 1
|
||||||
|
GROUP BY ${localEpochDaySql('first_seen')};
|
||||||
|
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||||
|
SELECT ${localEpochDaySql('first_seen')}, 0, 0, COUNT(*)
|
||||||
|
FROM imm_kanji
|
||||||
|
WHERE first_seen IS NOT NULL
|
||||||
|
GROUP BY ${localEpochDaySql('first_seen')}
|
||||||
|
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + excluded.kanji_count;
|
||||||
|
`);
|
||||||
|
markLexicalDailyRollupsReady(db);
|
||||||
|
db.exec('COMMIT');
|
||||||
|
} catch (error) {
|
||||||
|
if (transactionStarted) {
|
||||||
|
try {
|
||||||
|
db.exec('ROLLBACK');
|
||||||
|
} catch {
|
||||||
|
// Preserve the rebuild failure; it is the actionable cause.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getLexicalDailyRollups(db: DatabaseSync): LexicalDailyRollup[] {
|
||||||
|
return db
|
||||||
|
.prepare(
|
||||||
|
`
|
||||||
|
SELECT epoch_day AS epochDay, word_count AS wordCount,
|
||||||
|
word_count_without_names AS wordCountWithoutNames, kanji_count AS kanjiCount
|
||||||
|
FROM imm_lexical_daily_rollups
|
||||||
|
ORDER BY epoch_day ASC
|
||||||
|
`,
|
||||||
|
)
|
||||||
|
.all() as LexicalDailyRollup[];
|
||||||
|
}
|
||||||
@@ -1,6 +1,4 @@
|
|||||||
import type { DatabaseSync } from './sqlite';
|
import type { DatabaseSync } from './sqlite';
|
||||||
import { PartOfSpeech, type MergedToken } from '../../../types';
|
|
||||||
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
|
|
||||||
import type {
|
import type {
|
||||||
KanjiAnimeAppearanceRow,
|
KanjiAnimeAppearanceRow,
|
||||||
KanjiDetailRow,
|
KanjiDetailRow,
|
||||||
@@ -13,19 +11,38 @@ import type {
|
|||||||
SimilarWordRow,
|
SimilarWordRow,
|
||||||
StatsExcludedWordRow,
|
StatsExcludedWordRow,
|
||||||
VocabularyStatsRow,
|
VocabularyStatsRow,
|
||||||
|
VocabularyStatsSummary,
|
||||||
WordAnimeAppearanceRow,
|
WordAnimeAppearanceRow,
|
||||||
WordDetailRow,
|
WordDetailRow,
|
||||||
WordOccurrenceRow,
|
WordOccurrenceRow,
|
||||||
} from './types';
|
} from './types';
|
||||||
import { fromDbTimestamp, toDbTimestamp } from './query-shared';
|
import { fromDbTimestamp, toDbTimestamp } from './query-shared';
|
||||||
import { nowMs } from './time';
|
import { nowMs } from './time';
|
||||||
|
import {
|
||||||
|
areLexicalDailyRollupsReady,
|
||||||
|
getLexicalDailyRollups,
|
||||||
|
localEpochDaySql,
|
||||||
|
} from './lexical-rollups';
|
||||||
|
import { isVocabularyStatsRowVisible } from './vocabulary-visibility';
|
||||||
|
|
||||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
|
const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
|
||||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
|
const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
|
||||||
|
const VOCABULARY_CHART_LIMIT = 12;
|
||||||
|
const VOCABULARY_CHART_PAGE_SIZE = 100;
|
||||||
|
const EXCLUSION_ALIAS_BATCH_SIZE = 300;
|
||||||
|
const VOCABULARY_SUMMARY_SCAN_BATCH_SIZE = 5_000;
|
||||||
const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
|
const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
|
||||||
const SENTENCE_SEARCH_MAX_LIMIT = 100;
|
const SENTENCE_SEARCH_MAX_LIMIT = 100;
|
||||||
const KANJI_PATTERN = /\p{Script=Han}/gu;
|
const KANJI_PATTERN = /\p{Script=Han}/gu;
|
||||||
|
|
||||||
|
export interface VocabularyChartData {
|
||||||
|
ready: boolean;
|
||||||
|
topWords: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||||
|
topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||||
|
newWordsTimeline: Array<{ epochDay: number; wordCount: number }>;
|
||||||
|
newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>;
|
||||||
|
}
|
||||||
|
|
||||||
function resolveSentenceSearchLimit(limit: number): number {
|
function resolveSentenceSearchLimit(limit: number): number {
|
||||||
if (!Number.isFinite(limit)) return SENTENCE_SEARCH_DEFAULT_LIMIT;
|
if (!Number.isFinite(limit)) return SENTENCE_SEARCH_DEFAULT_LIMIT;
|
||||||
const normalized = Math.floor(limit);
|
const normalized = Math.floor(limit);
|
||||||
@@ -73,33 +90,6 @@ function uniqueKanji(text: string): string[] {
|
|||||||
return Array.from(new Set(text.match(KANJI_PATTERN) ?? []));
|
return Array.from(new Set(text.match(KANJI_PATTERN) ?? []));
|
||||||
}
|
}
|
||||||
|
|
||||||
function toVocabularyToken(row: VocabularyStatsRow): MergedToken {
|
|
||||||
const partOfSpeech =
|
|
||||||
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
|
|
||||||
? (row.partOfSpeech as PartOfSpeech)
|
|
||||||
: PartOfSpeech.other;
|
|
||||||
|
|
||||||
return {
|
|
||||||
surface: row.word,
|
|
||||||
reading: row.reading ?? '',
|
|
||||||
headword: row.headword,
|
|
||||||
startPos: 0,
|
|
||||||
endPos: row.word.length,
|
|
||||||
partOfSpeech,
|
|
||||||
pos1: row.pos1 ?? '',
|
|
||||||
pos2: row.pos2 ?? '',
|
|
||||||
pos3: row.pos3 ?? '',
|
|
||||||
frequencyRank: row.frequencyRank ?? undefined,
|
|
||||||
isMerged: false,
|
|
||||||
isKnown: false,
|
|
||||||
isNPlusOneTarget: false,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function isVocabularyStatsRowVisible(row: VocabularyStatsRow): boolean {
|
|
||||||
return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getVocabularyStats(
|
export function getVocabularyStats(
|
||||||
db: DatabaseSync,
|
db: DatabaseSync,
|
||||||
limit = 100,
|
limit = 100,
|
||||||
@@ -153,6 +143,198 @@ export function getVocabularyStats(
|
|||||||
return visibleRows.slice(0, limit);
|
return visibleRows.slice(0, limit);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Chart data is intentionally independent of the paginated vocabulary tables.
|
||||||
|
* Top words use the frequency index; new-word history reads permanent daily
|
||||||
|
* lexical rollups rather than loading every vocabulary row into the dashboard.
|
||||||
|
*/
|
||||||
|
export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData {
|
||||||
|
const ready = areLexicalDailyRollupsReady(db);
|
||||||
|
const excludedAliases = new Set(
|
||||||
|
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
|
||||||
|
);
|
||||||
|
const isExcluded = (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>): boolean =>
|
||||||
|
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias));
|
||||||
|
const topWords = getTopVocabularyChartWords(db, isExcluded);
|
||||||
|
const rollups = ready ? getLexicalDailyRollups(db) : [];
|
||||||
|
const timeline = new Map(rollups.map((row) => [row.epochDay, { ...row }]));
|
||||||
|
if (excludedAliases.size > 0 && ready) {
|
||||||
|
const aliases = [...excludedAliases];
|
||||||
|
const excludedRows = new Map<
|
||||||
|
number,
|
||||||
|
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
|
||||||
|
wordId: number;
|
||||||
|
epochDay: number;
|
||||||
|
}
|
||||||
|
>();
|
||||||
|
for (let offset = 0; offset < aliases.length; offset += EXCLUSION_ALIAS_BATCH_SIZE) {
|
||||||
|
const batch = aliases.slice(offset, offset + EXCLUSION_ALIAS_BATCH_SIZE);
|
||||||
|
const placeholders = batch.map(() => '?').join(', ');
|
||||||
|
const rows = db
|
||||||
|
.prepare(
|
||||||
|
`
|
||||||
|
SELECT id AS wordId, headword, word, reading, pos2,
|
||||||
|
${localEpochDaySql('first_seen')} AS epochDay
|
||||||
|
FROM imm_words
|
||||||
|
WHERE vocabulary_visible = 1
|
||||||
|
AND (headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}))
|
||||||
|
`,
|
||||||
|
)
|
||||||
|
.all(...batch, ...batch, ...batch) as Array<
|
||||||
|
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
|
||||||
|
wordId: number;
|
||||||
|
epochDay: number;
|
||||||
|
}
|
||||||
|
>;
|
||||||
|
for (const row of rows) excludedRows.set(row.wordId, row);
|
||||||
|
}
|
||||||
|
for (const word of excludedRows.values()) {
|
||||||
|
if (!isExcluded(word)) continue;
|
||||||
|
const rollup = timeline.get(word.epochDay);
|
||||||
|
if (!rollup) continue;
|
||||||
|
rollup.wordCount -= 1;
|
||||||
|
if (word.pos2 !== '固有名詞') rollup.wordCountWithoutNames -= 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
ready,
|
||||||
|
topWords: topWords.all.map((word) => ({
|
||||||
|
wordId: word.wordId,
|
||||||
|
headword: vocabularyDisplayHeadword(word),
|
||||||
|
frequency: word.frequency,
|
||||||
|
})),
|
||||||
|
topWordsWithoutNames: topWords.withoutNames.map((word) => ({
|
||||||
|
wordId: word.wordId,
|
||||||
|
headword: vocabularyDisplayHeadword(word),
|
||||||
|
frequency: word.frequency,
|
||||||
|
})),
|
||||||
|
newWordsTimeline: [...timeline.values()]
|
||||||
|
.filter((row) => row.wordCount > 0)
|
||||||
|
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCount })),
|
||||||
|
newWordsTimelineWithoutNames: [...timeline.values()]
|
||||||
|
.filter((row) => row.wordCountWithoutNames > 0)
|
||||||
|
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCountWithoutNames })),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTopVocabularyChartWords(
|
||||||
|
db: DatabaseSync,
|
||||||
|
isExcluded: (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>) => boolean,
|
||||||
|
): { all: VocabularyStatsRow[]; withoutNames: VocabularyStatsRow[] } {
|
||||||
|
const stmt = db.prepare(`
|
||||||
|
SELECT id AS wordId, headword, word, reading,
|
||||||
|
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
|
||||||
|
frequency, frequency_rank AS frequencyRank,
|
||||||
|
first_seen AS firstSeen, last_seen AS lastSeen,
|
||||||
|
0 AS animeCount
|
||||||
|
FROM imm_words
|
||||||
|
ORDER BY frequency DESC, id
|
||||||
|
LIMIT ? OFFSET ?
|
||||||
|
`);
|
||||||
|
const all: VocabularyStatsRow[] = [];
|
||||||
|
const withoutNames: VocabularyStatsRow[] = [];
|
||||||
|
let offset = 0;
|
||||||
|
|
||||||
|
while (all.length < VOCABULARY_CHART_LIMIT || withoutNames.length < VOCABULARY_CHART_LIMIT) {
|
||||||
|
const page = stmt.all(VOCABULARY_CHART_PAGE_SIZE, offset) as VocabularyStatsRow[];
|
||||||
|
if (page.length === 0) break;
|
||||||
|
for (const word of page) {
|
||||||
|
if (!isVocabularyStatsRowVisible(word) || isExcluded(word)) continue;
|
||||||
|
if (all.length < VOCABULARY_CHART_LIMIT) all.push(word);
|
||||||
|
if (word.pos2 !== '固有名詞' && withoutNames.length < VOCABULARY_CHART_LIMIT) {
|
||||||
|
withoutNames.push(word);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
offset += page.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { all, withoutNames };
|
||||||
|
}
|
||||||
|
|
||||||
|
function excludedVocabularyAliases(
|
||||||
|
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
|
||||||
|
): string[] {
|
||||||
|
const aliases = [word.headword?.trim() ?? '', word.word?.trim() ?? ''].filter(Boolean);
|
||||||
|
if (aliases.length === 0) aliases.push(word.reading?.trim() ?? '');
|
||||||
|
return [...new Set(aliases)];
|
||||||
|
}
|
||||||
|
|
||||||
|
function vocabularyDisplayHeadword(
|
||||||
|
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
|
||||||
|
): string {
|
||||||
|
return word.headword?.trim() || word.word?.trim() || word.reading?.trim() || '';
|
||||||
|
}
|
||||||
|
|
||||||
|
function timestampSeconds(timestamp: number): number {
|
||||||
|
return timestamp < 10_000_000_000 ? timestamp : Math.floor(timestamp / 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getVocabularySummary(
|
||||||
|
db: DatabaseSync,
|
||||||
|
knownWords: ReadonlySet<string> | null,
|
||||||
|
nowMs: number = Date.now(),
|
||||||
|
scanBatchSize: number = VOCABULARY_SUMMARY_SCAN_BATCH_SIZE,
|
||||||
|
): VocabularyStatsSummary {
|
||||||
|
// Visibility and exclusion rules live in JS, so rows are scanned in id-keyed
|
||||||
|
// batches to keep memory bounded on large vocabularies.
|
||||||
|
const scanStmt = db.prepare(`
|
||||||
|
SELECT id AS wordId, headword, word, reading,
|
||||||
|
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
|
||||||
|
frequency, frequency_rank AS frequencyRank,
|
||||||
|
first_seen AS firstSeen, last_seen AS lastSeen,
|
||||||
|
0 AS animeCount
|
||||||
|
FROM imm_words
|
||||||
|
WHERE id > ?
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT ?
|
||||||
|
`);
|
||||||
|
const excludedAliases = new Set(
|
||||||
|
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
|
||||||
|
);
|
||||||
|
const weekAgoSec = nowMs / 1000 - 7 * 86_400;
|
||||||
|
const summary: VocabularyStatsSummary = {
|
||||||
|
uniqueWords: 0,
|
||||||
|
uniqueWordsWithoutNames: 0,
|
||||||
|
uniqueKanji: (db.prepare('SELECT COUNT(*) AS count FROM imm_kanji').get() as { count: number })
|
||||||
|
.count,
|
||||||
|
newThisWeek: 0,
|
||||||
|
newThisWeekWithoutNames: 0,
|
||||||
|
knownWordCount: knownWords ? 0 : null,
|
||||||
|
knownWordCountWithoutNames: knownWords ? 0 : null,
|
||||||
|
};
|
||||||
|
|
||||||
|
let lastId = Number.MIN_SAFE_INTEGER;
|
||||||
|
for (;;) {
|
||||||
|
const words = scanStmt.all(lastId, scanBatchSize) as VocabularyStatsRow[];
|
||||||
|
if (words.length === 0) break;
|
||||||
|
lastId = words[words.length - 1]!.wordId;
|
||||||
|
for (const word of words) {
|
||||||
|
if (
|
||||||
|
!isVocabularyStatsRowVisible(word) ||
|
||||||
|
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias))
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const isName = word.pos2 === '固有名詞';
|
||||||
|
const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec;
|
||||||
|
const isKnown = knownWords?.has(vocabularyDisplayHeadword(word)) ?? false;
|
||||||
|
summary.uniqueWords += 1;
|
||||||
|
if (!isName) summary.uniqueWordsWithoutNames += 1;
|
||||||
|
if (isNewThisWeek) {
|
||||||
|
summary.newThisWeek += 1;
|
||||||
|
if (!isName) summary.newThisWeekWithoutNames += 1;
|
||||||
|
}
|
||||||
|
if (isKnown) {
|
||||||
|
summary.knownWordCount! += 1;
|
||||||
|
if (!isName) summary.knownWordCountWithoutNames! += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (words.length < scanBatchSize) break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return summary;
|
||||||
|
}
|
||||||
|
|
||||||
export function getStatsExcludedWords(db: DatabaseSync): StatsExcludedWordRow[] {
|
export function getStatsExcludedWords(db: DatabaseSync): StatsExcludedWordRow[] {
|
||||||
return db
|
return db
|
||||||
.prepare(
|
.prepare(
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import {
|
|||||||
toDbTimestamp,
|
toDbTimestamp,
|
||||||
} from './query-shared';
|
} from './query-shared';
|
||||||
import { getDailyRollups, getMonthlyRollups } from './query-sessions';
|
import { getDailyRollups, getMonthlyRollups } from './query-sessions';
|
||||||
|
import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
|
||||||
|
|
||||||
type TrendRange = '7d' | '30d' | '90d' | '365d' | 'all';
|
type TrendRange = '7d' | '30d' | '90d' | '365d' | 'all';
|
||||||
type TrendGroupBy = 'day' | 'month';
|
type TrendGroupBy = 'day' | 'month';
|
||||||
@@ -660,6 +661,16 @@ function buildNewWordsPerDay(
|
|||||||
cutoffMs: string | null,
|
cutoffMs: string | null,
|
||||||
axis: number[] | null,
|
axis: number[] | null,
|
||||||
): TrendChartPoint[] {
|
): TrendChartPoint[] {
|
||||||
|
if (areLexicalDailyRollupsReady(db)) {
|
||||||
|
// A trend range is defined in calendar buckets, so the rollup includes the
|
||||||
|
// complete local cutoff day rather than applying a time-of-day boundary.
|
||||||
|
const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
|
||||||
|
const rows = getLexicalDailyRollups(db).filter(
|
||||||
|
(row) => cutoffDay === null || row.epochDay >= cutoffDay,
|
||||||
|
);
|
||||||
|
return fillAxisPoints(axis, new Map(rows.map((row) => [row.epochDay, row.wordCount])));
|
||||||
|
}
|
||||||
|
|
||||||
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
||||||
const prepared = db.prepare(`
|
const prepared = db.prepare(`
|
||||||
SELECT
|
SELECT
|
||||||
@@ -691,6 +702,18 @@ function buildNewWordsPerMonth(
|
|||||||
cutoffMs: string | null,
|
cutoffMs: string | null,
|
||||||
axis: number[] | null,
|
axis: number[] | null,
|
||||||
): TrendChartPoint[] {
|
): TrendChartPoint[] {
|
||||||
|
if (areLexicalDailyRollupsReady(db)) {
|
||||||
|
const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
|
||||||
|
const byMonth = new Map<number, number>();
|
||||||
|
for (const row of getLexicalDailyRollups(db)) {
|
||||||
|
if (cutoffDay !== null && row.epochDay < cutoffDay) continue;
|
||||||
|
const { year, month } = dayPartsFromEpochDay(row.epochDay);
|
||||||
|
const monthKey = year * 100 + month;
|
||||||
|
byMonth.set(monthKey, (byMonth.get(monthKey) ?? 0) + row.wordCount);
|
||||||
|
}
|
||||||
|
return fillAxisPoints(axis, byMonth);
|
||||||
|
}
|
||||||
|
|
||||||
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
||||||
const prepared = db.prepare(`
|
const prepared = db.prepare(`
|
||||||
SELECT
|
SELECT
|
||||||
|
|||||||
@@ -184,6 +184,51 @@ test('ensureSchema adds manual assignment locks when upgrading the previous sche
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('ensureSchema preserves durable session rollups across unrelated schema upgrades', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
db.exec(`
|
||||||
|
INSERT INTO imm_videos (
|
||||||
|
video_id, video_key, canonical_title, source_type, duration_ms, CREATED_DATE, LAST_UPDATE_DATE
|
||||||
|
) VALUES (1, 'local:/tmp/preserved.mkv', 'Preserved', 1, 0, '1', '1');
|
||||||
|
INSERT INTO imm_daily_rollups (
|
||||||
|
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||||
|
total_tokens_seen, total_cards
|
||||||
|
) VALUES (20000, 1, 2, 30, 40, 50, 3);
|
||||||
|
INSERT INTO imm_monthly_rollups (
|
||||||
|
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||||
|
total_tokens_seen, total_cards
|
||||||
|
) VALUES (202410, 1, 2, 30, 40, 50, 3);
|
||||||
|
UPDATE imm_rollup_state
|
||||||
|
SET state_value = '123'
|
||||||
|
WHERE state_key = 'last_rollup_sample_ms';
|
||||||
|
UPDATE imm_schema_version SET schema_version = 21;
|
||||||
|
`);
|
||||||
|
|
||||||
|
ensureSchema(db);
|
||||||
|
|
||||||
|
const daily = db
|
||||||
|
.prepare('SELECT total_sessions AS totalSessions FROM imm_daily_rollups')
|
||||||
|
.get() as { totalSessions: number } | null;
|
||||||
|
const monthly = db
|
||||||
|
.prepare('SELECT total_sessions AS totalSessions FROM imm_monthly_rollups')
|
||||||
|
.get() as { totalSessions: number } | null;
|
||||||
|
const rollupState = db
|
||||||
|
.prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
|
||||||
|
.get('last_rollup_sample_ms') as { value: string } | null;
|
||||||
|
|
||||||
|
assert.equal(daily?.totalSessions, 2);
|
||||||
|
assert.equal(monthly?.totalSessions, 2);
|
||||||
|
assert.equal(rollupState?.value, '123');
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('stats excluded words are replaced and read from sqlite storage', () => {
|
test('stats excluded words are replaced and read from sqlite storage', () => {
|
||||||
const dbPath = makeDbPath();
|
const dbPath = makeDbPath();
|
||||||
const db = new Database(dbPath);
|
const db = new Database(dbPath);
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { parseMediaInfo } from '../../../jimaku/utils';
|
|||||||
import { normalizeTitleIdentity } from '../../utils/title-normalization';
|
import { normalizeTitleIdentity } from '../../utils/title-normalization';
|
||||||
import type { DatabaseSync } from './sqlite';
|
import type { DatabaseSync } from './sqlite';
|
||||||
import { nowMs } from './time';
|
import { nowMs } from './time';
|
||||||
|
import { ensureLexicalDailyRollupTables, markLexicalDailyRollupsReady } from './lexical-rollups';
|
||||||
import { SCHEMA_VERSION } from './types';
|
import { SCHEMA_VERSION } from './types';
|
||||||
import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types';
|
import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types';
|
||||||
import { toDbMs, toDbTimestamp } from './query-shared';
|
import { toDbMs, toDbTimestamp } from './query-shared';
|
||||||
@@ -890,11 +891,11 @@ export function ensureSchema(db: DatabaseSync): void {
|
|||||||
VALUES ('last_rollup_sample_ms', 0)
|
VALUES ('last_rollup_sample_ms', 0)
|
||||||
ON CONFLICT(state_key) DO NOTHING
|
ON CONFLICT(state_key) DO NOTHING
|
||||||
`);
|
`);
|
||||||
|
|
||||||
const currentVersion = db
|
const currentVersion = db
|
||||||
.prepare('SELECT schema_version FROM imm_schema_version ORDER BY schema_version DESC LIMIT 1')
|
.prepare('SELECT schema_version FROM imm_schema_version ORDER BY schema_version DESC LIMIT 1')
|
||||||
.get() as { schema_version: number } | null;
|
.get() as { schema_version: number } | null;
|
||||||
if (currentVersion?.schema_version === SCHEMA_VERSION) {
|
if (currentVersion?.schema_version === SCHEMA_VERSION) {
|
||||||
|
ensureLexicalDailyRollupTables(db);
|
||||||
ensureLifetimeSummaryTables(db);
|
ensureLifetimeSummaryTables(db);
|
||||||
ensureStatsExcludedWordsTable(db);
|
ensureStatsExcludedWordsTable(db);
|
||||||
ensureAnimeMergeTables(db);
|
ensureAnimeMergeTables(db);
|
||||||
@@ -1068,6 +1069,7 @@ export function ensureSchema(db: DatabaseSync): void {
|
|||||||
last_seen REAL,
|
last_seen REAL,
|
||||||
frequency INTEGER,
|
frequency INTEGER,
|
||||||
frequency_rank INTEGER,
|
frequency_rank INTEGER,
|
||||||
|
vocabulary_visible INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1)),
|
||||||
UNIQUE(headword, word, reading)
|
UNIQUE(headword, word, reading)
|
||||||
);
|
);
|
||||||
`);
|
`);
|
||||||
@@ -1451,8 +1453,18 @@ export function ensureSchema(db: DatabaseSync): void {
|
|||||||
addColumnIfMissing(db, 'imm_sessions', 'ended_media_ms', 'INTEGER');
|
addColumnIfMissing(db, 'imm_sessions', 'ended_media_ms', 'INTEGER');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (currentVersion?.schema_version && currentVersion.schema_version < 23) {
|
||||||
|
addColumnIfMissing(
|
||||||
|
db,
|
||||||
|
'imm_words',
|
||||||
|
'vocabulary_visible',
|
||||||
|
'INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1))',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
migrateSessionEventTimestampsToText(db);
|
migrateSessionEventTimestampsToText(db);
|
||||||
|
|
||||||
|
ensureLexicalDailyRollupTables(db);
|
||||||
ensureLifetimeSummaryTables(db);
|
ensureLifetimeSummaryTables(db);
|
||||||
ensureStatsExcludedWordsTable(db);
|
ensureStatsExcludedWordsTable(db);
|
||||||
|
|
||||||
@@ -1572,19 +1584,21 @@ export function ensureSchema(db: DatabaseSync): void {
|
|||||||
ON imm_youtube_videos(youtube_video_id)
|
ON imm_youtube_videos(youtube_video_id)
|
||||||
`);
|
`);
|
||||||
|
|
||||||
if (currentVersion?.schema_version && currentVersion.schema_version < SCHEMA_VERSION) {
|
// Session rollups intentionally outlive raw session and telemetry retention.
|
||||||
db.exec('DELETE FROM imm_daily_rollups');
|
// Preserve them across unrelated schema upgrades because deleted historical
|
||||||
db.exec('DELETE FROM imm_monthly_rollups');
|
// buckets cannot be rebuilt after their source rows have been pruned.
|
||||||
db.exec(
|
|
||||||
`UPDATE imm_rollup_state SET state_value = 0 WHERE state_key = 'last_rollup_sample_ms'`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
db.exec(`
|
db.exec(`
|
||||||
INSERT INTO imm_schema_version(schema_version, applied_at_ms)
|
INSERT INTO imm_schema_version(schema_version, applied_at_ms)
|
||||||
VALUES (${SCHEMA_VERSION}, ${toDbTimestamp(nowMs())})
|
VALUES (${SCHEMA_VERSION}, ${toDbTimestamp(nowMs())})
|
||||||
ON CONFLICT DO NOTHING
|
ON CONFLICT DO NOTHING
|
||||||
`);
|
`);
|
||||||
|
|
||||||
|
// A new database has no history to materialize. Upgrades are populated by the
|
||||||
|
// background worker so startup never scans the existing vocabulary table.
|
||||||
|
if (!currentVersion) {
|
||||||
|
markLexicalDailyRollupsReady(db);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPreparedStatements {
|
export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPreparedStatements {
|
||||||
@@ -1617,9 +1631,10 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
|
|||||||
`),
|
`),
|
||||||
wordUpsertStmt: db.prepare(`
|
wordUpsertStmt: db.prepare(`
|
||||||
INSERT INTO imm_words (
|
INSERT INTO imm_words (
|
||||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency, frequency_rank
|
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen,
|
||||||
|
frequency, frequency_rank, vocabulary_visible
|
||||||
) VALUES (
|
) VALUES (
|
||||||
?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?
|
?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, 1
|
||||||
)
|
)
|
||||||
ON CONFLICT(headword, word, reading) DO UPDATE SET
|
ON CONFLICT(headword, word, reading) DO UPDATE SET
|
||||||
frequency = COALESCE(frequency, 0) + 1,
|
frequency = COALESCE(frequency, 0) + 1,
|
||||||
@@ -1632,6 +1647,7 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
|
|||||||
pos1 = COALESCE(NULLIF(imm_words.pos1, ''), excluded.pos1),
|
pos1 = COALESCE(NULLIF(imm_words.pos1, ''), excluded.pos1),
|
||||||
pos2 = COALESCE(NULLIF(imm_words.pos2, ''), excluded.pos2),
|
pos2 = COALESCE(NULLIF(imm_words.pos2, ''), excluded.pos2),
|
||||||
pos3 = COALESCE(NULLIF(imm_words.pos3, ''), excluded.pos3),
|
pos3 = COALESCE(NULLIF(imm_words.pos3, ''), excluded.pos3),
|
||||||
|
vocabulary_visible = 1,
|
||||||
first_seen = MIN(COALESCE(first_seen, excluded.first_seen), excluded.first_seen),
|
first_seen = MIN(COALESCE(first_seen, excluded.first_seen), excluded.first_seen),
|
||||||
last_seen = MAX(COALESCE(last_seen, excluded.last_seen), excluded.last_seen),
|
last_seen = MAX(COALESCE(last_seen, excluded.last_seen), excluded.last_seen),
|
||||||
frequency_rank = CASE
|
frequency_rank = CASE
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
export const SCHEMA_VERSION = 21;
|
export const SCHEMA_VERSION = 23;
|
||||||
export const DEFAULT_QUEUE_CAP = 1_000;
|
export const DEFAULT_QUEUE_CAP = 1_000;
|
||||||
export const DEFAULT_BATCH_SIZE = 25;
|
export const DEFAULT_BATCH_SIZE = 25;
|
||||||
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
|
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
|
||||||
@@ -306,6 +306,16 @@ export interface VocabularyStatsRow {
|
|||||||
lastSeen: number;
|
lastSeen: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface VocabularyStatsSummary {
|
||||||
|
uniqueWords: number;
|
||||||
|
uniqueWordsWithoutNames: number;
|
||||||
|
uniqueKanji: number;
|
||||||
|
newThisWeek: number;
|
||||||
|
newThisWeekWithoutNames: number;
|
||||||
|
knownWordCount: number | null;
|
||||||
|
knownWordCountWithoutNames: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
export interface StatsExcludedWordRow {
|
export interface StatsExcludedWordRow {
|
||||||
headword: string;
|
headword: string;
|
||||||
word: string;
|
word: string;
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import test from 'node:test';
|
||||||
|
import {
|
||||||
|
resolveVocabularySummaryWorkerPath,
|
||||||
|
VocabularySummaryWorkerRuntime,
|
||||||
|
} from './vocabulary-summary-worker-runtime';
|
||||||
|
import { Database } from './sqlite';
|
||||||
|
import { applyPragmas, ensureSchema } from './storage';
|
||||||
|
|
||||||
|
test('vocabulary summary worker reads the database from a separate connection', async () => {
|
||||||
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-vocabulary-summary-worker-'));
|
||||||
|
const dbPath = path.join(tempDir, 'immersion.sqlite');
|
||||||
|
const runtime = new VocabularySummaryWorkerRuntime();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
applyPragmas(db);
|
||||||
|
ensureSchema(db);
|
||||||
|
db.prepare(
|
||||||
|
`
|
||||||
|
INSERT INTO imm_words (
|
||||||
|
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||||
|
first_seen, last_seen, frequency
|
||||||
|
) VALUES ('猫', '猫', 'ねこ', 'noun', '名詞', '一般', '', 1, 1, 1)
|
||||||
|
`,
|
||||||
|
).run();
|
||||||
|
db.close();
|
||||||
|
|
||||||
|
const summary = await runtime.run(dbPath, new Set(['猫']));
|
||||||
|
|
||||||
|
assert.equal(summary.uniqueWords, 1);
|
||||||
|
assert.equal(summary.knownWordCount, 1);
|
||||||
|
} finally {
|
||||||
|
runtime.destroy();
|
||||||
|
try {
|
||||||
|
db.close();
|
||||||
|
} catch {
|
||||||
|
// The worker needs the setup connection closed before it starts.
|
||||||
|
}
|
||||||
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('vocabulary summary worker module resolves in the current layout', () => {
|
||||||
|
const workerPath = resolveVocabularySummaryWorkerPath();
|
||||||
|
assert.ok(workerPath, 'expected the vocabulary summary worker module to resolve');
|
||||||
|
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('vocabulary summary worker never falls back to the caller thread', async () => {
|
||||||
|
const runtime = new VocabularySummaryWorkerRuntime({
|
||||||
|
resolveWorkerPath: () => null,
|
||||||
|
warn: () => {},
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await assert.rejects(
|
||||||
|
runtime.run('/tmp/subminer-summary-worker-not-used.sqlite', null),
|
||||||
|
/worker unavailable/i,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
runtime.destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('vocabulary summary worker times out when it never responds', async () => {
|
||||||
|
let terminated = false;
|
||||||
|
const runtime = new VocabularySummaryWorkerRuntime({
|
||||||
|
resolveWorkerPath: () => '/tmp/fake-worker.js',
|
||||||
|
createWorker: async () => ({
|
||||||
|
once() {
|
||||||
|
return this;
|
||||||
|
},
|
||||||
|
terminate: async () => {
|
||||||
|
terminated = true;
|
||||||
|
return 0;
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
timeoutMs: 1,
|
||||||
|
warn: () => {},
|
||||||
|
} as never);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const outcome = await Promise.race([
|
||||||
|
runtime.run('/tmp/not-used.sqlite', null).then(
|
||||||
|
() => 'resolved',
|
||||||
|
(error: unknown) => String(error),
|
||||||
|
),
|
||||||
|
new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.match(outcome, /timed out/);
|
||||||
|
assert.equal(terminated, true);
|
||||||
|
} finally {
|
||||||
|
runtime.destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { createLogger } from '../../../logger';
|
||||||
|
import type { VocabularyStatsSummary } from './types';
|
||||||
|
|
||||||
|
interface VocabularySummaryWorkerResponse {
|
||||||
|
summary?: VocabularyStatsSummary;
|
||||||
|
error?: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface VocabularySummaryWorkerHandle {
|
||||||
|
once(event: 'message', listener: (message: VocabularySummaryWorkerResponse) => void): this;
|
||||||
|
once(event: 'error', listener: (error: Error) => void): this;
|
||||||
|
once(event: 'exit', listener: (code: number) => void): this;
|
||||||
|
terminate(): Promise<number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface VocabularySummaryWorkerRuntimeOptions {
|
||||||
|
resolveWorkerPath?: () => string | null;
|
||||||
|
createWorker?: (
|
||||||
|
workerPath: string,
|
||||||
|
workerData: { dbPath: string; knownWords: string[] | null },
|
||||||
|
) => Promise<VocabularySummaryWorkerHandle>;
|
||||||
|
timeoutMs?: number;
|
||||||
|
warn?: (message: string, ...meta: unknown[]) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RunVocabularySummaryTask = (
|
||||||
|
dbPath: string,
|
||||||
|
knownWords: ReadonlySet<string> | null,
|
||||||
|
) => Promise<VocabularyStatsSummary>;
|
||||||
|
|
||||||
|
export function resolveVocabularySummaryWorkerPath(): string | null {
|
||||||
|
const fileName = __filename.endsWith('.ts')
|
||||||
|
? 'vocabulary-summary-worker-thread.ts'
|
||||||
|
: 'vocabulary-summary-worker-thread.js';
|
||||||
|
const workerPath = path.join(__dirname, fileName);
|
||||||
|
return fs.existsSync(workerPath) ? workerPath : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const logger = createLogger('main:immersion-tracker:vocabulary-summary-worker');
|
||||||
|
const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
|
||||||
|
|
||||||
|
export class VocabularySummaryWorkerRuntime {
|
||||||
|
private readonly activeWorkers = new Set<VocabularySummaryWorkerHandle>();
|
||||||
|
private destroyed = false;
|
||||||
|
|
||||||
|
constructor(private readonly options: VocabularySummaryWorkerRuntimeOptions = {}) {}
|
||||||
|
|
||||||
|
async run(
|
||||||
|
dbPath: string,
|
||||||
|
knownWords: ReadonlySet<string> | null,
|
||||||
|
): Promise<VocabularyStatsSummary> {
|
||||||
|
if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
|
||||||
|
const workerData = { dbPath, knownWords: knownWords ? [...knownWords] : null };
|
||||||
|
let worker: VocabularySummaryWorkerHandle;
|
||||||
|
try {
|
||||||
|
const workerPath = (this.options.resolveWorkerPath ?? resolveVocabularySummaryWorkerPath)();
|
||||||
|
if (!workerPath) throw new Error('Emitted vocabulary summary worker module was not found');
|
||||||
|
const createWorker =
|
||||||
|
this.options.createWorker ??
|
||||||
|
(async (resolvedPath, data) => {
|
||||||
|
const { Worker } = await import('node:worker_threads');
|
||||||
|
return new Worker(resolvedPath, { workerData: data });
|
||||||
|
});
|
||||||
|
worker = await createWorker(workerPath, workerData);
|
||||||
|
} catch (error) {
|
||||||
|
if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
|
||||||
|
(this.options.warn ?? logger.warn)(
|
||||||
|
'Vocabulary summary worker unavailable; refusing to scan vocabulary on the current thread',
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
throw new Error('Vocabulary summary worker unavailable');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.destroyed) {
|
||||||
|
await worker.terminate().catch(() => undefined);
|
||||||
|
throw new Error('Vocabulary summary worker is shut down');
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Promise<VocabularyStatsSummary>((resolve, reject) => {
|
||||||
|
let settled = false;
|
||||||
|
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
this.activeWorkers.add(worker);
|
||||||
|
const settle = (result: VocabularyStatsSummary | Error) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
if (timeout) clearTimeout(timeout);
|
||||||
|
this.activeWorkers.delete(worker);
|
||||||
|
void worker.terminate().catch(() => undefined);
|
||||||
|
if (result instanceof Error) reject(result);
|
||||||
|
else resolve(result);
|
||||||
|
};
|
||||||
|
timeout = setTimeout(
|
||||||
|
() => settle(new Error('Vocabulary summary worker timed out')),
|
||||||
|
this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
|
||||||
|
);
|
||||||
|
|
||||||
|
worker.once('message', (message) => {
|
||||||
|
if (message.summary) {
|
||||||
|
settle(message.summary);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
settle(
|
||||||
|
new Error(
|
||||||
|
`Vocabulary summary failed: ${String(message.error ?? 'unknown worker error')}`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
worker.once('error', (error) => settle(error));
|
||||||
|
worker.once('exit', (code) => {
|
||||||
|
if (!settled) {
|
||||||
|
settle(
|
||||||
|
new Error(
|
||||||
|
code === 0
|
||||||
|
? 'Vocabulary summary worker exited without a response'
|
||||||
|
: `Vocabulary summary worker exited with code ${code}`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
destroy(): void {
|
||||||
|
if (this.destroyed) return;
|
||||||
|
this.destroyed = true;
|
||||||
|
for (const worker of this.activeWorkers) {
|
||||||
|
void worker.terminate().catch(() => undefined);
|
||||||
|
}
|
||||||
|
this.activeWorkers.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
import { parentPort, workerData } from 'node:worker_threads';
|
||||||
|
import { executeVocabularySummaryTask } from './vocabulary-summary-worker';
|
||||||
|
|
||||||
|
interface VocabularySummaryWorkerData {
|
||||||
|
dbPath: string;
|
||||||
|
knownWords: string[] | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!parentPort) throw new Error('vocabulary summary worker missing parent port');
|
||||||
|
|
||||||
|
const request = workerData as VocabularySummaryWorkerData;
|
||||||
|
|
||||||
|
try {
|
||||||
|
parentPort.postMessage({
|
||||||
|
summary: executeVocabularySummaryTask(request.dbPath, request.knownWords),
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
import { getVocabularySummary } from './query-lexical';
|
||||||
|
import { Database } from './sqlite';
|
||||||
|
import { applyPragmas } from './storage';
|
||||||
|
import type { VocabularyStatsSummary } from './types';
|
||||||
|
|
||||||
|
export function executeVocabularySummaryTask(
|
||||||
|
dbPath: string,
|
||||||
|
knownWords: string[] | null,
|
||||||
|
): VocabularyStatsSummary {
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
try {
|
||||||
|
applyPragmas(db);
|
||||||
|
return getVocabularySummary(db, knownWords ? new Set(knownWords) : null);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
import { PartOfSpeech, type MergedToken } from '../../../types';
|
||||||
|
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
|
||||||
|
|
||||||
|
export interface VocabularyVisibilityRow {
|
||||||
|
word: string | null;
|
||||||
|
headword: string | null;
|
||||||
|
reading?: string | null;
|
||||||
|
partOfSpeech?: string | null;
|
||||||
|
pos1?: string | null;
|
||||||
|
pos2?: string | null;
|
||||||
|
pos3?: string | null;
|
||||||
|
frequencyRank?: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function toVocabularyToken(row: VocabularyVisibilityRow): MergedToken {
|
||||||
|
const word = row.word ?? '';
|
||||||
|
const headword = row.headword ?? word;
|
||||||
|
const partOfSpeech =
|
||||||
|
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
|
||||||
|
? (row.partOfSpeech as PartOfSpeech)
|
||||||
|
: PartOfSpeech.other;
|
||||||
|
|
||||||
|
return {
|
||||||
|
surface: word,
|
||||||
|
reading: row.reading ?? '',
|
||||||
|
headword,
|
||||||
|
startPos: 0,
|
||||||
|
endPos: word.length,
|
||||||
|
partOfSpeech,
|
||||||
|
pos1: row.pos1 ?? '',
|
||||||
|
pos2: row.pos2 ?? '',
|
||||||
|
pos3: row.pos3 ?? '',
|
||||||
|
frequencyRank: row.frequencyRank ?? undefined,
|
||||||
|
isMerged: false,
|
||||||
|
isKnown: false,
|
||||||
|
isNPlusOneTarget: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isVocabularyStatsRowVisible(row: VocabularyVisibilityRow): boolean {
|
||||||
|
if (!(row.word?.trim() || row.headword?.trim())) return false;
|
||||||
|
return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
|
||||||
|
}
|
||||||
@@ -34,6 +34,7 @@ import {
|
|||||||
parseSubsyncManualRunRequest,
|
parseSubsyncManualRunRequest,
|
||||||
parseYoutubePickerResolveRequest,
|
parseYoutubePickerResolveRequest,
|
||||||
} from '../../shared/ipc/validators';
|
} from '../../shared/ipc/validators';
|
||||||
|
import { applyOverlayClickThrough } from './overlay-click-through';
|
||||||
|
|
||||||
const { ipcMain } = electron;
|
const { ipcMain } = electron;
|
||||||
|
|
||||||
@@ -442,7 +443,13 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar
|
|||||||
const senderWindow =
|
const senderWindow =
|
||||||
electron.BrowserWindow?.fromWebContents((event as IpcMainEvent).sender) ?? null;
|
electron.BrowserWindow?.fromWebContents((event as IpcMainEvent).sender) ?? null;
|
||||||
if (senderWindow && !senderWindow.isDestroyed()) {
|
if (senderWindow && !senderWindow.isDestroyed()) {
|
||||||
senderWindow.setIgnoreMouseEvents(ignore, parsedOptions);
|
// Route forwarding requests through the platform-aware helper so Windows never
|
||||||
|
// installs Electron's global mouse hook (see overlay-click-through.ts).
|
||||||
|
if (ignore && parsedOptions?.forward) {
|
||||||
|
applyOverlayClickThrough(senderWindow);
|
||||||
|
} else {
|
||||||
|
senderWindow.setIgnoreMouseEvents(ignore, parsedOptions);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
deps.onOverlayMouseInteractionChanged?.(!ignore, senderWindow);
|
deps.onOverlayMouseInteractionChanged?.(!ignore, senderWindow);
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -125,9 +125,83 @@ test('mineSentenceCard creates sentence card from mpv subtitle state', async ()
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('mineSentenceCard refreshes secondary subtitle text before creating card', async () => {
|
test('mineSentenceCard prefers a canonical primary subtitle snapshot', async () => {
|
||||||
|
const created: Array<{
|
||||||
|
sentence: string;
|
||||||
|
startTime: number;
|
||||||
|
endTime: number;
|
||||||
|
secondarySub?: string;
|
||||||
|
}> = [];
|
||||||
|
|
||||||
|
await mineSentenceCard({
|
||||||
|
ankiIntegration: {
|
||||||
|
updateLastAddedFromClipboard: async () => {},
|
||||||
|
triggerFieldGroupingForLastAddedCard: async () => {},
|
||||||
|
markLastCardAsAudioCard: async () => {},
|
||||||
|
createSentenceCard: async (sentence, startTime, endTime, secondarySub) => {
|
||||||
|
created.push({ sentence, startTime, endTime, secondarySub });
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
mpvClient: {
|
||||||
|
connected: true,
|
||||||
|
currentSubText: '今今今手手手',
|
||||||
|
currentSubStart: 11.4,
|
||||||
|
currentSubEnd: 11.8,
|
||||||
|
currentSecondarySubText: 'English subtitle',
|
||||||
|
},
|
||||||
|
primarySubtitle: {
|
||||||
|
text: '今 手にある物差しでは',
|
||||||
|
startTime: 11.13,
|
||||||
|
endTime: 13.83,
|
||||||
|
},
|
||||||
|
showMpvOsd: () => {},
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.deepEqual(created, [
|
||||||
|
{
|
||||||
|
sentence: '今 手にある物差しでは',
|
||||||
|
startTime: 11.13,
|
||||||
|
endTime: 13.83,
|
||||||
|
secondarySub: 'English subtitle',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('mineSentenceCard uses normalized secondary subtitle state instead of raw mpv text', async () => {
|
||||||
|
const created: Array<{ sentence: string; secondarySub?: string }> = [];
|
||||||
|
let requestedRawSecondaryText = false;
|
||||||
|
|
||||||
|
await mineSentenceCard({
|
||||||
|
ankiIntegration: {
|
||||||
|
updateLastAddedFromClipboard: async () => {},
|
||||||
|
triggerFieldGroupingForLastAddedCard: async () => {},
|
||||||
|
markLastCardAsAudioCard: async () => {},
|
||||||
|
createSentenceCard: async (sentence, _startTime, _endTime, secondarySub) => {
|
||||||
|
created.push({ sentence, secondarySub });
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
mpvClient: {
|
||||||
|
connected: true,
|
||||||
|
currentSubText: '日本語字幕',
|
||||||
|
currentSubStart: 10,
|
||||||
|
currentSubEnd: 12,
|
||||||
|
currentSecondarySubText: 'Your\nmosaic',
|
||||||
|
requestProperty: async () => {
|
||||||
|
requestedRawSecondaryText = true;
|
||||||
|
return 'Your\nYour\nYour\nYour\nmosaic';
|
||||||
|
},
|
||||||
|
},
|
||||||
|
showMpvOsd: () => {},
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(requestedRawSecondaryText, false);
|
||||||
|
assert.deepEqual(created, [{ sentence: '日本語字幕', secondarySub: 'Your\nmosaic' }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('mineSentenceCard omits normalized secondary text that matches the primary subtitle', async () => {
|
||||||
const created: Array<{ sentence: string; secondarySub?: string }> = [];
|
const created: Array<{ sentence: string; secondarySub?: string }> = [];
|
||||||
const requestedProperties: string[] = [];
|
|
||||||
|
|
||||||
await mineSentenceCard({
|
await mineSentenceCard({
|
||||||
ankiIntegration: {
|
ankiIntegration: {
|
||||||
@@ -145,43 +219,6 @@ test('mineSentenceCard refreshes secondary subtitle text before creating card',
|
|||||||
currentSubStart: 10,
|
currentSubStart: 10,
|
||||||
currentSubEnd: 12,
|
currentSubEnd: 12,
|
||||||
currentSecondarySubText: '日本語字幕',
|
currentSecondarySubText: '日本語字幕',
|
||||||
requestProperty: async (name: string) => {
|
|
||||||
requestedProperties.push(name);
|
|
||||||
return name === 'secondary-sub-text' ? 'English subtitle' : null;
|
|
||||||
},
|
|
||||||
},
|
|
||||||
showMpvOsd: () => {},
|
|
||||||
});
|
|
||||||
|
|
||||||
assert.deepEqual(requestedProperties, ['secondary-sub-text']);
|
|
||||||
assert.deepEqual(created, [{ sentence: '日本語字幕', secondarySub: 'English subtitle' }]);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('mineSentenceCard does not fall back to stale cached secondary subtitle after successful refresh', async () => {
|
|
||||||
const created: Array<{ sentence: string; secondarySub?: string }> = [];
|
|
||||||
|
|
||||||
await mineSentenceCard({
|
|
||||||
ankiIntegration: {
|
|
||||||
updateLastAddedFromClipboard: async () => {},
|
|
||||||
triggerFieldGroupingForLastAddedCard: async () => {},
|
|
||||||
markLastCardAsAudioCard: async () => {},
|
|
||||||
createSentenceCard: async (sentence, _startTime, _endTime, secondarySub) => {
|
|
||||||
created.push({ sentence, secondarySub });
|
|
||||||
return true;
|
|
||||||
},
|
|
||||||
},
|
|
||||||
mpvClient: {
|
|
||||||
connected: true,
|
|
||||||
currentSubText: '日本語字幕',
|
|
||||||
currentSubStart: 10,
|
|
||||||
currentSubEnd: 12,
|
|
||||||
currentSecondarySubText: 'stale cached subtitle',
|
|
||||||
requestProperty: async (name: string) => {
|
|
||||||
if (name === 'secondary-sub-text') {
|
|
||||||
return '';
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
showMpvOsd: () => {},
|
showMpvOsd: () => {},
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -129,18 +129,10 @@ function normalizeSecondarySubText(text: unknown, primaryText: string): string |
|
|||||||
return trimmed;
|
return trimmed;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getCurrentSecondarySubTextForSentenceCard(
|
function getCurrentSecondarySubTextForSentenceCard(
|
||||||
mpvClient: MpvClientLike,
|
mpvClient: MpvClientLike,
|
||||||
): Promise<string | undefined> {
|
primaryText: string,
|
||||||
const primaryText = mpvClient.currentSubText;
|
): string | undefined {
|
||||||
if (mpvClient.requestProperty) {
|
|
||||||
try {
|
|
||||||
const latestSecondaryText = await mpvClient.requestProperty('secondary-sub-text');
|
|
||||||
return normalizeSecondarySubText(latestSecondaryText, primaryText);
|
|
||||||
} catch {
|
|
||||||
// Fall back to the cached secondary subtitle below.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return normalizeSecondarySubText(mpvClient.currentSecondarySubText, primaryText);
|
return normalizeSecondarySubText(mpvClient.currentSecondarySubText, primaryText);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -175,6 +167,7 @@ export async function markLastCardAsAudioCard(deps: {
|
|||||||
export async function mineSentenceCard(deps: {
|
export async function mineSentenceCard(deps: {
|
||||||
ankiIntegration: AnkiIntegrationLike | null;
|
ankiIntegration: AnkiIntegrationLike | null;
|
||||||
mpvClient: MpvClientLike | null;
|
mpvClient: MpvClientLike | null;
|
||||||
|
primarySubtitle?: Pick<SubtitleMiningContext, 'text' | 'startTime' | 'endTime'>;
|
||||||
showMpvOsd: (text: string) => void;
|
showMpvOsd: (text: string) => void;
|
||||||
}): Promise<boolean> {
|
}): Promise<boolean> {
|
||||||
const anki = requireAnkiIntegration(deps.ankiIntegration, deps.showMpvOsd);
|
const anki = requireAnkiIntegration(deps.ankiIntegration, deps.showMpvOsd);
|
||||||
@@ -185,16 +178,17 @@ export async function mineSentenceCard(deps: {
|
|||||||
deps.showMpvOsd('MPV not connected');
|
deps.showMpvOsd('MPV not connected');
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!mpvClient.currentSubText) {
|
const primaryText = deps.primarySubtitle?.text ?? mpvClient.currentSubText;
|
||||||
|
if (!primaryText) {
|
||||||
deps.showMpvOsd('No current subtitle');
|
deps.showMpvOsd('No current subtitle');
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
const secondarySubText = await getCurrentSecondarySubTextForSentenceCard(mpvClient);
|
const secondarySubText = getCurrentSecondarySubTextForSentenceCard(mpvClient, primaryText);
|
||||||
return await anki.createSentenceCard(
|
return await anki.createSentenceCard(
|
||||||
mpvClient.currentSubText,
|
primaryText,
|
||||||
mpvClient.currentSubStart,
|
deps.primarySubtitle?.startTime ?? mpvClient.currentSubStart,
|
||||||
mpvClient.currentSubEnd,
|
deps.primarySubtitle?.endTime ?? mpvClient.currentSubEnd,
|
||||||
secondarySubText,
|
secondarySubText,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ const MPV_SUBTITLE_PROPERTY_OBSERVATIONS: string[] = [
|
|||||||
'sub-scale-by-window',
|
'sub-scale-by-window',
|
||||||
'osd-height',
|
'osd-height',
|
||||||
'osd-dimensions',
|
'osd-dimensions',
|
||||||
'sub-text-ass',
|
'sub-text/ass',
|
||||||
'sub-border-size',
|
'sub-border-size',
|
||||||
'sub-shadow-offset',
|
'sub-shadow-offset',
|
||||||
'sub-ass-override',
|
'sub-ass-override',
|
||||||
@@ -65,6 +65,8 @@ const MPV_SUBTITLE_PROPERTY_OBSERVATIONS: string[] = [
|
|||||||
'secondary-sub-visibility',
|
'secondary-sub-visibility',
|
||||||
'sub-visibility',
|
'sub-visibility',
|
||||||
'sid',
|
'sid',
|
||||||
|
'secondary-sid',
|
||||||
|
'secondary-sub-delay',
|
||||||
'track-list',
|
'track-list',
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -74,7 +76,7 @@ const MPV_INITIAL_PROPERTY_REQUESTS: Array<MpvProtocolCommand> = [
|
|||||||
request_id: MPV_REQUEST_ID_SUBTEXT,
|
request_id: MPV_REQUEST_ID_SUBTEXT,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
command: ['get_property', 'sub-text-ass'],
|
command: ['get_property', 'sub-text/ass'],
|
||||||
request_id: MPV_REQUEST_ID_SUBTEXT_ASS,
|
request_id: MPV_REQUEST_ID_SUBTEXT_ASS,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -63,6 +63,8 @@ function createDeps(overrides: Partial<MpvProtocolHandleMessageDeps> = {}): {
|
|||||||
emitSubtitleTiming: (payload) => state.events.push(payload),
|
emitSubtitleTiming: (payload) => state.events.push(payload),
|
||||||
emitSecondarySubtitleChange: (payload) => state.events.push(payload),
|
emitSecondarySubtitleChange: (payload) => state.events.push(payload),
|
||||||
emitSubtitleTrackChange: (payload) => state.events.push(payload),
|
emitSubtitleTrackChange: (payload) => state.events.push(payload),
|
||||||
|
emitSecondarySubtitleTrackChange: (payload) => state.events.push(payload),
|
||||||
|
emitSecondarySubtitleDelayChange: (payload) => state.events.push(payload),
|
||||||
emitSubtitleTrackListChange: (payload) => state.events.push(payload),
|
emitSubtitleTrackListChange: (payload) => state.events.push(payload),
|
||||||
getCurrentSubText: () => state.subText,
|
getCurrentSubText: () => state.subText,
|
||||||
setCurrentSubText: (text) => {
|
setCurrentSubText: (text) => {
|
||||||
@@ -129,6 +131,28 @@ test('dispatchMpvProtocolMessage emits subtitle text on property change', async
|
|||||||
assert.deepEqual(state.events, [{ text: '字幕', isOverlayVisible: false }]);
|
assert.deepEqual(state.events, [{ text: '字幕', isOverlayVisible: false }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('dispatchMpvProtocolMessage emits ASS subtitle text from the current mpv property', async () => {
|
||||||
|
const { deps, state } = createDeps();
|
||||||
|
|
||||||
|
await dispatchMpvProtocolMessage(
|
||||||
|
{ event: 'property-change', name: 'sub-text/ass', data: '{\\b1}字幕' },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.deepEqual(state.events, [{ text: '{\\b1}字幕' }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('dispatchMpvProtocolMessage emits ASS subtitle text from the legacy mpv property', async () => {
|
||||||
|
const { deps, state } = createDeps();
|
||||||
|
|
||||||
|
await dispatchMpvProtocolMessage(
|
||||||
|
{ event: 'property-change', name: 'sub-text-ass', data: '{\\b1}字幕' },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.deepEqual(state.events, [{ text: '{\\b1}字幕' }]);
|
||||||
|
});
|
||||||
|
|
||||||
test('dispatchMpvProtocolMessage emits subtitle track changes', async () => {
|
test('dispatchMpvProtocolMessage emits subtitle track changes', async () => {
|
||||||
const { deps, state } = createDeps({
|
const { deps, state } = createDeps({
|
||||||
emitSubtitleTrackChange: (payload) => state.events.push(payload),
|
emitSubtitleTrackChange: (payload) => state.events.push(payload),
|
||||||
@@ -136,12 +160,42 @@ test('dispatchMpvProtocolMessage emits subtitle track changes', async () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sid', data: '3' }, deps);
|
await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sid', data: '3' }, deps);
|
||||||
|
await dispatchMpvProtocolMessage(
|
||||||
|
{ event: 'property-change', name: 'secondary-sid', data: '4' },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
await dispatchMpvProtocolMessage(
|
||||||
|
{ event: 'property-change', name: 'secondary-sub-delay', data: '0.5' },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
await dispatchMpvProtocolMessage(
|
await dispatchMpvProtocolMessage(
|
||||||
{ event: 'property-change', name: 'track-list', data: [{ type: 'sub', id: 3 }] },
|
{ event: 'property-change', name: 'track-list', data: [{ type: 'sub', id: 3 }] },
|
||||||
deps,
|
deps,
|
||||||
);
|
);
|
||||||
|
|
||||||
assert.deepEqual(state.events, [{ sid: 3 }, { trackList: [{ type: 'sub', id: 3 }] }]);
|
assert.deepEqual(state.events, [
|
||||||
|
{ sid: 3 },
|
||||||
|
{ sid: 4 },
|
||||||
|
{ delay: 0.5 },
|
||||||
|
{ trackList: [{ type: 'sub', id: 3 }] },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('dispatchMpvProtocolMessage rejects decimal subtitle track IDs', async () => {
|
||||||
|
const { deps, state } = createDeps();
|
||||||
|
|
||||||
|
await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sid', data: '4.5' }, deps);
|
||||||
|
await dispatchMpvProtocolMessage(
|
||||||
|
{ event: 'property-change', name: 'secondary-sid', data: '4.5' },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sid', data: 4.5 }, deps);
|
||||||
|
await dispatchMpvProtocolMessage(
|
||||||
|
{ event: 'property-change', name: 'secondary-sid', data: 4.5 },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.deepEqual(state.events, [{ sid: null }, { sid: null }, { sid: null }, { sid: null }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('dispatchMpvProtocolMessage enforces sub-visibility hidden when overlay suppression is enabled', async () => {
|
test('dispatchMpvProtocolMessage enforces sub-visibility hidden when overlay suppression is enabled', async () => {
|
||||||
|
|||||||
@@ -54,6 +54,8 @@ export interface MpvProtocolHandleMessageDeps {
|
|||||||
emitSubtitleTiming: (payload: { text: string; start: number; end: number }) => void;
|
emitSubtitleTiming: (payload: { text: string; start: number; end: number }) => void;
|
||||||
emitSecondarySubtitleChange: (payload: { text: string }) => void;
|
emitSecondarySubtitleChange: (payload: { text: string }) => void;
|
||||||
emitSubtitleTrackChange: (payload: { sid: number | null }) => void;
|
emitSubtitleTrackChange: (payload: { sid: number | null }) => void;
|
||||||
|
emitSecondarySubtitleTrackChange: (payload: { sid: number | null }) => void;
|
||||||
|
emitSecondarySubtitleDelayChange: (payload: { delay: number }) => void;
|
||||||
emitSubtitleTrackListChange: (payload: { trackList: unknown[] | null }) => void;
|
emitSubtitleTrackListChange: (payload: { trackList: unknown[] | null }) => void;
|
||||||
getCurrentSubText: () => string;
|
getCurrentSubText: () => string;
|
||||||
setCurrentSubText: (text: string) => void;
|
setCurrentSubText: (text: string) => void;
|
||||||
@@ -248,7 +250,7 @@ export async function dispatchMpvProtocolMessage(
|
|||||||
isOverlayVisible: overlayVisible,
|
isOverlayVisible: overlayVisible,
|
||||||
});
|
});
|
||||||
deps.setCurrentSubText(nextSubText);
|
deps.setCurrentSubText(nextSubText);
|
||||||
} else if (msg.name === 'sub-text-ass') {
|
} else if (msg.name === 'sub-text/ass' || msg.name === 'sub-text-ass') {
|
||||||
deps.emitSubtitleAssChange({ text: (msg.data as string) || '' });
|
deps.emitSubtitleAssChange({ text: (msg.data as string) || '' });
|
||||||
} else if (msg.name === 'sub-start') {
|
} else if (msg.name === 'sub-start') {
|
||||||
deps.setCurrentSubStart((msg.data as number) || 0);
|
deps.setCurrentSubStart((msg.data as number) || 0);
|
||||||
@@ -281,7 +283,25 @@ export async function dispatchMpvProtocolMessage(
|
|||||||
: typeof msg.data === 'string'
|
: typeof msg.data === 'string'
|
||||||
? Number(msg.data)
|
? Number(msg.data)
|
||||||
: null;
|
: null;
|
||||||
deps.emitSubtitleTrackChange({ sid: sid !== null && Number.isFinite(sid) ? sid : null });
|
deps.emitSubtitleTrackChange({ sid: sid !== null && Number.isInteger(sid) ? sid : null });
|
||||||
|
} else if (msg.name === 'secondary-sid') {
|
||||||
|
const sid =
|
||||||
|
typeof msg.data === 'number'
|
||||||
|
? msg.data
|
||||||
|
: typeof msg.data === 'string'
|
||||||
|
? Number(msg.data)
|
||||||
|
: null;
|
||||||
|
deps.emitSecondarySubtitleTrackChange({
|
||||||
|
sid: sid !== null && Number.isInteger(sid) ? sid : null,
|
||||||
|
});
|
||||||
|
} else if (msg.name === 'secondary-sub-delay') {
|
||||||
|
const delay =
|
||||||
|
typeof msg.data === 'number'
|
||||||
|
? msg.data
|
||||||
|
: typeof msg.data === 'string'
|
||||||
|
? Number(msg.data)
|
||||||
|
: 0;
|
||||||
|
deps.emitSecondarySubtitleDelayChange({ delay: Number.isFinite(delay) ? delay : 0 });
|
||||||
} else if (msg.name === 'track-list') {
|
} else if (msg.name === 'track-list') {
|
||||||
deps.emitSubtitleTrackListChange({
|
deps.emitSubtitleTrackListChange({
|
||||||
trackList: Array.isArray(msg.data) ? (msg.data as unknown[]) : null,
|
trackList: Array.isArray(msg.data) ? (msg.data as unknown[]) : null,
|
||||||
|
|||||||
@@ -38,7 +38,15 @@ class ManualCloseSocket extends FakeSocket {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const wait = () => new Promise((resolve) => setTimeout(resolve, 0));
|
class HangingSocket extends FakeSocket {
|
||||||
|
override connect(path: string): void {
|
||||||
|
this.connectedPaths.push(path);
|
||||||
|
// Never emits 'connect', 'error', or 'close' on its own: models a named
|
||||||
|
// pipe dial that stalls indefinitely.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const wait = (ms = 0) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
|
||||||
test('getMpvReconnectDelay follows existing reconnect ramp', () => {
|
test('getMpvReconnectDelay follows existing reconnect ramp', () => {
|
||||||
assert.equal(getMpvReconnectDelay(0, true), 1000);
|
assert.equal(getMpvReconnectDelay(0, true), 1000);
|
||||||
@@ -232,6 +240,75 @@ test('MpvSocketTransport.shutdown clears socket and lifecycle flags', async () =
|
|||||||
assert.deepEqual(events, []);
|
assert.deepEqual(events, []);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('MpvSocketTransport aborts a hung connect after the timeout and allows a fresh dial', async () => {
|
||||||
|
const events: string[] = [];
|
||||||
|
const errors: Error[] = [];
|
||||||
|
const sockets: HangingSocket[] = [];
|
||||||
|
const transport = new MpvSocketTransport({
|
||||||
|
socketPath: '/tmp/mpv.sock',
|
||||||
|
connectTimeoutMs: 5,
|
||||||
|
onConnect: () => {
|
||||||
|
events.push('connect');
|
||||||
|
},
|
||||||
|
onData: () => {},
|
||||||
|
onError: (error) => {
|
||||||
|
events.push('error');
|
||||||
|
errors.push(error);
|
||||||
|
},
|
||||||
|
onClose: () => {
|
||||||
|
events.push('close');
|
||||||
|
},
|
||||||
|
socketFactory: () => {
|
||||||
|
const socket = new HangingSocket();
|
||||||
|
sockets.push(socket);
|
||||||
|
return socket as unknown as net.Socket;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
transport.connect();
|
||||||
|
assert.equal(transport.isConnecting, true);
|
||||||
|
|
||||||
|
await wait(20);
|
||||||
|
|
||||||
|
assert.deepEqual(events, ['error', 'close']);
|
||||||
|
assert.match(errors[0]!.message, /connect timed out/);
|
||||||
|
assert.equal(sockets[0]!.destroyed, true);
|
||||||
|
assert.equal(transport.isConnecting, false);
|
||||||
|
assert.equal(transport.isConnected, false);
|
||||||
|
|
||||||
|
transport.connect();
|
||||||
|
assert.equal(transport.isConnecting, true);
|
||||||
|
assert.equal(sockets.length, 2);
|
||||||
|
assert.equal(sockets[1]!.connectedPaths.at(0), '/tmp/mpv.sock');
|
||||||
|
|
||||||
|
transport.shutdown();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('MpvSocketTransport does not fire the connect timeout after a successful connect', async () => {
|
||||||
|
const events: string[] = [];
|
||||||
|
const transport = new MpvSocketTransport({
|
||||||
|
socketPath: '/tmp/mpv.sock',
|
||||||
|
connectTimeoutMs: 5,
|
||||||
|
onConnect: () => {
|
||||||
|
events.push('connect');
|
||||||
|
},
|
||||||
|
onData: () => {},
|
||||||
|
onError: () => {
|
||||||
|
events.push('error');
|
||||||
|
},
|
||||||
|
onClose: () => {
|
||||||
|
events.push('close');
|
||||||
|
},
|
||||||
|
socketFactory: () => new FakeSocket() as unknown as net.Socket,
|
||||||
|
});
|
||||||
|
|
||||||
|
transport.connect();
|
||||||
|
await wait(20);
|
||||||
|
|
||||||
|
assert.deepEqual(events, ['connect']);
|
||||||
|
assert.equal(transport.isConnected, true);
|
||||||
|
});
|
||||||
|
|
||||||
test('MpvSocketTransport ignores stale socket events after shutdown and reconnect', async () => {
|
test('MpvSocketTransport ignores stale socket events after shutdown and reconnect', async () => {
|
||||||
const events: string[] = [];
|
const events: string[] = [];
|
||||||
const sockets: ManualCloseSocket[] = [];
|
const sockets: ManualCloseSocket[] = [];
|
||||||
|
|||||||