Compare commits

..

19 Commits

Author SHA1 Message Date
sudacode 84c75f50aa fix(release): skip gh attribution without CI token 2026-07-09 02:26:00 -07:00
sudacode 8b838f2c7d chore(release): prepare 0.18.0-beta.3 2026-07-09 01:20:02 -07:00
sudacode db4139ba0b fix(tokenizer): block kanji reading collisions and name/generic length t
- Add allowReadingOnlyMatch:false to kanji token known-word lookups so 渓谷/けいこく no longer matches a mined 警告/けいこく card via reading-only index
- Greedy name pre-pass yields when a strictly longer generic word starts at the same position (空 no longer splits 空気; ties still go to the name)
2026-07-09 01:03:58 -07:00
sudacode 6c251502b3 fix(tokenizer): exclude unparsed-run tokens from annotations and N+1 (#153) 2026-07-09 00:28:38 -07:00
sudacode ae40934d3a fix(tokenizer): greedy name pre-pass to prevent generic matches swallowing character names (#151) 2026-07-08 23:53:45 -07:00
sudacode cdb1475a54 chore(assets): update icons and favicons 2026-07-08 22:35:47 -07:00
sudacode a2e49b369b fix(tokenizer): merge scanner metadata per token instead of all-or-nothi
- Replace hasSameTokenSpans + full-discard with mergeScannerTokensIntoParseTokens
- Grafts isNameMatch/frequencyRank/etc onto matching parse spans; filler chunks degrade only themselves
- Fixes name annotations dropping for entire subtitle lines containing unmatched interjections
2026-07-08 22:26:34 -07:00
sudacode 7f13aed50a fix(overlay): keep frequency/JLPT highlight for kanji non-independent nouns (#150) 2026-07-08 22:25:32 -07:00
sudacode 8b21a2bca8 fix(overlay): remove content adverbs from annotation stop-word list
- 確かに and やはり no longer excluded from frequency/JLPT highlighting and vocab stats
- Stop-word list now covers only interjections, pronouns, and grammar fragments
2026-07-08 20:15:45 -07:00
sudacode 925413adfe chore(release): prepare 0.18.0-beta.2 2026-07-08 02:31:14 -07:00
sudacode d0644ab2eb fix(stats): parse v3 reading-aware known-word cache in stats server (#149) 2026-07-08 02:15:58 -07:00
sudacode d253710c2e fix(stats): fetch cover art eagerly at session start instead of on series page visit (#148) 2026-07-08 00:59:35 -07:00
sudacode c3df510e4f docs(release): reclassify audio normalization as added, not fixed
- Move card audio normalization entry from Fixed → Added in prerelease notes
- Update changes/audio-normalization.md type: fixed → added
2026-07-08 00:38:38 -07:00
sudacode 187f68e5b6 fix(tokenizer): prevent grammar tokens from borrowing known-word highlight via unrelated readings (#147) 2026-07-07 23:57:47 -07:00
sudacode 0e254cbbef fix(launcher): move fzf previews below menus 2026-07-07 22:36:56 -07:00
sudacode 7b94adafbd chore(release): prepare 0.18.0-beta.1 2026-07-07 02:40:05 -07:00
sudacode 61f39d1e09 fix(overlay): resolve unspaced Japanese name splits and scan recovery (#146) 2026-07-07 02:33:46 -07:00
sudacode e7739de51c fix(tokenizer): keep unparsed Yomitan tokens hoverable (#145) 2026-07-07 02:01:04 -07:00
sudacode ad1d240f20 refactor(tests): centralize lane definitions and add per-file isolation (#143) 2026-07-07 01:51:05 -07:00
70 changed files with 3087 additions and 493 deletions
+3 -3
View File
@@ -55,9 +55,6 @@ jobs:
- name: Verify generated config examples
run: bun run verify:config-example
- name: Internal docs knowledge-base checks
run: bun run test:docs:kb
- name: Test suite (source)
run: bun run test:fast
@@ -71,6 +68,9 @@ jobs:
path: coverage/test-src/lcov.info
if-no-files-found: error
- name: Stats UI tests
run: bun run test:stats
- name: Launcher smoke suite (source)
run: bun run test:launcher:smoke:src
+3
View File
@@ -70,6 +70,9 @@ jobs:
path: coverage/test-src/lcov.info
if-no-files-found: error
- name: Stats UI tests
run: bun run test:stats
- name: Launcher smoke suite (source)
run: bun run test:launcher:smoke:src
+3
View File
@@ -61,6 +61,9 @@ jobs:
path: coverage/test-src/lcov.info
if-no-files-found: error
- name: Stats UI tests
run: bun run test:stats
- name: Launcher smoke suite (source)
run: bun run test:launcher:smoke:src
+3
View File
@@ -42,7 +42,10 @@ Start here, then leave this file.
- Config/schema/defaults: `bun run test:config`; if template/defaults changed, `bun run generate:config-example`
- Launcher/plugin: `bun run test:launcher` or `bun run test:env`
- Runtime-compat / dist-sensitive: `bun run test:runtime:compat`
- Stats dashboard UI (`stats/`): `bun run test:stats`
- Build/release scripts (`scripts/**`): `bun run test:scripts`
- Docs-only: `bun run docs:test`, then `bun run docs:build`
- Test lanes are directory-discovered via `scripts/test-lanes.ts`; never hand-list test files in `package.json`
## Docs Upkeep
Binary file not shown.

Before

Width:  |  Height:  |  Size: 436 B

After

Width:  |  Height:  |  Size: 253 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 549 B

After

Width:  |  Height:  |  Size: 366 B

+1 -1
View File
@@ -1,4 +1,4 @@
type: fixed
type: added
area: mining
- Normalized generated card audio by default during media extraction, with `ankiConnect.media.normalizeAudio` available to keep raw source loudness when needed.
@@ -0,0 +1,6 @@
type: fixed
area: overlay
- Fixed character-dictionary surname/given-name splitting for unspaced AniList native names where romanized reading length misled the boundary guess (東紫乃 split as 東紫+乃, 渡辺真奈美 as 渡辺真+奈美), so surnames like 東 and 渡辺 were never highlighted with character portraits and unmatched leading kanji (渡) lost hover lookup entirely. Name parts are now resolved with MeCab person-name POS tags and reading validation when MeCab is available; without MeCab, terms are generated for both plausible boundaries so the real surname still matches. Existing snapshots regenerate automatically (format version bump), and snapshots built without MeCab are upgraded to exact MeCab splits once MeCab is detected.
- Fixed subtitle tokenization dropping a character entirely when Yomitan's greedy scan consumed trailing punctuation/whitespace past the term (e.g. 平 in 「(東)平 (平)え?」 matched across the paren with no exact-source headword). The scanner now retries shorter windows so the prefix term still tokenizes with its name match and portrait.
- Fixed character-name highlights being stripped when MeCab's context-dependent POS tag put the token in the annotation noise filter (e.g. 平 tagged as prefix 接頭詞 in 「あっ 平 これ…」 lost its highlight and portrait even though hover lookup showed the character). Confirmed name matches now survive the POS exclusion filter, and name matching alone keeps the annotation stage active when all other annotations are disabled.
@@ -0,0 +1,4 @@
type: fixed
area: overlay
- Removed 確かに and やはり from the annotation stop-word list so they get frequency/JLPT highlighting again and count toward vocabulary stats; the list now only covers interjections, pronouns, and grammar fragments.
+4
View File
@@ -0,0 +1,4 @@
type: fixed
area: stats
- Cover art is now fetched eagerly when a new series starts playing, instead of waiting for the first visit to its series detail page, so the stats timeline shows the best-guess AniList image right away. The stats covers endpoint also backfills missing series art in the background, so existing series without an image pick one up on the next stats page load.
+4
View File
@@ -0,0 +1,4 @@
type: changed
area: launcher
- Moved fzf previews below launcher menus so long titles and metadata have more horizontal room.
+4
View File
@@ -0,0 +1,4 @@
type: fixed
area: overlay
- Fixed character names being swallowed by longer generic dictionary matches during subtitle tokenization (e.g. ヨータ in 美姫とヨータ was lost because とヨー normalizes to とよう and matched 渡洋), so the name never tokenized and got no highlight, portrait, or hover lookup. Character-dictionary matches now claim their spans in a greedy pre-pass before the left-to-right scan, and re-segmented name spans override the parse tokenization on merge. A name only claims its span when no strictly longer generic word starts at the same position, so a character named 空 no longer splits 空気 (ties still go to the name). The pre-pass runs only when a SubMiner character dictionary is enabled in the active Yomitan profile.
@@ -0,0 +1,4 @@
type: fixed
area: overlay
- Kanji-bearing nouns that MeCab tags as non-independent (非自立) — e.g. 日 in いい日だったな, 点, 以外 — now keep frequency/JLPT highlighting and count toward vocabulary stats. Yomitan segments them as standalone vocabulary tokens, so the MeCab POS filter only suppresses kana grammar nouns (こと, もの, とき) it was meant for.
+5 -2
View File
@@ -1,4 +1,7 @@
type: fixed
type: changed
area: overlay
- Fixed words being highlighted green as known when a same-spelled Anki card taught a different reading (e.g. とこ parsed as 床 "bed" matching a known 床/ゆか "floor" card). The known-word cache now stores each card's word together with its reading and only matches when the token's reading agrees; cards without a reading field keep matching in any reading as before.
- New reading-aware subtitle parsing and known-word matching: the known-word cache now stores each Anki card's word together with its reading (cache format v3), and a token only gets the known-word highlight when its parsed reading agrees with the card. The cache and the stats server upgrade automatically.
- Fixes words being highlighted green as known when a same-spelled Anki card taught a different reading (e.g. とこ parsed as 床 "bed" no longer matches a known 床/ゆか "floor" card). Cards without a reading field keep matching in any reading as before.
- Single-kana grammar tokens (よ in 全然いいよ, standalone え) no longer borrow the reading of an unrelated card (such as 夜 or 絵) and get painted as known: reading-only matching requires at least two kana, while single-kana cards still match by their word field.
- Kanji words no longer count as known just because a different mined kanji word shares their reading (渓谷/けいこく no longer matches a known 警告/けいこく card). Words mined in kana form still highlight their kanji spelling in subtitles, and kana subtitle text still matches cards mined in kanji form.
+4
View File
@@ -0,0 +1,4 @@
type: fixed
area: overlay
- Fixed character-name annotations dropping for an entire subtitle line when it contained any chunk the dictionary scanner could not match (e.g. an interjection like やほっ before a name): scanner metadata is now merged per token into the parseText segmentation instead of being discarded on any span mismatch.
+4
View File
@@ -0,0 +1,4 @@
type: changed
area: overlay
- Standalone suffix tokens (MeCab pos2 接尾, e.g. さん, れる) are now excluded from JLPT/frequency/N+1 annotations by default, matching how particles and interjections are treated. Cache-backed known-word highlighting still applies; override via the pos2 exclusion config if you want them annotated.
+7
View File
@@ -0,0 +1,7 @@
type: internal
area: testing
- Test lanes are now defined once in `scripts/test-lanes.ts` and discovered by directory instead of hand-maintained file lists in `package.json`; the unused `test:core:*`/`test:config:dist`/`test:full` scripts were removed.
- `scripts/run-test-lane.mjs` runs each test file in an isolated `bun test` process with a wall timeout, so a hanging test or leaked global can no longer cascade failures across the lane.
- Previously orphaned suites now run in CI: the stats dashboard tests (`bun run test:stats`), the `scripts/**` tests (`bun run test:scripts`, including the change-verification skill tests), the `test-plugin-process-start-retries.lua` plugin test, and the runtime-compat dist slice (now part of `bun run test:fast`).
- The change-verification skill gained a `stats` lane for `stats/` edits.
+4
View File
@@ -0,0 +1,4 @@
type: fixed
area: overlay
- Fixed subtitle text that Yomitan's parser cannot match (truncated inflections like the volitional in とこ戻ろ…, elongation runs like ぅ~/ぉ〜) being rendered as plain, non-interactive text with no hover lookup. Unparsed runs are now kept as hoverable tokens matching Yomitan's own segmentation, while being fully ignored by frequency/JLPT highlighting, the N+1 candidate math, and vocabulary stats. Bracketed SFX/speaker captions and punctuation-only runs are still skipped.
+2
View File
@@ -56,6 +56,8 @@ A single character produces many searchable terms so that names are recognized r
- Family name alone: 須々木
- Given name alone: 心一
Unspaced native names (AniList often stores 渡辺真奈美 without a separator) are split into family/given parts with MeCab when it is available: person-name POS tags (姓/名) decide the boundary, validated against AniList's romanized first/last name readings. Without MeCab, a length heuristic based on the romanized readings guesses the boundary — and because that guess can be ambiguous (東紫乃 could be 東+紫乃 or 東紫+乃), terms are generated for the top two candidate boundaries so the real surname still matches. Snapshots built without MeCab are regenerated automatically once MeCab becomes available, upgrading them to the exact splits.
**Middle-dot removal** (common in katakana foreign names):
- ア・リ・ス → アリス (combined), plus individual segments
+11 -7
View File
@@ -80,18 +80,24 @@ Default lanes:
```bash
bun run test # alias for test:fast
bun run test:fast # default fast lane
bun run test:full # maintained source + launcher-unit + runtime compat surface
bun run test:fast # full source lanes: src + launcher-unit + scripts + runtime compat
bun run test:runtime:compat # compiled/runtime compatibility slice only
bun run test:env # launcher/plugin + env-sensitive verification
bun run test:stats # stats dashboard UI suite
bun run test:immersion:sqlite # SQLite persistence lane
bun run test:subtitle # maintained alass/ffsubsync subtitle surface
```
- `bun run test` and `bun run test:fast` cover config/core suites plus representative entry/runtime, Anki integration, release-workflow coverage, typecheck, and runtime-registry checks.
- `bun run test:full` is the maintained full surface: Bun-compatible `src/**` discovery, Bun-compatible launcher unit discovery, and the compiled/runtime compatibility lane for suites routed through `dist/**`.
Test lane membership is defined once in `scripts/test-lanes.ts` and discovered by
directory, so new test files join their lane automatically. `scripts/run-test-lane.mjs`
runs each test file in its own `bun test` process (per-file isolation) so a hanging
test or leaked global in one file cannot cascade into the rest of the lane; pass
`--jobs N` to parallelize or `--single-process` for one shared process.
- `bun run test` and `bun run test:fast` cover the full discovered `src/**` suite, launcher unit tests, `scripts/**` tests, and the compiled/runtime compatibility lane.
- `bun run test:runtime:compat` covers the compiled/runtime slice directly: `ipc`, `anki-jimaku-ipc`, `overlay-manager`, `config-validation`, `startup-config`, and `registry`.
- `bun run test:env` covers environment-sensitive checks: launcher smoke/plugin verification plus the Bun source SQLite lane.
- `bun run test:stats` runs the stats dashboard suite under `stats/src/**`.
- `bun run test:immersion:sqlite` is the reproducible persistence lane when you need real DB-backed SQLite coverage under Bun.
The Bun-managed discovery lanes intentionally exclude a small compiled/runtime-focused set: `src/core/services/ipc.test.ts`, `src/core/services/anki-jimaku-ipc.test.ts`, `src/core/services/overlay-manager.test.ts`, `src/main/config-validation.test.ts`, `src/main/runtime/startup-config.test.ts`, and `src/main/runtime/registry.test.ts`. `bun run test:runtime:compat` keeps them in the standard workflow via `dist/**`.
@@ -126,11 +132,11 @@ Focused commands:
```bash
bun run test:config # Source-level config schema/validation tests
bun run test:launcher # Launcher regression tests (config discovery + command routing)
bun run test:core # Source-level core regression tests (default lane)
bun run test:launcher:smoke:src # Launcher e2e smoke: launcher -> mpv IPC -> overlay start/stop wiring
bun run test:launcher:env:src # Launcher smoke + Lua plugin gate
bun run test:src # Bun-managed maintained src/** discovery lane
bun run test:launcher:unit:src # Bun-managed maintained launcher unit lane
bun run test:scripts # Bun-managed scripts/** test lane
bun run test:immersion:sqlite:src # Bun source lane
```
@@ -144,8 +150,6 @@ Smoke and optional deep dist commands:
bun run build # compile dist artifacts
bun run test:immersion:sqlite # compile + run SQLite-backed immersion tests under Bun
bun run test:smoke:dist # explicit smoke scope for compiled runtime
bun run test:config:dist # optional full dist config suite
bun run test:core:dist # optional full dist core suite
```
Use `bun run test:immersion:sqlite` when you need real DB-backed coverage for the immersion tracker.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.0 KiB

After

Width:  |  Height:  |  Size: 8.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 862 B

After

Width:  |  Height:  |  Size: 633 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.6 KiB

After

Width:  |  Height:  |  Size: 1.4 KiB

+15 -1
View File
@@ -3,10 +3,22 @@
# Verification
Status: active
Last verified: 2026-05-23
Last verified: 2026-07-06
Owner: Kyle Yasuda
Read when: selecting the right verification lane for a change
## Lane Infrastructure
- Lane membership is defined once in `scripts/test-lanes.ts` and discovered by
directory — new test files join their lane automatically; never hand-list test
files in `package.json`.
- `scripts/run-test-lane.mjs` runs each test file in its own `bun test` process
(per-file isolation with a wall timeout) so a hanging test or leaked global in
one file cannot cascade into the rest of the lane. `--jobs N` parallelizes;
`--single-process` restores the shared-process mode for debugging.
- `bun run test:fast` is the full source gate: discovered `src/**`, launcher
unit, `scripts/**`, and the compiled runtime-compat slice.
## Default Handoff Gate
```bash
@@ -31,6 +43,8 @@ bun run docs:build
- Config/schema/defaults: `bun run test:config`, then `bun run generate:config-example` if template/defaults changed
- Launcher/plugin: `bun run test:launcher` or `bun run test:env`
- Runtime-compat / compiled behavior: `bun run test:runtime:compat`
- Stats dashboard UI: `bun run test:stats`
- Build/release scripts (`scripts/**`): `bun run test:scripts`
- Coverage for the maintained source lane: `bun run test:coverage:src`
- Deep/local full gate: default handoff gate above
+46 -16
View File
@@ -65,11 +65,36 @@ function makeTestEnv(homeDir: string, xdgConfigHome: string): NodeJS.ProcessEnv
APPDATA: xdgConfigHome,
LOCALAPPDATA: path.join(homeDir, 'AppData', 'Local'),
XDG_CONFIG_HOME: xdgConfigHome,
// Pin the data dir under the temp home so the Linux runtime-plugin preflight
// resolves managed asset paths deterministically (not the CI runner's).
XDG_DATA_HOME: path.join(homeDir, '.local', 'share'),
PATH: pathValue,
Path: pathValue,
};
}
// On Linux the playback path runs `ensureLinuxRuntimePluginAvailable`, which —
// when the runtime plugin/theme are missing — spawns the app with
// `--ensure-linux-runtime-plugin-assets` and polls up to 30s
// (RESPONSE_TIMEOUT_MS) for an install response. A fake app that just exits
// never writes that response, so the launcher hangs and the test times out on
// Linux CI (the preflight is a no-op on macOS/Windows). This shell prelude makes
// the fake app install the managed plugin/theme and write the response, matching
// launcher/smoke.e2e.test.ts. Prepend it to each fake app that reaches playback.
const RUNTIME_PLUGIN_PREFLIGHT_SH = `if [ "$1" = "--ensure-linux-runtime-plugin-assets" ]; then
data="\${XDG_DATA_HOME:-$HOME/.local/share}/SubMiner"
mkdir -p "$data/plugin/subminer" "$data/themes"
printf -- '-- test plugin\\n' > "$data/plugin/subminer/main.lua"
printf 'test=true\\n' > "$data/plugin/subminer.conf"
printf '/* test theme */\\n' > "$data/themes/subminer.rasi"
if [ "$2" = "--ensure-linux-runtime-plugin-assets-response-path" ] && [ -n "$3" ]; then
mkdir -p "$(dirname "$3")"
printf '{"ok":true,"status":"installed","path":"%s"}' "$data/plugin/subminer/main.lua" > "$3"
fi
exit 0
fi
`;
test('config path uses XDG_CONFIG_HOME override', () => {
withTempDir((root) => {
const xdgConfigHome = path.join(root, 'xdg');
@@ -237,7 +262,7 @@ test('doctor refresh-known-words forwards app refresh command without requiring
const capturePath = path.join(root, 'captured-args.txt');
fs.writeFileSync(
appPath,
'#!/bin/sh\nif [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n',
`#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}if [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n`,
);
fs.chmodSync(appPath, 0o755);
@@ -264,7 +289,7 @@ test('launcher settings option forwards app settings window command', () => {
const capturePath = path.join(root, 'captured-args.txt');
fs.writeFileSync(
appPath,
'#!/bin/sh\nif [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n',
`#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}if [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n`,
);
fs.chmodSync(appPath, 0o755);
@@ -288,7 +313,7 @@ test('launcher settings command forwards app settings window command', () => {
const capturePath = path.join(root, 'captured-args.txt');
fs.writeFileSync(
appPath,
'#!/bin/sh\nif [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n',
`#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}if [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n`,
);
fs.chmodSync(appPath, 0o755);
@@ -369,7 +394,7 @@ test('launcher forwards --args to mpv as parsed tokens', { timeout: 15000 }, ()
},
}),
);
fs.writeFileSync(appPath, '#!/bin/sh\nexit 0\n');
fs.writeFileSync(appPath, `#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}exit 0\n`);
fs.chmodSync(appPath, 0o755);
fs.writeFileSync(
@@ -460,7 +485,7 @@ test('launcher forwards non-info log level into mpv logging args', { timeout: 15
},
}),
);
fs.writeFileSync(appPath, '#!/bin/sh\nexit 0\n');
fs.writeFileSync(appPath, `#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}exit 0\n`);
fs.chmodSync(appPath, 0o755);
fs.writeFileSync(
@@ -539,7 +564,7 @@ test('launcher routes youtube urls through regular playback startup', { timeout:
);
fs.writeFileSync(
appPath,
'#!/bin/sh\nif [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n',
`#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}if [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n`,
);
fs.chmodSync(appPath, 0o755);
@@ -566,17 +591,22 @@ ${bunBinary} -e "const net=require('node:net'); const fs=require('node:fs'); con
fs.chmodSync(path.join(binDir, 'yt-dlp'), 0o755);
fs.chmodSync(path.join(binDir, 'ffmpeg'), 0o755);
// Note: no SUBMINER_TEST_CAPTURE here. When set, the launcher intercepts
// *every* app command — including the Linux runtime-plugin preflight's
// `--ensure-linux-runtime-plugin-assets` install — and returns without
// running the fake app, so the preflight would poll 30s for a response that
// never arrives and time out. This test asserts on the mpv args, not on
// captured app args, so capture isn't needed.
const env = {
...makeTestEnv(homeDir, xdgConfigHome),
PATH: `${binDir}${path.delimiter}${process.env.Path || process.env.PATH || ''}`,
Path: `${binDir}${path.delimiter}${process.env.Path || process.env.PATH || ''}`,
DISPLAY: ':99',
XDG_SESSION_TYPE: 'x11',
SUBMINER_APPIMAGE_PATH: appPath,
SUBMINER_TEST_MPV_ARGS: mpvArgsPath,
SUBMINER_TEST_CAPTURE: path.join(root, 'captured-args.txt'),
};
const result = runLauncher(['https://www.youtube.com/watch?v=abc123'], env);
// Pass an explicit backend so overlay startup doesn't probe for a display
// (headless CI has none), matching launcher/smoke.e2e.test.ts.
const result = runLauncher(['--backend', 'x11', 'https://www.youtube.com/watch?v=abc123'], env);
assert.equal(result.status, 0, `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`);
const forwardedArgs = fs
@@ -597,7 +627,7 @@ test('dictionary command forwards --dictionary and --dictionary-target to app co
const capturePath = path.join(root, 'captured-args.txt');
fs.writeFileSync(
appPath,
'#!/bin/sh\nif [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n',
`#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}if [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n`,
);
fs.chmodSync(appPath, 0o755);
@@ -626,7 +656,7 @@ test('dictionary command forwards manual AniList selection modes to app command
const capturePath = path.join(root, 'captured-args.txt');
fs.writeFileSync(
appPath,
'#!/bin/sh\nif [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n',
`#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}if [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n`,
);
fs.chmodSync(appPath, 0o755);
@@ -763,7 +793,7 @@ test('jellyfin discovery routes to app --background and remote announce with log
const capturePath = path.join(root, 'captured-args.txt');
fs.writeFileSync(
appPath,
'#!/bin/sh\nif [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n',
`#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}if [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n`,
);
fs.chmodSync(appPath, 0o755);
@@ -790,7 +820,7 @@ test('jellyfin discovery via jf alias forwards remote announce for cast visibili
const capturePath = path.join(root, 'captured-args.txt');
fs.writeFileSync(
appPath,
'#!/bin/sh\nif [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n',
`#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}if [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n`,
);
fs.chmodSync(appPath, 0o755);
@@ -817,7 +847,7 @@ test('jellyfin login routes credentials to app command', () => {
const capturePath = path.join(root, 'captured-args.txt');
fs.writeFileSync(
appPath,
'#!/bin/sh\nif [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n',
`#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}if [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n`,
);
fs.chmodSync(appPath, 0o755);
@@ -856,7 +886,7 @@ test('jellyfin setup forwards password-store to app command', () => {
const capturePath = path.join(root, 'captured-args.txt');
fs.writeFileSync(
appPath,
'#!/bin/sh\nif [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n',
`#!/bin/sh\n${RUNTIME_PLUGIN_PREFLIGHT_SH}if [ -n "$SUBMINER_TEST_CAPTURE" ]; then printf "%s\\n" "$@" > "$SUBMINER_TEST_CAPTURE"; fi\nexit 0\n`,
);
fs.chmodSync(appPath, 0o755);
+2 -2
View File
@@ -56,7 +56,7 @@ export function showFzfFlatMenu(
`--prompt=${prompt}`,
'--delimiter=\t',
'--with-nth=2',
'--preview-window=right:50%:wrap',
'--preview-window=down:50%:wrap',
'--preview',
previewCommand,
];
@@ -468,7 +468,7 @@ thumb=$(get_thumb)
'--prompt=Select Video: ',
'--delimiter=\t',
'--with-nth=1',
'--preview-window=right:50%:wrap',
'--preview-window=down:50%:wrap',
'--preview',
previewCmd,
],
+8 -11
View File
File diff suppressed because one or more lines are too long
@@ -105,6 +105,8 @@ bash plugins/subminer-workflow/skills/subminer-change-verification/scripts/verif
- For `docs-site/`, `docs/`, and doc-only edits.
- `config`
- For `src/config/` and config-template-sensitive edits.
- `stats`
- For `stats/` dashboard UI edits.
- `core`
- For general source changes where `typecheck` + `test:fast` is the best cheap signal.
- `launcher-plugin`
@@ -108,6 +108,14 @@ for path in "${PATHS[@]}"; do
;;
esac
case "$path" in
stats/*)
add_lane "stats"
add_reason "$path -> stats"
specialized=1
;;
esac
case "$path" in
launcher/*|plugin/subminer/*|plugin/subminer.conf|scripts/test-plugin-*|scripts/get-mpv-window-*|scripts/configure-plugin-binary-path.mjs)
add_lane "launcher-plugin"
@@ -255,8 +255,18 @@ write_summary_files() {
local lane_lines
lane_lines=$(printf '%s\n' "${SELECTED_LANES[@]}")
printf '%s\n' "$lane_lines" >"$ARTIFACT_DIR/lanes.txt"
printf '%s\n' "${BLOCKERS[@]}" >"$ARTIFACT_DIR/blockers.txt"
printf '%s\n' "${PATH_ARGS[@]}" >"$ARTIFACT_DIR/requested-paths.txt"
# bash 3.2 raises "unbound variable" under set -u when expanding an empty
# array, so guard on length (matching the idiom used elsewhere here).
if [[ ${#BLOCKERS[@]} -gt 0 ]]; then
printf '%s\n' "${BLOCKERS[@]}" >"$ARTIFACT_DIR/blockers.txt"
else
: >"$ARTIFACT_DIR/blockers.txt"
fi
if [[ ${#PATH_ARGS[@]} -gt 0 ]]; then
printf '%s\n' "${PATH_ARGS[@]}" >"$ARTIFACT_DIR/requested-paths.txt"
else
: >"$ARTIFACT_DIR/requested-paths.txt"
fi
ARTIFACT_DIR_ENV="$ARTIFACT_DIR" \
SESSION_ID_ENV="$SESSION_ID" \
@@ -482,6 +492,9 @@ for lane in "${SELECTED_LANES[@]}"; do
config)
run_step "$lane" "config" "bun run test:config" || break
;;
stats)
run_step "$lane" "stats" "bun run test:stats" || break
;;
core)
run_step "$lane" "typecheck" "bun run typecheck" || break
run_step "$lane" "fast-tests" "bun run test:fast" || break
+59 -10
View File
@@ -1,24 +1,73 @@
> This is a prerelease build for testing. Stable changelog and docs-site updates remain pending until the final stable release.
<!-- prerelease-base-version: 0.17.1 -->
<!-- prerelease-base-version: 0.18.0 -->
## Highlights
### Added
- **Watch History Browser**
- New `subminer -H` / `--history` command lets you browse your local watch history, replay the last episode, jump to the next one, or pick an episode via fzf or rofi.
- The rofi picker now shows AniList cover art for each show, making it easier to spot the right title at a glance.
- **Card Audio Normalization**
- Audio extracted for Anki cards is now volume-normalized by default, giving more consistent playback loudness across cards.
- Prefer the original source volume? Disable it via the new `ankiConnect.media.normalizeAudio` setting.
- **YouTube Media Cache Mode**: A new `youtube.mediaCache.mode` setting (`direct` or `background`) lets you choose how SubMiner extracts audio and image from YouTube cards.
- In background mode, SubMiner creates a text-only card immediately, downloads a yt-dlp media cache (capped at 720p by default), and fills audio and image fields once the file is ready — with overlay and OSD notifications when the download starts and when media is available.
- If a background download fails, SubMiner now notifies you and clears any pending media updates rather than leaving cards silently incomplete.
### Changed
- **New App Icon**
- SubMiner now ships pixel-art submarine artwork contributed by an anonymous community member.
- Applied across the app icon, tray icon, notifications, README, docs site, and stats page.
- **Launcher Preview Layout**
- fzf previews in the launcher now sit below the menu instead of beside it, giving long titles and metadata more horizontal room.
### Fixed
- **Log Export**: Log filenames now use your local date, so exporting logs near midnight no longer pulls stale files from the previous UTC day. Export redaction has also been expanded to mask a broader range of sensitive data, including IP addresses, email addresses, authentication and cookie headers, yt-dlp cookie arguments, URL credentials, and signed YouTube media URLs.
- **YouTube Card Media Reliability**: Direct stream extraction now uses safer ffmpeg options and skips stale or cached stream map entries to reduce failed media generation. Background cache downloads are hardened with IPv4 and extractor retry flags, stale cache files are cleaned up on startup and before new downloads, and in-flight background downloads are stopped automatically when switching back to direct mode.
- **Character Name Highlighting in Subtitles**
- Fixed unspaced Japanese names (e.g. 東紫乃, 渡辺真奈美) being split at the wrong point, which left surnames like 東 and 渡辺 without their character portrait or hover lookup.
- Fixed names getting cut off or losing their highlight when caught by the subtitle scanner's punctuation handling, misclassified by grammar tagging, or swallowed entirely by a longer generic dictionary match (e.g. ヨータ disappearing inside a false とヨー match).
- Fixed a single unrecognized word in a subtitle line (like a stray interjection) causing character-name highlighting to drop for the whole line instead of just that word.
- No action needed — existing data upgrades automatically the next time a matching name is seen.
- **Known-Word Highlighting**
- Words are no longer marked "known" (green) just because they share spelling with a known Anki card that actually teaches a different reading (e.g. 床 read as とこ no longer falsely matches a known 床/ゆか card).
- Kanji words are also no longer marked known just because a different mined word happens to share their reading (e.g. 渓谷/けいこく no longer falsely matches a known 警告/けいこく card).
- Single-kana grammar tokens (particles like よ, え) no longer borrow an unrelated card's reading and get falsely painted as known.
- Stats sessions now correctly reflect known-word counts again after the reading-aware matching upgrade, instead of showing 0 everywhere.
- **Annotation Highlighting Refinements**
- Restored frequency/JLPT highlighting and vocabulary-stat counting for words like 確かに and やはり, which were wrongly treated as grammar noise.
- Kanji nouns that MeCab tags as "non-independent" (e.g. 日, 点, 以外) also keep their highlighting and stats counting again.
- Suffix-only tokens (e.g. さん, れる) are now excluded from JLPT/frequency highlighting by default to match how particles and interjections are treated; known-word highlighting for them still works, and this is configurable.
- **Unparsed Subtitle Text**
- Subtitle text the dictionary can't recognize (like a truncated verb form) is now still hoverable for lookup and correctly counted toward a sentence's difficulty, instead of showing as dead, non-interactive text.
- **Kiku Manual Field Grouping**
- Fixed the field-grouping dialog getting stuck invisible behind fullscreen video on Hyprland/Wayland, and failing silently on repeated attempts after the first use.
- Fixed a timed-out or failed grouping request leaving an invisible, stuck dialog covering the video; it now closes automatically so the overlay recovers.
- Fixed a duplicate "Field grouping cancelled" notification appearing when grouping was cancelled via the trigger shortcut, and added a proper error message for the previously-silent case where the original card can no longer be loaded.
- **Secondary Subtitles**
- Karaoke-style secondary subtitles (common in opening/ending songs) no longer spam dozens of lines down the screen; repeated lines are now collapsed and the subtitle area is capped to a strip at the top.
- **YouTube Extraction**
- Fixed direct YouTube stream extraction occasionally corrupting the stream URL and causing failed audio/video capture.
- **Background Stats Server**
- Launching SubMiner in the background now correctly auto-starts the stats server when enabled, and won't start a duplicate if one's already running.
- **Stats Trend Charts**
- All trend chart titles now show by default, with the ability to hide specific titles (remembered across sessions) and cap how many top titles a chart displays.
- **Stats Cover Art**
- Cover art now loads as soon as a series starts playing instead of waiting for your first visit to its detail page, so the stats timeline shows artwork right away.
- Existing series missing art are backfilled automatically the next time you open the stats page.
## What's Changed
- feat(youtube): add mediaCache mode and safer stream media extraction by @ksyasuda in #130
- fix(logs): use local date for log filenames and expand export redaction by @ksyasuda in #131
- fix(youtube): parse mpv EDL stream URLs with byte-length guards by @ksyasuda in #134
- Normalize generated Anki audio by default by @ksyasuda in #135
- feat(launcher): add -H/--history command to browse local watch history by @ksyasuda in #136
- fix(overlay): prevent field grouping modal from freezing overlay on Hyprland by @ksyasuda in #138
- fix(overlay): collapse karaoke syllable spam in secondary subtitles by @ksyasuda in #139
- feat(stats): Trends dashboard overhaul — title visibility, ranking modes, calendar-accurate windows, tooltips by @ksyasuda in #140
- feat(branding): replace app icon with contributed pixel-art set by @ksyasuda in #141
- feat(anki): reading-aware known-word matching (cache v3) by @ksyasuda in #142
- fix(stats): start stats server on background app launch by @ksyasuda in #144
- fix(tokenizer): keep unparsed Yomitan tokens hoverable by @ksyasuda in #145
- fix(overlay): resolve unspaced Japanese name splits and scan recovery by @ksyasuda in #146
- fix(tokenizer): prevent grammar tokens from borrowing known-word highlight via unrelated readings by @ksyasuda in #147
- fix(stats): fetch cover art eagerly at session start instead of on series page visit by @ksyasuda in #148
- fix(overlay): keep frequency/JLPT highlight for kanji non-independent nouns by @ksyasuda in #150
- fix(tokenizer): greedy name pre-pass to prevent generic matches swallowing character names by @ksyasuda in #151
## Installation
+96
View File
@@ -1213,6 +1213,102 @@ test('writeChangelogArtifacts appends contributor attribution and a new-contribu
}
});
test('writeChangelogArtifacts skips contributor attribution in GitHub Actions without a token', async () => {
const { writeChangelogArtifacts } = await loadModule();
const workspace = createWorkspace('release-notes-actions-no-token');
const projectRoot = path.join(workspace, 'SubMiner');
const originalActions = process.env.GITHUB_ACTIONS;
const originalGhToken = process.env.GH_TOKEN;
const originalGithubToken = process.env.GITHUB_TOKEN;
const originalPath = process.env.PATH;
const originalWarn = console.warn;
const warnings: string[] = [];
fs.mkdirSync(path.join(projectRoot, 'changes'), { recursive: true });
fs.writeFileSync(path.join(projectRoot, 'CHANGELOG.md'), '# Changelog\n', 'utf8');
fs.writeFileSync(
path.join(projectRoot, 'changes', '001.md'),
['type: added', 'area: release', '', '- Added a feature.'].join('\n'),
'utf8',
);
try {
process.env.GITHUB_ACTIONS = 'true';
delete process.env.GH_TOKEN;
delete process.env.GITHUB_TOKEN;
process.env.PATH = workspace;
console.warn = (message?: unknown) => {
warnings.push(String(message));
};
writeChangelogArtifacts({
cwd: projectRoot,
version: '0.6.0',
date: '2026-05-06',
deps: { runClaude: defaultStubClaude().runClaude },
});
assert.deepEqual(warnings, []);
const releaseNotes = fs.readFileSync(
path.join(projectRoot, 'release', 'release-notes.md'),
'utf8',
);
assert.doesNotMatch(releaseNotes, /## What's Changed/);
} finally {
console.warn = originalWarn;
if (originalActions === undefined) {
delete process.env.GITHUB_ACTIONS;
} else {
process.env.GITHUB_ACTIONS = originalActions;
}
if (originalGhToken === undefined) {
delete process.env.GH_TOKEN;
} else {
process.env.GH_TOKEN = originalGhToken;
}
if (originalGithubToken === undefined) {
delete process.env.GITHUB_TOKEN;
} else {
process.env.GITHUB_TOKEN = originalGithubToken;
}
if (originalPath === undefined) {
delete process.env.PATH;
} else {
process.env.PATH = originalPath;
}
fs.rmSync(workspace, { recursive: true, force: true });
}
});
test('shouldSkipDefaultContributionLookup skips GitHub Actions without a gh token', async () => {
const { shouldSkipDefaultContributionLookup } = await loadModule();
assert.equal(
shouldSkipDefaultContributionLookup({
GITHUB_ACTIONS: 'true',
GH_TOKEN: undefined,
GITHUB_TOKEN: undefined,
}),
true,
);
assert.equal(
shouldSkipDefaultContributionLookup({
GITHUB_ACTIONS: 'true',
GH_TOKEN: 'ghs_test',
GITHUB_TOKEN: undefined,
}),
false,
);
assert.equal(
shouldSkipDefaultContributionLookup({
GITHUB_ACTIONS: undefined,
GH_TOKEN: undefined,
GITHUB_TOKEN: undefined,
}),
false,
);
});
test('writeReleaseNotesForVersion preserves committed contributor attribution before installation', async () => {
const { writeReleaseNotesForVersion } = await loadModule();
const workspace = createWorkspace('release-notes-preserve-attribution');
+9
View File
@@ -345,6 +345,12 @@ function resolveFragmentRelativePath(fragmentPath: string, cwd: string): string
return path.relative(cwd, fragmentPath).split(path.sep).join('/');
}
export function shouldSkipDefaultContributionLookup(
env: Partial<Record<'GITHUB_ACTIONS' | 'GH_TOKEN' | 'GITHUB_TOKEN', string>> = process.env,
): boolean {
return env.GITHUB_ACTIONS === 'true' && !env.GH_TOKEN && !env.GITHUB_TOKEN;
}
// Walks git history + the GitHub API to attribute each released fragment to the
// PR (and author) that introduced it. One git call and one gh call per fragment,
// plus one gh call per unique author for the first-contribution check. Best
@@ -354,6 +360,9 @@ function defaultResolveContributions(fragmentPaths: string[], cwd: string): Cont
if (fragmentPaths.length === 0) {
return [];
}
if (shouldSkipDefaultContributionLookup()) {
return [];
}
try {
const slug = execFileSync(
+9 -66
View File
@@ -1,12 +1,7 @@
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { spawnSync } from 'node:child_process';
import { isAbsolute, join, relative, resolve } from 'node:path';
type LaneConfig = {
roots: string[];
include: string[];
exclude: Set<string>;
};
import { collectLaneFiles } from './test-lanes';
type LcovRecord = {
sourceFile: string;
@@ -18,64 +13,6 @@ type LcovRecord = {
const repoRoot = resolve(__dirname, '..');
const lanes: Record<string, LaneConfig> = {
'bun-src-full': {
roots: ['src'],
include: ['.test.ts', '.type-test.ts'],
exclude: new Set([
'src/core/services/anki-jimaku-ipc.test.ts',
'src/core/services/ipc.test.ts',
'src/core/services/overlay-manager.test.ts',
'src/main/config-validation.test.ts',
'src/main/runtime/registry.test.ts',
'src/main/runtime/startup-config.test.ts',
]),
},
'bun-launcher-unit': {
roots: ['launcher'],
include: ['.test.ts'],
exclude: new Set(['launcher/smoke.e2e.test.ts']),
},
};
function collectFiles(
rootDir: string,
includeSuffixes: string[],
excludeSet: Set<string>,
): string[] {
const out: string[] = [];
const visit = (currentDir: string) => {
for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
const fullPath = resolve(currentDir, entry.name);
if (entry.isDirectory()) {
visit(fullPath);
continue;
}
const relPath = relative(repoRoot, fullPath).replaceAll('\\', '/');
if (excludeSet.has(relPath)) continue;
if (includeSuffixes.some((suffix) => relPath.endsWith(suffix))) {
out.push(relPath);
}
}
};
visit(resolve(repoRoot, rootDir));
out.sort();
return out;
}
function getLaneFiles(laneName: string): string[] {
const lane = lanes[laneName];
if (!lane) {
throw new Error(`Unknown coverage lane: ${laneName}`);
}
const files = lane.roots.flatMap((rootDir) => collectFiles(rootDir, lane.include, lane.exclude));
if (files.length === 0) {
throw new Error(`No test files found for coverage lane: ${laneName}`);
}
return files;
}
function parseCoverageDirArg(argv: string[]): string {
for (let index = 0; index < argv.length; index += 1) {
if (argv[index] === '--coverage-dir') {
@@ -277,7 +214,13 @@ function runCoverageLane(): number {
rmSync(shardRoot, { recursive: true, force: true });
mkdirSync(shardRoot, { recursive: true });
const files = getLaneFiles(laneName);
let files: string[];
try {
files = collectLaneFiles(repoRoot, laneName);
} catch (error) {
process.stderr.write(`${error instanceof Error ? error.message : error}\n`);
return 1;
}
const reports: string[] = [];
try {
+133 -53
View File
@@ -1,73 +1,153 @@
import { readdirSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { relative, resolve } from 'node:path';
import { spawnSync } from 'node:child_process';
import { resolve } from 'node:path';
import { spawn, spawnSync } from 'node:child_process';
import { collectLaneFiles } from './test-lanes.ts';
// Runs a test lane with per-file process isolation: one `bun test` process per
// test file so a hanging test or leaked global in one file cannot poison the
// rest of the lane. Use --single-process for the old all-in-one-process mode.
//
// Usage: bun scripts/run-test-lane.mjs <lane> [--jobs N] [--timeout-secs N] [--single-process]
const repoRoot = resolve(fileURLToPath(new URL('..', import.meta.url)));
const lanes = {
'bun-src-full': {
roots: ['src'],
include: ['.test.ts', '.type-test.ts'],
exclude: new Set([
'src/core/services/anki-jimaku-ipc.test.ts',
'src/core/services/ipc.test.ts',
'src/core/services/overlay-manager.test.ts',
'src/main/config-validation.test.ts',
'src/main/runtime/registry.test.ts',
'src/main/runtime/startup-config.test.ts',
]),
},
'bun-launcher-unit': {
roots: ['launcher'],
include: ['.test.ts'],
exclude: new Set(['launcher/smoke.e2e.test.ts']),
},
};
// Cap per-file buffered output so a long or noisy test cannot grow the string
// without bound and exhaust memory.
const MAX_OUTPUT_BYTES = 1024 * 1024;
function collectFiles(rootDir, includeSuffixes, excludeSet) {
const out = [];
const visit = (currentDir) => {
for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
const fullPath = resolve(currentDir, entry.name);
if (entry.isDirectory()) {
visit(fullPath);
continue;
// Track spawned `bun test` children so we can kill them if the runner is
// interrupted, avoiding orphaned in-flight test processes.
const activeChildren = new Set();
function terminateChildren() {
for (const child of activeChildren) {
child.kill('SIGKILL');
}
activeChildren.clear();
}
for (const signal of ['SIGINT', 'SIGTERM']) {
process.on(signal, () => {
terminateChildren();
process.exit(130);
});
}
function parseArgs(argv) {
const options = { lane: undefined, jobs: 1, timeoutSecs: 300, singleProcess: false };
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === '--jobs') {
options.jobs = Math.max(1, Number(argv[(index += 1)]) || 1);
} else if (arg === '--timeout-secs') {
options.timeoutSecs = Math.max(1, Number(argv[(index += 1)]) || 300);
} else if (arg === '--single-process') {
options.singleProcess = true;
} else if (!arg.startsWith('--') && options.lane === undefined) {
options.lane = arg;
} else {
process.stderr.write(`Unknown argument: ${arg}\n`);
process.exit(1);
}
}
return options;
}
function runFile(file, timeoutSecs) {
return new Promise((resolvePromise) => {
const child = spawn('bun', ['test', `./${file}`], { cwd: repoRoot });
activeChildren.add(child);
let output = '';
let truncated = false;
let timedOut = false;
const append = (chunk) => {
if (truncated) return;
output += chunk;
if (output.length > MAX_OUTPUT_BYTES) {
output = `${output.slice(0, MAX_OUTPUT_BYTES)}\n[output truncated at ${MAX_OUTPUT_BYTES} bytes]\n`;
truncated = true;
}
const relPath = relative(repoRoot, fullPath).replaceAll('\\', '/');
if (excludeSet.has(relPath)) continue;
if (includeSuffixes.some((suffix) => relPath.endsWith(suffix))) {
out.push(relPath);
};
child.stdout.on('data', append);
child.stderr.on('data', append);
const timer = setTimeout(() => {
timedOut = true;
child.kill('SIGKILL');
}, timeoutSecs * 1000);
child.on('close', (code) => {
clearTimeout(timer);
activeChildren.delete(child);
resolvePromise({ file, code: timedOut ? 124 : (code ?? 1), output, timedOut });
});
child.on('error', (error) => {
clearTimeout(timer);
activeChildren.delete(child);
resolvePromise({ file, code: 1, output: String(error), timedOut: false });
});
});
}
async function runIsolated(files, options) {
const failures = [];
let nextIndex = 0;
let completed = 0;
async function worker() {
while (nextIndex < files.length) {
const file = files[nextIndex];
nextIndex += 1;
const result = await runFile(file, options.timeoutSecs);
completed += 1;
if (result.code !== 0) {
failures.push(result);
const reason = result.timedOut ? `timed out after ${options.timeoutSecs}s` : 'failed';
process.stderr.write(`\n[${completed}/${files.length}] ${file} ${reason}\n`);
process.stderr.write(result.output);
}
}
};
}
visit(resolve(repoRoot, rootDir));
out.sort();
return out;
await Promise.all(Array.from({ length: Math.min(options.jobs, files.length) }, worker));
if (failures.length > 0) {
process.stderr.write(`\n${failures.length} of ${files.length} test files failed:\n`);
for (const failure of failures) {
process.stderr.write(` ${failure.file}${failure.timedOut ? ' (timeout)' : ''}\n`);
}
return 1;
}
process.stdout.write(`All ${files.length} test files passed.\n`);
return 0;
}
const lane = lanes[process.argv[2]];
function runSingleProcess(files) {
const result = spawnSync('bun', ['test', ...files.map((file) => `./${file}`)], {
cwd: repoRoot,
stdio: 'inherit',
});
if (result.error) {
throw result.error;
}
return result.status ?? 1;
}
if (!lane) {
process.stderr.write(`Unknown test lane: ${process.argv[2] ?? '(missing)'}\n`);
const options = parseArgs(process.argv.slice(2));
if (!options.lane) {
process.stderr.write('Missing test lane name\n');
process.exit(1);
}
const files = lane.roots.flatMap((rootDir) => collectFiles(rootDir, lane.include, lane.exclude));
if (files.length === 0) {
process.stderr.write(`No test files found for lane: ${process.argv[2]}\n`);
let files;
try {
files = collectLaneFiles(repoRoot, options.lane);
} catch (error) {
process.stderr.write(`${error instanceof Error ? error.message : error}\n`);
process.exit(1);
}
const result = spawnSync('bun', ['test', ...files.map((file) => `./${file}`)], {
cwd: repoRoot,
stdio: 'inherit',
});
if (result.error) {
throw result.error;
if (options.singleProcess) {
process.exit(runSingleProcess(files));
}
process.exit(result.status ?? 1);
process.exit(await runIsolated(files, options));
+97
View File
@@ -0,0 +1,97 @@
import { readdirSync } from 'node:fs';
import { relative, resolve } from 'node:path';
export type TestLane = {
roots: string[];
include: string[];
exclude?: string[];
extraFiles?: string[];
};
// Single source of truth for test-lane membership. Consumed by
// scripts/run-test-lane.mjs (plain runs) and scripts/run-coverage-lane.ts
// (per-file coverage shards). Lanes discover files by directory so new test
// files join their lane automatically.
export const testLanes: Record<string, TestLane> = {
'bun-src-full': {
roots: ['src'],
include: ['.test.ts', '.type-test.ts'],
// Node-compat suites; their dist builds run via test:runtime:compat.
exclude: [
'src/core/services/anki-jimaku-ipc.test.ts',
'src/core/services/ipc.test.ts',
'src/core/services/overlay-manager.test.ts',
'src/main/config-validation.test.ts',
'src/main/runtime/registry.test.ts',
'src/main/runtime/startup-config.test.ts',
],
},
config: {
roots: ['src/config'],
include: ['.test.ts'],
extraFiles: ['src/generate-config-example.test.ts', 'src/verify-config-example.test.ts'],
},
launcher: {
roots: ['launcher'],
include: ['.test.ts'],
},
'bun-launcher-unit': {
roots: ['launcher'],
include: ['.test.ts'],
exclude: ['launcher/smoke.e2e.test.ts'],
},
scripts: {
roots: ['scripts'],
include: ['.test.ts'],
},
stats: {
roots: ['stats/src'],
include: ['.test.ts', '.test.tsx'],
},
};
function collectFiles(
repoRoot: string,
rootDir: string,
includeSuffixes: string[],
excludeSet: Set<string>,
): string[] {
const out: string[] = [];
const visit = (currentDir: string): void => {
for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
const fullPath = resolve(currentDir, entry.name);
if (entry.isDirectory()) {
visit(fullPath);
continue;
}
const relPath = relative(repoRoot, fullPath).replaceAll('\\', '/');
if (excludeSet.has(relPath)) continue;
if (includeSuffixes.some((suffix) => relPath.endsWith(suffix))) {
out.push(relPath);
}
}
};
visit(resolve(repoRoot, rootDir));
out.sort();
return out;
}
export function collectLaneFiles(repoRoot: string, laneName: string): string[] {
const lane = testLanes[laneName];
if (!lane) {
throw new Error(`Unknown test lane: ${laneName}`);
}
const excludeSet = new Set(lane.exclude ?? []);
const files = lane.roots.flatMap((rootDir) =>
collectFiles(repoRoot, rootDir, lane.include, excludeSet),
);
for (const extra of lane.extraFiles ?? []) {
if (!files.includes(extra)) files.push(extra);
}
files.sort();
if (files.length === 0) {
throw new Error(`No test files found for lane: ${laneName}`);
}
return files;
}
+6 -2
View File
@@ -703,8 +703,12 @@ export class AnkiIntegration {
});
}
isKnownWord(text: string, reading?: string): boolean {
return this.knownWordCache.isKnownWord(text, reading);
isKnownWord(
text: string,
reading?: string,
options?: { allowReadingOnlyMatch?: boolean },
): boolean {
return this.knownWordCache.isKnownWord(text, reading, options);
}
getKnownWordMatchMode(): NPlusOneMatchMode {
@@ -734,6 +734,85 @@ test('KnownWordCacheManager disambiguates known words by note reading', async ()
}
});
test('KnownWordCacheManager suppresses reading-only matches when disallowed', async () => {
const config: AnkiConnectConfig = {
fields: {
word: 'Word',
},
knownWords: {
highlightEnabled: true,
},
};
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
try {
clientState.findNotesResult = [1];
clientState.notesInfoResult = [
{
noteId: 1,
fields: {
Word: { value: '警告' },
'Word Reading': { value: 'けいこく' },
},
},
];
await manager.refresh(true);
// Reading-only match stays available for kana subtitle text…
assert.equal(manager.isKnownWord('けいこく'), true);
// …but a kanji token's reading (渓谷/けいこく) must not borrow 警告's.
assert.equal(manager.isKnownWord('けいこく', undefined, { allowReadingOnlyMatch: false }), false);
// Mined word texts still match regardless of the flag.
assert.equal(manager.isKnownWord('警告', undefined, { allowReadingOnlyMatch: false }), true);
} finally {
cleanup();
}
});
test('KnownWordCacheManager does not match single-kana text by reading alone', async () => {
const config: AnkiConnectConfig = {
fields: {
word: 'Word',
},
knownWords: {
highlightEnabled: true,
},
};
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
try {
clientState.findNotesResult = [1, 2];
clientState.notesInfoResult = [
{
noteId: 1,
fields: {
Word: { value: '夜' },
'Word Reading': { value: 'よ' },
},
},
{
noteId: 2,
fields: {
Word: { value: 'え' },
},
},
];
await manager.refresh(true);
// よ must not count as known just because 夜 is read よ.
assert.equal(manager.isKnownWord('よ'), false);
assert.equal(manager.isKnownWord('ヨ'), false);
assert.equal(manager.isKnownWord('夜'), true);
assert.equal(manager.isKnownWord('夜', 'よ'), true);
// A literal single-kana word entry still matches via the word map.
assert.equal(manager.isKnownWord('え'), true);
} finally {
cleanup();
}
});
test('KnownWordCacheManager probes reading fields even with per-deck word fields configured', async () => {
const config: AnkiConnectConfig = {
fields: {
+21 -2
View File
@@ -142,7 +142,11 @@ export class KnownWordCacheManager {
);
}
isKnownWord(text: string, reading?: string): boolean {
isKnownWord(
text: string,
reading?: string,
options?: { allowReadingOnlyMatch?: boolean },
): boolean {
if (!this.isKnownWordCacheEnabled()) {
return false;
}
@@ -163,7 +167,22 @@ export class KnownWordCacheManager {
);
}
return this.readingCounts.has(convertKatakanaToHiragana(normalized));
// Callers that look up a kanji token's reading (not subtitle text) must
// opt out of the reading-only fallback: readingCounts holds readings of
// every note including kanji words, so 渓谷's けいこく would match a
// mined 警告/けいこく.
if (options?.allowReadingOnlyMatch === false) {
return false;
}
// Reading-only fallback, except for single-kana text: particles and
// interjections (よ, ね, え…) would otherwise borrow the reading of an
// unrelated note (夜「よ」, 絵「え」) and count as known.
const hiragana = convertKatakanaToHiragana(normalized);
if ([...hiragana].length === 1) {
return false;
}
return this.readingCounts.has(hiragana);
}
refresh(force = false): Promise<void> {
@@ -301,6 +301,7 @@ function createMockTracker(
{ epochDay: Math.floor(Date.now() / 86_400_000) - 1, totalActiveMin: 30 },
{ epochDay: Math.floor(Date.now() / 86_400_000), totalActiveMin: 45 },
],
ensureAnimeCoverArt: async () => false,
getAnimeCoverArt: async (animeId: number) =>
animeId === 1
? {
@@ -520,6 +521,44 @@ describe('stats server API routes', () => {
});
});
it('GET /api/stats/sessions enriches known-word metrics from a v3 reading-aware cache', async () => {
await withTempDir(async (dir) => {
const cachePath = path.join(dir, 'known-words.json');
fs.writeFileSync(
cachePath,
JSON.stringify({
version: 3,
refreshedAtMs: 1,
scope: 'deck:test',
notes: {
'101': [{ word: 'する', reading: 'する' }],
'102': [{ word: '猫', reading: null }],
},
}),
);
const app = createStatsApp(
createMockTracker({
getSessionWordsByLine: async (sessionId: number) =>
sessionId === 1
? [
{ lineIndex: 1, headword: 'する', occurrenceCount: 2 },
{ lineIndex: 2, headword: '未知', occurrenceCount: 1 },
]
: [],
}),
{ knownWordCachePath: cachePath },
);
const res = await app.request('/api/stats/sessions?limit=5');
assert.equal(res.status, 200);
const body = await res.json();
const first = body[0];
assert.equal(first.knownWordsSeen, 2);
assert.equal(first.knownWordRate, 66.7);
});
});
it('GET /api/stats/sessions/:id/events forwards event type filters to the tracker', async () => {
let seenSessionId = 0;
let seenLimit = 0;
@@ -994,8 +1033,9 @@ describe('stats server API routes', () => {
assert.equal(res.status, 404);
});
it('POST /api/stats/covers batches stored cover art without fetching missing art', async () => {
it('POST /api/stats/covers batches stored cover art and backfills missing anime art in the background', async () => {
let ensureCoverArtCalls = 0;
const ensureAnimeCoverArtCalls: number[] = [];
const app = createStatsApp(
createMockTracker({
getCoverArt: async (videoId: number) =>
@@ -1015,6 +1055,10 @@ describe('stats server API routes', () => {
ensureCoverArtCalls += 1;
return true;
},
ensureAnimeCoverArt: async (animeId: number) => {
ensureAnimeCoverArtCalls.push(animeId);
return false;
},
}),
);
@@ -1042,6 +1086,68 @@ describe('stats server API routes', () => {
},
});
assert.equal(ensureCoverArtCalls, 0);
assert.deepEqual(ensureAnimeCoverArtCalls, [99999]);
});
it('POST /api/stats/covers limits concurrent missing anime cover backfills', async () => {
let activeBackfills = 0;
let maxActiveBackfills = 0;
const pendingBackfills: Array<() => void> = [];
const app = createStatsApp(
createMockTracker({
getAnimeCoverArt: async () => null,
ensureAnimeCoverArt: async () => {
activeBackfills += 1;
maxActiveBackfills = Math.max(maxActiveBackfills, activeBackfills);
await new Promise<void>((resolve) => {
pendingBackfills.push(resolve);
});
activeBackfills -= 1;
return false;
},
}),
);
const res = await app.request('/api/stats/covers', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ animeIds: [101, 102, 103, 104, 105] }),
});
assert.equal(res.status, 200);
assert.equal(maxActiveBackfills, 3);
for (const resolveBackfill of pendingBackfills) {
resolveBackfill();
}
});
it('GET /api/stats/anime/:animeId/cover fetches missing art before serving', async () => {
let fetched = false;
const app = createStatsApp(
createMockTracker({
getAnimeCoverArt: async () =>
fetched
? {
videoId: 1,
anilistId: 21858,
coverUrl: 'https://example.com/cover.jpg',
coverBlob: Buffer.from([0xff, 0xd8, 0xff, 0xd9]),
titleRomaji: 'Little Witch Academia',
titleEnglish: 'Little Witch Academia',
episodesTotal: 25,
fetchedAtMs: Date.now(),
}
: null,
ensureAnimeCoverArt: async () => {
fetched = true;
return true;
},
}),
);
const res = await app.request('/api/stats/anime/1/cover');
assert.equal(res.status, 200);
assert.equal(res.headers.get('content-type'), 'image/jpeg');
});
it('GET /api/stats/anime/:animeId/words returns top words for an anime', async () => {
@@ -4041,3 +4041,91 @@ test('markActiveVideoWatched returns false when no active session', async () =>
cleanupDbPath(dbPath);
}
});
test('handleMediaChange prefetches cover art at session start', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath });
const fetchedVideoIds: number[] = [];
tracker.setCoverArtFetcher({
fetchIfMissing: async (_db, videoId) => {
fetchedVideoIds.push(videoId);
return false;
},
});
tracker.handleMediaChange('/tmp/Little Witch Academia S02E05.mkv', 'Episode 5');
await waitForPendingAnimeMetadata(tracker);
await waitForCondition(() => fetchedVideoIds.length > 0);
const privateApi = tracker as unknown as {
sessionState: { videoId: number } | null;
};
assert.deepEqual(fetchedVideoIds, [privateApi.sessionState?.videoId]);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('ensureAnimeCoverArt fetches art via the latest video of the anime', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath });
const privateApi = tracker as unknown as { db: DatabaseSync };
privateApi.db.exec(`
INSERT INTO imm_anime (
anime_id,
normalized_title_key,
canonical_title,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (
1,
'little witch academia',
'Little Witch Academia',
1000,
1000
);
INSERT INTO imm_videos (
video_id,
video_key,
canonical_title,
source_type,
duration_ms,
anime_id,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES
(1, 'local:/tmp/lwa-1.mkv', 'Little Witch Academia S01E01', 1, 0, 1, 1000, 1000),
(2, 'local:/tmp/lwa-2.mkv', 'Little Witch Academia S01E02', 1, 0, 1, 1000, 1000);
`);
const fetchedVideoIds: number[] = [];
tracker.setCoverArtFetcher({
fetchIfMissing: async (_db, videoId) => {
fetchedVideoIds.push(videoId);
return false;
},
});
const result = await tracker.ensureAnimeCoverArt(1);
assert.equal(result, false);
assert.deepEqual(fetchedVideoIds, [2]);
const missing = await tracker.ensureAnimeCoverArt(999);
assert.equal(missing, false);
assert.deepEqual(fetchedVideoIds, [2]);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
+41 -2
View File
@@ -854,6 +854,22 @@ export class ImmersionTrackerService {
this.coverArtFetcher = fetcher;
}
async ensureAnimeCoverArt(animeId: number): Promise<boolean> {
const existing = await this.getAnimeCoverArt(animeId);
if (existing?.coverBlob) {
return true;
}
const row = this.db
.prepare(
'SELECT video_id AS videoId FROM imm_videos WHERE anime_id = ? ORDER BY video_id DESC LIMIT 1',
)
.get(animeId) as { videoId: number } | undefined;
if (!row?.videoId) {
return false;
}
return this.ensureCoverArt(row.videoId);
}
async ensureCoverArt(videoId: number): Promise<boolean> {
const existing = await this.getCoverArt(videoId);
if (existing?.coverBlob) {
@@ -879,8 +895,10 @@ export class ImmersionTrackerService {
}
const fetchPromise = (async () => {
const detail = getMediaDetail(this.db, videoId);
const canonicalTitle = detail?.canonicalTitle?.trim();
const titleRow = this.db
.prepare('SELECT canonical_title AS canonicalTitle FROM imm_videos WHERE video_id = ?')
.get(videoId) as { canonicalTitle: string | null } | undefined;
const canonicalTitle = titleRow?.canonicalTitle?.trim();
if (!canonicalTitle) {
return false;
}
@@ -1342,6 +1360,9 @@ export class ImmersionTrackerService {
} else if (!this.hasJellyfinMetadata(sessionInfo.videoId)) {
this.captureAnimeMetadataAsync(sessionInfo.videoId, normalizedPath, normalizedTitle || null);
}
if (!youtubeVideoId) {
this.prefetchCoverArtAsync(sessionInfo.videoId);
}
this.captureVideoMetadataAsync(sessionInfo.videoId, sourceType, normalizedPath);
}
@@ -1924,6 +1945,24 @@ export class ImmersionTrackerService {
});
}
// Fetch cover art eagerly at session start (after anime metadata parsing
// settles) so new series show art on the stats timeline without requiring a
// visit to the series detail page first.
private prefetchCoverArtAsync(videoId: number): void {
const pendingMetadata = this.pendingAnimeMetadataUpdates.get(videoId);
void (async () => {
try {
await pendingMetadata;
if (this.isDestroyed) {
return;
}
await this.ensureCoverArt(videoId);
} catch (error) {
this.logger.warn('Unable to prefetch cover art', (error as Error).message);
}
})();
}
private updateVideoTitleForActiveSession(canonicalTitle: string): void {
if (!this.sessionState) return;
updateVideoTitleRecord(this.db, this.sessionState.videoId, canonicalTitle);
+171
View File
@@ -0,0 +1,171 @@
import type { Hono } from 'hono';
import type { ImmersionTrackerService } from './immersion-tracker-service.js';
type StatsCoverImagePayload = {
contentType: string;
dataUrl: string;
} | null;
type StatsCoverBatchBody = {
animeIds?: unknown;
videoIds?: unknown;
};
const MAX_BACKGROUND_ANIME_COVER_FETCHES = 3;
function parseIntQuery(raw: string | undefined, fallback: number, maxLimit?: number): number {
if (raw === undefined) return fallback;
const n = Number(raw);
if (!Number.isFinite(n) || n < 0) {
return fallback;
}
const parsed = Math.floor(n);
return maxLimit === undefined ? parsed : Math.min(parsed, maxLimit);
}
function parsePositiveIdList(raw: unknown, maxItems = 100): number[] {
if (!Array.isArray(raw)) return [];
const ids = new Set<number>();
for (const rawId of raw) {
const id = typeof rawId === 'number' ? rawId : typeof rawId === 'string' ? Number(rawId) : NaN;
if (Number.isFinite(id) && id > 0) {
ids.add(Math.floor(id));
if (ids.size >= maxItems) break;
}
}
return Array.from(ids).sort((a, b) => a - b);
}
function coverImagePayload(
art: { coverBlob?: Uint8Array | null } | null | undefined,
): StatsCoverImagePayload {
if (!art?.coverBlob) return null;
const bytes = new Uint8Array(art.coverBlob);
const contentType = detectImageContentType(bytes);
return {
contentType,
dataUrl: `data:${contentType};base64,${Buffer.from(bytes).toString('base64')}`,
};
}
function detectImageContentType(bytes: Uint8Array): string {
if (
bytes.length >= 8 &&
bytes[0] === 0x89 &&
bytes[1] === 0x50 &&
bytes[2] === 0x4e &&
bytes[3] === 0x47
) {
return 'image/png';
}
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
return 'image/jpeg';
}
if (
bytes.length >= 12 &&
bytes[0] === 0x52 &&
bytes[1] === 0x49 &&
bytes[2] === 0x46 &&
bytes[3] === 0x46 &&
bytes[8] === 0x57 &&
bytes[9] === 0x45 &&
bytes[10] === 0x42 &&
bytes[11] === 0x50
) {
return 'image/webp';
}
return 'application/octet-stream';
}
function createLimitedTaskRunner(maxConcurrentTasks: number): (task: () => Promise<void>) => void {
const queue: Array<() => Promise<void>> = [];
let activeTasks = 0;
const drain = (): void => {
while (activeTasks < maxConcurrentTasks && queue.length > 0) {
const task = queue.shift();
if (!task) return;
activeTasks += 1;
void task()
.catch(() => {})
.finally(() => {
activeTasks -= 1;
drain();
});
}
};
return (task: () => Promise<void>): void => {
queue.push(task);
drain();
};
}
export function registerStatsCoverRoutes(app: Hono, tracker: ImmersionTrackerService): void {
const enqueueAnimeCoverBackfill = createLimitedTaskRunner(MAX_BACKGROUND_ANIME_COVER_FETCHES);
app.post('/api/stats/covers', async (c) => {
const body = (await c.req.json().catch(() => null)) as StatsCoverBatchBody | null;
const animeIds = parsePositiveIdList(body?.animeIds);
const videoIds = parsePositiveIdList(body?.videoIds);
const anime: Record<number, StatsCoverImagePayload> = {};
const media: Record<number, StatsCoverImagePayload> = {};
await Promise.all(
animeIds.map(async (animeId) => {
const art = await tracker.getAnimeCoverArt(animeId);
if (!art?.coverBlob) {
enqueueAnimeCoverBackfill(async () => {
await tracker.ensureAnimeCoverArt(animeId);
});
}
anime[animeId] = coverImagePayload(art);
}),
);
await Promise.all(
videoIds.map(async (videoId) => {
media[videoId] = coverImagePayload(await tracker.getCoverArt(videoId));
}),
);
return c.json({ anime, media });
});
app.get('/api/stats/anime/:animeId/cover', async (c) => {
const animeId = parseIntQuery(c.req.param('animeId'), 0);
if (animeId <= 0) return c.body(null, 404);
let art = await tracker.getAnimeCoverArt(animeId);
if (!art?.coverBlob) {
await tracker.ensureAnimeCoverArt(animeId);
art = await tracker.getAnimeCoverArt(animeId);
}
if (!art?.coverBlob) return c.body(null, 404);
const bytes = new Uint8Array(art.coverBlob);
return new Response(bytes, {
headers: {
'Content-Type': detectImageContentType(bytes),
'Cache-Control': 'public, max-age=86400',
},
});
});
app.get('/api/stats/media/:videoId/cover', async (c) => {
const videoId = parseIntQuery(c.req.param('videoId'), 0);
if (videoId <= 0) return c.body(null, 404);
let art = await tracker.getCoverArt(videoId);
if (!art?.coverBlob) {
await tracker.ensureCoverArt(videoId);
art = await tracker.getCoverArt(videoId);
}
if (!art?.coverBlob) return c.body(null, 404);
const bytes = new Uint8Array(art.coverBlob);
return new Response(bytes, {
headers: {
'Content-Type': detectImageContentType(bytes),
'Cache-Control': 'public, max-age=604800',
},
});
});
}
+17 -118
View File
@@ -17,6 +17,7 @@ import {
} from '../../anki-field-config.js';
import { resolveAnimatedImageLeadInSeconds } from '../../anki-integration/animated-image-sync.js';
import type { AnilistRateLimiter } from './anilist/rate-limiter.js';
import { registerStatsCoverRoutes } from './stats-cover-routes.js';
import {
resolveRetimedSecondarySubtitleTextFromSidecar,
resolveSecondarySubtitleTextFromSidecar,
@@ -51,16 +52,6 @@ type StatsExcludedWordPayload = {
reading: string;
};
type StatsCoverImagePayload = {
contentType: string;
dataUrl: string;
} | null;
type StatsCoverBatchBody = {
animeIds?: unknown;
videoIds?: unknown;
};
function parseIntQuery(raw: string | undefined, fallback: number, maxLimit?: number): number {
if (raw === undefined) return fallback;
const n = Number(raw);
@@ -113,62 +104,6 @@ function parseExcludedWordsBody(body: unknown): StatsExcludedWordPayload[] | nul
return words;
}
function parsePositiveIdList(raw: unknown, maxItems = 100): number[] {
if (!Array.isArray(raw)) return [];
const ids = new Set<number>();
for (const rawId of raw) {
const id = typeof rawId === 'number' ? rawId : typeof rawId === 'string' ? Number(rawId) : NaN;
if (Number.isFinite(id) && id > 0) {
ids.add(Math.floor(id));
if (ids.size >= maxItems) break;
}
}
return Array.from(ids).sort((a, b) => a - b);
}
function coverImagePayload(
art: { coverBlob?: Uint8Array | null } | null | undefined,
): StatsCoverImagePayload {
if (!art?.coverBlob) return null;
const bytes = new Uint8Array(art.coverBlob);
const contentType = detectImageContentType(bytes);
return {
contentType,
dataUrl: `data:${contentType};base64,${Buffer.from(bytes).toString('base64')}`,
};
}
function detectImageContentType(bytes: Uint8Array): string {
if (
bytes.length >= 8 &&
bytes[0] === 0x89 &&
bytes[1] === 0x50 &&
bytes[2] === 0x4e &&
bytes[3] === 0x47
) {
return 'image/png';
}
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
return 'image/jpeg';
}
if (
bytes.length >= 12 &&
bytes[0] === 0x52 &&
bytes[1] === 0x49 &&
bytes[2] === 0x46 &&
bytes[3] === 0x46 &&
bytes[8] === 0x57 &&
bytes[9] === 0x45 &&
bytes[10] === 0x42 &&
bytes[11] === 0x50
) {
return 'image/webp';
}
return 'application/octet-stream';
}
function resolveStatsNoteFieldName(
noteInfo: StatsServerNoteInfo,
...preferredNames: (string | undefined)[]
@@ -326,10 +261,25 @@ function loadKnownWordsSet(cachePath: string | undefined): Set<string> | null {
const raw = JSON.parse(readFileSync(cachePath, 'utf-8')) as {
version?: number;
words?: string[];
notes?: Record<string, Array<{ word?: unknown; reading?: unknown }>>;
};
if ((raw.version === 1 || raw.version === 2) && Array.isArray(raw.words)) {
return new Set(raw.words);
}
// v3 stores reading-aware entries per note; stats rows only carry
// headwords, so flatten to a word set (reading-agnostic, fail-open).
if (raw.version === 3 && raw.notes && typeof raw.notes === 'object') {
const words = new Set<string>();
for (const entries of Object.values(raw.notes)) {
if (!Array.isArray(entries)) continue;
for (const entry of entries) {
if (entry && typeof entry.word === 'string' && entry.word) {
words.add(entry.word);
}
}
}
return words;
}
} catch {
/* ignore */
}
@@ -1017,58 +967,7 @@ export function createStatsApp(
return c.json({ ok: true });
});
app.post('/api/stats/covers', async (c) => {
const body = (await c.req.json().catch(() => null)) as StatsCoverBatchBody | null;
const animeIds = parsePositiveIdList(body?.animeIds);
const videoIds = parsePositiveIdList(body?.videoIds);
const anime: Record<number, StatsCoverImagePayload> = {};
const media: Record<number, StatsCoverImagePayload> = {};
await Promise.all(
animeIds.map(async (animeId) => {
anime[animeId] = coverImagePayload(await tracker.getAnimeCoverArt(animeId));
}),
);
await Promise.all(
videoIds.map(async (videoId) => {
media[videoId] = coverImagePayload(await tracker.getCoverArt(videoId));
}),
);
return c.json({ anime, media });
});
app.get('/api/stats/anime/:animeId/cover', async (c) => {
const animeId = parseIntQuery(c.req.param('animeId'), 0);
if (animeId <= 0) return c.body(null, 404);
const art = await tracker.getAnimeCoverArt(animeId);
if (!art?.coverBlob) return c.body(null, 404);
const bytes = new Uint8Array(art.coverBlob);
return new Response(bytes, {
headers: {
'Content-Type': detectImageContentType(bytes),
'Cache-Control': 'public, max-age=86400',
},
});
});
app.get('/api/stats/media/:videoId/cover', async (c) => {
const videoId = parseIntQuery(c.req.param('videoId'), 0);
if (videoId <= 0) return c.body(null, 404);
let art = await tracker.getCoverArt(videoId);
if (!art?.coverBlob) {
await tracker.ensureCoverArt(videoId);
art = await tracker.getCoverArt(videoId);
}
if (!art?.coverBlob) return c.body(null, 404);
const bytes = new Uint8Array(art.coverBlob);
return new Response(bytes, {
headers: {
'Content-Type': detectImageContentType(bytes),
'Cache-Control': 'public, max-age=604800',
},
});
});
registerStatsCoverRoutes(app, tracker);
app.get('/api/stats/episode/:videoId/detail', async (c) => {
const videoId = parseIntQuery(c.req.param('videoId'), 0);
+40 -4
View File
@@ -28,6 +28,7 @@ interface YomitanTokenInput {
frequencyRank?: number;
isNameMatch?: boolean;
wordClasses?: string[];
isUnparsedRun?: boolean;
}
function makeDepsFromYomitanTokens(
@@ -60,6 +61,7 @@ function makeDepsFromYomitanTokens(
isNameMatch: token.isNameMatch ?? false,
frequencyRank: token.frequencyRank,
wordClasses: token.wordClasses,
isUnparsedRun: token.isUnparsedRun,
};
});
},
@@ -1821,7 +1823,7 @@ test('tokenizeSubtitle returns null tokens when mecab throws', async () => {
assert.deepEqual(result, { text: '猫です', tokens: null });
});
test('tokenizeSubtitle uses Yomitan parser result when available and drops no-headword groups', async () => {
test('tokenizeSubtitle uses Yomitan parser result and keeps no-headword groups as surface tokens', async () => {
const parserWindow = {
isDestroyed: () => false,
webContents: {
@@ -1859,10 +1861,12 @@ test('tokenizeSubtitle uses Yomitan parser result when available and drops no-he
);
assert.equal(result.text, '猫です');
assert.equal(result.tokens?.length, 1);
assert.equal(result.tokens?.length, 2);
assert.equal(result.tokens?.[0]?.surface, '猫');
assert.equal(result.tokens?.[0]?.reading, 'ねこ');
assert.equal(result.tokens?.[0]?.isKnown, false);
assert.equal(result.tokens?.[1]?.surface, 'です');
assert.equal(result.tokens?.[1]?.headword, 'です');
});
test('tokenizeSubtitle logs selected Yomitan groups when debug toggle is enabled', async () => {
@@ -3484,8 +3488,8 @@ test('tokenizeSubtitle keeps known-word highlight for exact non-independent kanj
assert.equal(result.tokens?.[1]?.surface, '点');
assert.equal(result.tokens?.[1]?.isKnown, true);
assert.equal(result.tokens?.[1]?.isNPlusOneTarget, false);
assert.equal(result.tokens?.[1]?.frequencyRank, undefined);
assert.equal(result.tokens?.[1]?.jlptLevel, undefined);
assert.equal(result.tokens?.[1]?.frequencyRank, 1384);
assert.equal(result.tokens?.[1]?.jlptLevel, 'N3');
});
test('tokenizeSubtitle keeps mecab-tagged interjections tokenized while clearing annotation metadata', async () => {
@@ -4221,6 +4225,38 @@ test('tokenizeSubtitle clears all annotations for explanatory pondering endings'
);
});
test('tokenizeSubtitle ignores unparsed-run tokens for annotations and N+1', async () => {
// もう いるぅ~!: the ぅ~ elongation has no Yomitan dictionary entry; it must
// not become the sole N+1 candidate or receive frequency/JLPT annotations.
const result = await tokenizeSubtitle(
'もう いるぅ~!',
makeDepsFromYomitanTokens(
[
{ surface: 'もう', reading: 'もう', headword: 'もう' },
{ surface: 'いる', reading: 'いる', headword: 'いる' },
{ surface: 'ぅ~', reading: '', headword: 'ぅ~', isUnparsedRun: true, frequencyRank: 999 },
],
{
getFrequencyDictionaryEnabled: () => true,
getJlptLevel: (text) => (text === 'ぅ~' ? 'N5' : null),
isKnownWord: (text) => text === 'もう' || text === 'いる',
getMinSentenceWordsForNPlusOne: () => 2,
tokenizeWithMecab: async () => null,
},
),
);
const filler = result.tokens?.find((token) => token.surface === 'ぅ~');
assert.ok(filler);
assert.equal(filler?.isNPlusOneTarget, false);
assert.equal(filler?.frequencyRank, undefined);
assert.equal(filler?.jlptLevel, undefined);
assert.equal(
result.tokens?.some((token) => token.isNPlusOneTarget),
false,
);
});
test('tokenizeSubtitle keeps frequency for content-led merged token with trailing colloquial suffixes', async () => {
const result = await tokenizeSubtitle(
'張り切ってんじゃ',
+15 -4
View File
@@ -33,6 +33,15 @@ type MecabTokenEnrichmentFn = (
mecabTokens: MergedToken[] | null,
) => Promise<MergedToken[]>;
// allowReadingOnlyMatch: false suppresses the cache's reading-only index for
// lookups that pass a kanji token's reading as the text (see
// computeTokenKnownStatus in annotation-stage).
export type KnownWordLookupFn = (
text: string,
reading?: string,
options?: { allowReadingOnlyMatch?: boolean },
) => boolean;
export interface TokenizerServiceDeps {
getYomitanExt: () => Extension | null;
getYomitanSession?: () => Session | null;
@@ -42,7 +51,7 @@ export interface TokenizerServiceDeps {
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
getYomitanParserInitPromise: () => Promise<boolean> | null;
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
isKnownWord: (text: string, reading?: string) => boolean;
isKnownWord: KnownWordLookupFn;
getKnownWordMatchMode: () => NPlusOneMatchMode;
getKnownWordsEnabled?: () => boolean;
getJlptLevel: (text: string) => JlptLevel | null;
@@ -77,7 +86,7 @@ export interface TokenizerDepsRuntimeOptions {
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
getYomitanParserInitPromise: () => Promise<boolean> | null;
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
isKnownWord: (text: string, reading?: string) => boolean;
isKnownWord: KnownWordLookupFn;
getKnownWordMatchMode: () => NPlusOneMatchMode;
getKnownWordsEnabled?: () => boolean;
getJlptLevel: (text: string) => JlptLevel | null;
@@ -129,7 +138,7 @@ const INVISIBLE_SEPARATOR_PATTERN = /[\u200b\u2060\ufeff]/g;
function getKnownWordLookup(
deps: TokenizerServiceDeps,
options: TokenizerAnnotationOptions,
): (text: string, reading?: string) => boolean {
): KnownWordLookupFn {
if (!options.knownWordsEnabled && !options.nPlusOneEnabled) {
return () => false;
}
@@ -150,7 +159,8 @@ function hasAnyAnnotationEnabled(options: TokenizerAnnotationOptions): boolean {
options.knownWordsEnabled ||
options.nPlusOneEnabled ||
options.jlptEnabled ||
options.frequencyEnabled
options.frequencyEnabled ||
options.nameMatchEnabled
);
}
@@ -733,6 +743,7 @@ async function parseWithYomitanInternalParser(
isNPlusOneTarget: false,
isNameMatch: token.isNameMatch ?? false,
frequencyRank: token.frequencyRank,
isUnparsedRun: token.isUnparsedRun === true ? true : undefined,
};
}),
);
@@ -33,6 +33,50 @@ function makeDeps(overrides: Partial<AnnotationStageDeps> = {}): AnnotationStage
};
}
test('annotateTokens keeps name matches on tokens the POS noise filter would strip', () => {
// MeCab tags 平 as 接頭詞 in contexts like あっ 平 これ…, which is in the
// POS1 exclusion list; a confirmed character-name match must survive it.
const tokens = [
makeToken({
surface: '平',
headword: '平',
reading: 'たいら',
pos1: '接頭詞',
isNameMatch: true,
}),
makeToken({
surface: '平',
headword: '平',
reading: 'ひら',
pos1: '接頭詞',
isNameMatch: false,
jlptLevel: 'N1',
}),
];
const result = annotateTokens(tokens, makeDeps(), { nameMatchEnabled: true });
assert.equal(result[0]?.isNameMatch, true);
assert.equal(result[1]?.isNameMatch, false);
assert.equal(result[1]?.jlptLevel, undefined);
});
test('annotateTokens strips name matches from POS-excluded tokens when name matching is disabled', () => {
const tokens = [
makeToken({
surface: '平',
headword: '平',
reading: 'たいら',
pos1: '接頭詞',
isNameMatch: true,
}),
];
const result = annotateTokens(tokens, makeDeps(), { nameMatchEnabled: false });
assert.equal(result[0]?.isNameMatch, false);
});
test('annotateTokens known-word match mode uses headword vs surface', () => {
const tokens = [makeToken({ surface: '食べた', headword: '食べる', reading: 'タベタ' })];
const isKnownWord = (text: string): boolean => text === '食べる';
@@ -164,6 +208,48 @@ test('annotateTokens hides known-word marks while still using known words for N+
assert.equal(result[2]?.isNPlusOneTarget, true);
});
test('shouldExcludeTokenFromSubtitleAnnotations excludes unparsed-run tokens', () => {
// 戻 from 「…とこ戻ろ…」: Yomitan had no dictionary entry, headword falls back
// to the surface. Without the flag the token passes every other filter.
const unflagged = makeToken({ surface: '戻', headword: '戻', reading: '' });
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(unflagged), false);
const flagged = makeToken({ surface: '戻', headword: '戻', reading: '', isUnparsedRun: true });
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(flagged), true);
assert.equal(shouldExcludeTokenFromVocabularyPersistence(flagged), true);
});
test('annotateTokens ignores unparsed-run tokens for annotations and N+1', () => {
// もう いるぅ~!-style line: the elongation run is the only unknown token and
// used to become the sole N+1 candidate despite having no dictionary entry.
const tokens = [
makeToken({ surface: 'みんな', headword: '皆', reading: 'みんな', startPos: 0, endPos: 3 }),
makeToken({ surface: 'とこ', headword: '所', reading: 'とこ', startPos: 3, endPos: 5 }),
makeToken({
surface: '戻',
headword: '戻',
reading: '',
startPos: 5,
endPos: 6,
isUnparsedRun: true,
frequencyRank: 12,
}),
];
const result = annotateTokens(
tokens,
makeDeps({
isKnownWord: (text) => text === '皆' || text === '所',
getJlptLevel: (text) => (text === '戻' ? 'N5' : null),
}),
{ minSentenceWordsForNPlusOne: 2 },
);
assert.equal(result[2]?.isNPlusOneTarget, false);
assert.equal(result[2]?.jlptLevel, undefined);
assert.equal(result[2]?.frequencyRank, undefined);
});
test('annotateTokens falls back to reading for known-word matches when headword lookup misses', () => {
const tokens = [
makeToken({
@@ -187,6 +273,29 @@ test('annotateTokens falls back to reading for known-word matches when headword
assert.equal(result[0]?.frequencyRank, 1895);
});
test('annotateTokens reading fallback does not match kanji tokens sharing a mined reading', () => {
const tokens = [
makeToken({
surface: '渓谷',
headword: '渓谷',
reading: 'けいこく',
endPos: 2,
}),
];
const result = annotateTokens(
tokens,
makeDeps({
// Mimics the cache with a mined 警告/けいこく: けいこく matches through
// the reading-only index unless the lookup opts out of it.
isKnownWord: (text, _reading, options) =>
text === '警告' || (options?.allowReadingOnlyMatch !== false && text === 'けいこく'),
}),
);
assert.equal(result[0]?.isKnown, false);
});
test('annotateTokens ignores partial furigana readings for known-word fallback', () => {
const tokens = [
makeToken({
@@ -531,7 +640,9 @@ test('shouldExcludeTokenFromSubtitleAnnotations keeps lexical tokens outside exp
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
});
test('shouldExcludeTokenFromSubtitleAnnotations still excludes lexical non-independent kanji nouns from non-known annotations', () => {
test('shouldExcludeTokenFromSubtitleAnnotations keeps lexical non-independent kanji nouns', () => {
// Yomitan segments 以外/日/方 as standalone vocabulary tokens; MeCab's
// 非自立 tag must only suppress kana grammar nouns (こと, もの, とき).
const token = makeToken({
surface: '以外',
headword: '以外',
@@ -542,6 +653,21 @@ test('shouldExcludeTokenFromSubtitleAnnotations still excludes lexical non-indep
pos3: '副詞可能',
});
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
assert.equal(shouldExcludeTokenFromVocabularyPersistence(token), false);
});
test('shouldExcludeTokenFromSubtitleAnnotations still excludes kana non-independent nouns', () => {
const token = makeToken({
surface: 'こと',
headword: 'こと',
reading: 'コト',
partOfSpeech: PartOfSpeech.noun,
pos1: '名詞',
pos2: '非自立',
pos3: '一般',
});
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
assert.equal(shouldExcludeTokenFromVocabularyPersistence(token), true);
});
@@ -684,15 +810,6 @@ test('shouldExcludeTokenFromVocabularyPersistence excludes common frequency stop
pos2: '代名詞|副助詞/並立助詞/終助詞',
pos3: '一般|*',
}),
makeToken({
surface: '確かに',
headword: '確かに',
reading: 'たしかに',
partOfSpeech: PartOfSpeech.other,
pos1: '名詞|助詞',
pos2: '形容動詞語幹|副詞化',
pos3: '*',
}),
makeToken({
surface: 'あなた',
headword: '貴方',
@@ -709,6 +826,34 @@ test('shouldExcludeTokenFromVocabularyPersistence excludes common frequency stop
}
});
test('content adverbs are not excluded from annotations or vocabulary persistence', () => {
const tokens = [
makeToken({
surface: '確かに',
headword: '確かに',
reading: 'たしかに',
partOfSpeech: PartOfSpeech.other,
pos1: '名詞|助詞',
pos2: '形容動詞語幹|副詞化',
pos3: '*',
}),
makeToken({
surface: 'やはり',
headword: 'やはり',
reading: 'ヤハリ',
partOfSpeech: PartOfSpeech.other,
pos1: '副詞',
pos2: '一般',
pos3: '*',
}),
];
for (const token of tokens) {
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false, token.surface);
assert.equal(shouldExcludeTokenFromVocabularyPersistence(token), false, token.surface);
}
});
test('shouldExcludeTokenFromSubtitleAnnotations excludes standalone して grammar helper fragments', () => {
const token = makeToken({
surface: 'して',
@@ -1358,8 +1503,8 @@ test('annotateTokens keeps known-word status for non-independent kanji noun toke
assert.equal(result[0]?.isKnown, true);
assert.equal(result[0]?.isNPlusOneTarget, false);
assert.equal(result[0]?.frequencyRank, undefined);
assert.equal(result[0]?.jlptLevel, undefined);
assert.equal(result[0]?.frequencyRank, 1384);
assert.equal(result[0]?.jlptLevel, 'N3');
});
test('annotateTokens keeps known-word status for lexical non-independent kanji nouns', () => {
@@ -1387,23 +1532,54 @@ test('annotateTokens keeps known-word status for lexical non-independent kanji n
);
assert.equal(result[0]?.isKnown, true);
assert.equal(result[0]?.frequencyRank, undefined);
assert.equal(result[0]?.frequencyRank, 437);
assert.equal(result[0]?.isNPlusOneTarget, false);
});
test('annotateTokens clears all annotations for non-independent kanji noun tokens under unified gate', () => {
test('annotateTokens keeps frequency for unknown non-independent kanji noun tokens', () => {
// 日 in いい日だったな: MeCab tags it 名詞/非自立 but Yomitan segments it as
// a standalone vocabulary token, so frequency highlighting must survive.
const tokens = [
makeToken({
surface: '',
reading: 'もの',
headword: '',
surface: '',
reading: '',
headword: '',
partOfSpeech: PartOfSpeech.noun,
pos1: '名詞',
pos2: '非自立',
pos3: '副詞可能',
startPos: 2,
endPos: 3,
frequencyRank: 718,
}),
];
const result = annotateTokens(
tokens,
makeDeps({
getJlptLevel: (text) => (text === '日' ? 'N4' : null),
}),
{ minSentenceWordsForNPlusOne: 1 },
);
assert.equal(result[0]?.isKnown, false);
assert.equal(result[0]?.frequencyRank, 718);
assert.equal(result[0]?.jlptLevel, 'N4');
});
test('annotateTokens still clears annotations for kana non-independent noun tokens', () => {
const tokens = [
makeToken({
surface: 'こと',
reading: 'こと',
headword: 'こと',
partOfSpeech: PartOfSpeech.other,
pos1: '名詞',
pos2: '非自立',
pos3: '一般',
startPos: 0,
endPos: 1,
frequencyRank: 475,
endPos: 2,
frequencyRank: 96,
}),
];
@@ -1678,6 +1854,36 @@ test('annotateTokens keeps known status while clearing other annotations for sta
}
});
test('annotateTokens excludes standalone noun-suffix tokens from annotations while keeping cache-backed known status', () => {
const tokens = [
makeToken({
surface: 'さん',
headword: 'さん',
reading: 'サン',
partOfSpeech: PartOfSpeech.noun,
pos1: '名詞',
pos2: '接尾',
startPos: 0,
endPos: 2,
frequencyRank: 33,
}),
];
const result = annotateTokens(
tokens,
makeDeps({
isKnownWord: (text) => text === 'さん',
getJlptLevel: (text) => (text === 'さん' ? 'N5' : null),
}),
{ minSentenceWordsForNPlusOne: 1 },
);
assert.equal(result[0]?.isKnown, true);
assert.equal(result[0]?.isNPlusOneTarget, false);
assert.equal(result[0]?.frequencyRank, undefined);
assert.equal(result[0]?.jlptLevel, undefined);
});
test('annotateTokens keeps known status while clearing other annotations for auxiliary-only te-kureru helper spans', () => {
const tokens = [
makeToken({
@@ -1994,7 +2200,8 @@ test('annotateTokens keeps known status while clearing other annotations for aru
assert.equal(result[0]?.headword, '有る');
assert.equal(result[0]?.isKnown, true);
assert.equal(result[0]?.isNPlusOneTarget, false);
assert.equal(result[0]?.isNameMatch, false);
// Name matches take precedence over the annotation noise filter.
assert.equal(result[0]?.isNameMatch, true);
assert.equal(result[0]?.frequencyRank, undefined);
assert.equal(result[0]?.jlptLevel, undefined);
});
+24 -52
View File
@@ -10,6 +10,7 @@ import {
import { JlptLevel, MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types';
import { shouldIgnoreJlptByTerm, shouldIgnoreJlptForMecabPos1 } from '../jlpt-token-filter';
import {
isKanjiNonIndependentNounToken,
shouldExcludeTokenFromSubtitleAnnotations as sharedShouldExcludeTokenFromSubtitleAnnotations,
stripSubtitleAnnotationMetadata as sharedStripSubtitleAnnotationMetadata,
} from './subtitle-annotation-filter';
@@ -25,7 +26,11 @@ const jlptLevelLookupCaches = new WeakMap<
>();
export interface AnnotationStageDeps {
isKnownWord: (text: string, reading?: string) => boolean;
isKnownWord: (
text: string,
reading?: string,
options?: { allowReadingOnlyMatch?: boolean },
) => boolean;
knownWordMatchMode: NPlusOneMatchMode;
getJlptLevel: (text: string) => JlptLevel | null;
}
@@ -94,23 +99,6 @@ function normalizePos2Tag(pos2: string | undefined): string {
return typeof pos2 === 'string' ? pos2.trim() : '';
}
function hasKanjiChar(text: string): boolean {
for (const char of text) {
const code = char.codePointAt(0);
if (code === undefined) {
continue;
}
if (
(code >= 0x3400 && code <= 0x4dbf) ||
(code >= 0x4e00 && code <= 0x9fff) ||
(code >= 0xf900 && code <= 0xfaff)
) {
return true;
}
}
return false;
}
function isExcludedComponent(
pos1: string | undefined,
pos2: string | undefined,
@@ -283,34 +271,6 @@ function isFrequencyExcludedByPos(
);
}
function shouldKeepFrequencyForNonIndependentKanjiNoun(
token: MergedToken,
pos1Exclusions: ReadonlySet<string>,
): boolean {
if (pos1Exclusions.has('名詞')) {
return false;
}
const rank =
typeof token.frequencyRank === 'number' && Number.isFinite(token.frequencyRank)
? Math.max(1, Math.floor(token.frequencyRank))
: null;
if (rank === null) {
return false;
}
const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
const pos2Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos2));
if (pos1Parts.length !== 1 || pos2Parts.length !== 1) {
return false;
}
if (pos1Parts[0] !== '名詞' || pos2Parts[0] !== '非自立') {
return false;
}
return hasKanjiChar(token.surface) || hasKanjiChar(token.headword);
}
export function shouldExcludeTokenFromVocabularyPersistence(
token: MergedToken,
options: Pick<AnnotationStageOptions, 'pos1Exclusions' | 'pos2Exclusions'> = {},
@@ -320,7 +280,8 @@ export function shouldExcludeTokenFromVocabularyPersistence(
return (
sharedShouldExcludeTokenFromSubtitleAnnotations(token, { pos1Exclusions, pos2Exclusions }) ||
isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions)
(isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) &&
!isKanjiNonIndependentNounToken(token, pos1Exclusions))
);
}
@@ -697,7 +658,7 @@ function resolveKnownWordReadingForMatch(
function computeTokenKnownStatus(
token: MergedToken,
isKnownWord: (text: string, reading?: string) => boolean,
isKnownWord: AnnotationStageDeps['isKnownWord'],
knownWordMatchMode: NPlusOneMatchMode,
): boolean {
const matchText = resolveKnownWordText(token.surface, token.headword, knownWordMatchMode);
@@ -711,7 +672,14 @@ function computeTokenKnownStatus(
return false;
}
return fallbackReading !== matchText.trim() && isKnownWord(fallbackReading);
// This fallback covers words mined in kana (token 大体, mined word だいたい),
// so the reading must only match mined word texts — the cache's reading-only
// index would let any kanji token match an unrelated note that shares its
// reading (渓谷/けいこく vs a mined 警告/けいこく).
return (
fallbackReading !== matchText.trim() &&
isKnownWord(fallbackReading, undefined, { allowReadingOnlyMatch: false })
);
}
function filterTokenFrequencyRank(
@@ -721,7 +689,7 @@ function filterTokenFrequencyRank(
): number | undefined {
if (
isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) &&
!shouldKeepFrequencyForNonIndependentKanjiNoun(token, pos1Exclusions)
!isKanjiNonIndependentNounToken(token, pos1Exclusions)
) {
return undefined;
}
@@ -778,7 +746,13 @@ export function annotateTokens(
: false;
nPlusOneKnownStatuses[index] = isKnownForMatching;
const prioritizedNameMatch = nameMatchEnabled && token.isNameMatch === true;
// A confirmed character-name match must survive the POS noise filter:
// MeCab can tag a name like 平 as a prefix (接頭詞) depending on context,
// which would otherwise strip the name match and its portrait.
if (
!prioritizedNameMatch &&
sharedShouldExcludeTokenFromSubtitleAnnotations(token, {
pos1Exclusions,
pos2Exclusions,
@@ -794,8 +768,6 @@ export function annotateTokens(
};
}
const prioritizedNameMatch = nameMatchEnabled && token.isNameMatch === true;
const frequencyRank =
frequencyEnabled && !prioritizedNameMatch
? filterTokenFrequencyRank(token, pos1Exclusions, pos2Exclusions)
@@ -128,6 +128,65 @@ test('drops scanning parser tokens which have no dictionary headword', () => {
);
});
// Regression: 「…とこ戻ろ…」 — Yomitan cannot deinflect the truncated volitional 戻ろ
// (the bare ろ rule is ichidan-only, 戻る is godan), so 戻 comes back with no headword
// while ろ matches an unrelated term (櫓). Dropping 戻 made it a plain text node:
// unhoverable, unannotated, and invisible to the n+1 candidate count.
test('emits unparsed non-caption text as a token with surface headword', () => {
const parseResults = [
makeParseItem('scanning-parser', [
[{ text: 'みんな', reading: 'みんな', headword: '皆' }],
[{ text: 'の', reading: 'の', headword: 'の' }],
[{ text: 'とこ', reading: 'とこ', headword: '所' }],
[{ text: '戻', reading: '' }],
[{ text: 'ろ', reading: 'ろ', headword: '櫓' }],
[{ text: '…', reading: '' }],
]),
];
const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
assert.deepEqual(
tokens?.map((token) => ({
surface: token.surface,
headword: token.headword,
isUnparsedRun: token.isUnparsedRun ?? false,
})),
[
{ surface: 'みんな', headword: '皆', isUnparsedRun: false },
{ surface: 'の', headword: 'の', isUnparsedRun: false },
{ surface: 'とこ', headword: '所', isUnparsedRun: false },
{ surface: '戻', headword: '戻', isUnparsedRun: true },
{ surface: 'ろ', headword: '櫓', isUnparsedRun: false },
],
);
});
test('still drops punctuation-only and whitespace-only unparsed runs', () => {
const parseResults = [
makeParseItem('scanning-parser', [
[{ text: '猫', reading: 'ねこ', headword: '猫' }],
[{ text: '…', reading: '' }],
[{ text: ' ', reading: '' }],
[{ text: '犬', reading: 'いぬ', headword: '犬' }],
]),
];
const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
assert.equal(tokens?.map((token) => token.surface).join(','), '猫,犬');
});
test('candidate with only unparsed tokens still yields no dictionary match', () => {
const parseResults = [
makeParseItem('scanning-parser', [
[{ text: '戻', reading: '' }],
[{ text: '轟', reading: '' }],
]),
];
const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
assert.equal(tokens, null);
});
test('prefers the longest dictionary headword across merged segments', () => {
const parseResults = [
makeParseItem('scanning-parser', [
@@ -151,6 +151,39 @@ function isStandaloneGrammarEndingSegment(segment: YomitanParseSegment): boolean
);
}
// Caption-style asides (SFX/speaker labels) start with a bracket and should stay
// non-interactive; dropping punctuation-only runs also keeps the source-text gap that
// sentence-boundary detection relies on (e.g. a dropped 「…」).
const CAPTION_OPENING_BRACKETS = new Set([
'(',
'',
'[',
'',
'{',
'',
'「',
'『',
'【',
'〈',
'《',
'≪',
'',
'<',
]);
function hasLookupWorthyText(text: string): boolean {
return /[\p{L}\p{N}]/u.test(text);
}
function isCaptionLikeUnparsedText(text: string): boolean {
const firstChar = Array.from(text.trim())[0];
return firstChar !== undefined && CAPTION_OPENING_BRACKETS.has(firstChar);
}
function shouldEmitUnparsedRunAsToken(text: string): boolean {
return hasLookupWorthyText(text) && !isCaptionLikeUnparsedText(text);
}
function shouldMergeKanaContinuation(
previousToken: MergedToken | undefined,
continuationSurface: string,
@@ -206,6 +239,7 @@ export function mapYomitanParseResultItemToMergedTokens(
headword: string,
start: number,
end: number,
isUnparsedRun = false,
): void => {
tokens.push({
surface,
@@ -221,6 +255,7 @@ export function mapYomitanParseResultItemToMergedTokens(
const matchText = resolveKnownWordText(surface, headword, knownWordMatchMode);
return matchText ? isKnownWord(matchText) : false;
})(),
...(isUnparsedRun ? { isUnparsedRun: true } : {}),
});
};
@@ -241,6 +276,12 @@ export function mapYomitanParseResultItemToMergedTokens(
previousToken.surface += combinedSurface;
previousToken.reading += combinedReading;
previousToken.endPos = end;
} else if (shouldEmitUnparsedRunAsToken(combinedSurface)) {
// Yomitan couldn't parse this run (e.g. 戻ろ… truncated volitional, ぅ~
// elongations). Keep it as a token with its surface as headword so it stays
// hoverable, but flag it so annotation/N+1/vocab logic ignores it — there is
// no dictionary entry behind it.
pushToken(combinedSurface, combinedReading, combinedSurface, combinedStart, end, true);
}
} else {
hasDictionaryMatch = true;
@@ -47,7 +47,6 @@ export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
'へえ',
'ふう',
'ほう',
'やはり',
'何か',
'何だ',
'何も',
@@ -55,7 +54,6 @@ export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
'有る',
'在る',
'様',
'確かに',
'誰も',
'貴方',
'もんか',
@@ -139,6 +137,46 @@ function resolvePos2Exclusions(options: SubtitleAnnotationFilterOptions = {}): R
return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
}
function hasKanjiChar(text: string): boolean {
for (const char of text) {
const code = char.codePointAt(0);
if (code === undefined) {
continue;
}
if (
(code >= 0x3400 && code <= 0x4dbf) ||
(code >= 0x4e00 && code <= 0x9fff) ||
(code >= 0xf900 && code <= 0xfaff)
) {
return true;
}
}
return false;
}
// Kanji-bearing non-independent nouns (日, 方, 上, …) are real vocabulary that
// Yomitan segments as standalone tokens; MeCab's 非自立 tag exists to suppress
// kana grammar nouns (こと, もの, とき) and must not hide these.
export function isKanjiNonIndependentNounToken(
token: MergedToken,
pos1Exclusions: ReadonlySet<string>,
): boolean {
if (pos1Exclusions.has('名詞')) {
return false;
}
const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
if (pos1Parts.length !== 1 || pos2Parts.length !== 1) {
return false;
}
if (pos1Parts[0] !== '名詞' || pos2Parts[0] !== '非自立') {
return false;
}
return hasKanjiChar(token.surface) || hasKanjiChar(token.headword);
}
function normalizeKana(text: string): string {
const raw = text.trim();
if (!raw) {
@@ -436,6 +474,13 @@ export function shouldExcludeTokenFromSubtitleAnnotations(
token: MergedToken,
options: SubtitleAnnotationFilterOptions = {},
): boolean {
// No Yomitan dictionary entry backs this token (ぅ~ elongations, truncated
// inflections) — it exists only to stay hoverable and must never receive
// annotations or count in the N+1 math.
if (token.isUnparsedRun === true) {
return true;
}
const pos1Exclusions = resolvePos1Exclusions(options);
const pos2Exclusions = resolvePos2Exclusions(options);
const normalizedPos1 = normalizePosTag(token.pos1);
@@ -447,7 +492,10 @@ export function shouldExcludeTokenFromSubtitleAnnotations(
return true;
}
if (isExcludedByTagSet(normalizedPos2, pos2Exclusions)) {
if (
isExcludedByTagSet(normalizedPos2, pos2Exclusions) &&
!isKanjiNonIndependentNounToken(token, pos1Exclusions)
) {
return true;
}
@@ -820,6 +820,135 @@ test('requestYomitanScanTokens keeps scanner metadata when parse spans agree', a
]);
});
test('requestYomitanScanTokens keeps scanner metadata for matching spans when parse segmentation has filler chunks', async () => {
const deps = createDeps(async (script) => {
if (script.includes('optionsGetFull')) {
return {
profileCurrent: 0,
profiles: [
{
options: {
scanning: { length: 40 },
},
},
],
};
}
if (script.includes('parseText')) {
return [
{
source: 'scanning-parser',
index: 0,
content: [
[
{
text: 'や',
reading: '',
headwords: [[{ term: 'や' }]],
},
],
[
{
text: 'ほ',
reading: '',
headwords: [[{ term: '帆' }]],
},
],
[
{
text: 'っ ',
reading: '',
},
],
[
{
text: 'ミナト',
reading: '',
headwords: [[{ term: 'ミナト' }]],
},
],
],
},
];
}
// The termsFind scanner skips the unmatched っ+space chunk, so its spans
// do not line up 1:1 with the parseText segmentation above.
return [
{
surface: 'や',
reading: 'や',
headword: 'や',
headwordReading: 'や',
startPos: 0,
endPos: 1,
frequencyRank: 57,
},
{
surface: 'ほ',
reading: 'ほ',
headword: '帆',
headwordReading: 'ほ',
startPos: 1,
endPos: 2,
frequencyRank: 15414,
},
{
surface: 'ミナト',
reading: 'ミナト',
headword: 'ミナト',
headwordReading: 'みなと',
startPos: 4,
endPos: 7,
isNameMatch: true,
frequencyRank: 75133,
},
];
});
const result = await requestYomitanScanTokens('やほっ ミナト', deps, {
error: () => undefined,
});
assert.deepEqual(result, [
{
surface: 'や',
reading: 'や',
headword: 'や',
headwordReading: 'や',
startPos: 0,
endPos: 1,
frequencyRank: 57,
},
{
surface: 'ほ',
reading: 'ほ',
headword: '帆',
headwordReading: 'ほ',
startPos: 1,
endPos: 2,
frequencyRank: 15414,
},
{
surface: 'っ ',
reading: '',
headword: 'っ ',
startPos: 2,
endPos: 4,
isUnparsedRun: true,
},
{
surface: 'ミナト',
reading: 'ミナト',
headword: 'ミナト',
headwordReading: 'みなと',
startPos: 4,
endPos: 7,
isNameMatch: true,
frequencyRank: 75133,
},
]);
});
test('requestYomitanScanTokens falls back to left-to-right termsFind scanning', async () => {
const scripts: string[] = [];
const deps = createDeps(async (script) => {
@@ -975,6 +1104,105 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF
]);
});
test('requestYomitanScanTokens retries shorter windows when a greedy match has no exact-source headword', async () => {
let scannerScript = '';
const deps = createDeps(async (script) => {
if (script.includes('termsFind')) {
scannerScript = script;
return [];
}
if (script.includes('optionsGetFull')) {
return {
profileCurrent: 0,
profileIndex: 0,
scanLength: 40,
dictionaries: ['JMdict'],
dictionaryPriorityByName: { JMdict: 0 },
dictionaryFrequencyModeByName: {},
profiles: [
{
options: {
scanning: { length: 40 },
dictionaries: [{ name: 'JMdict', enabled: true, id: 0 }],
},
},
],
};
}
return null;
});
await requestYomitanScanTokens('平 (平)', deps, {
error: () => undefined,
});
const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
if (action !== 'termsFind') {
throw new Error(`unexpected action: ${action}`);
}
const text = (params as { text?: string } | undefined)?.text ?? '';
if (!text.startsWith('平')) {
return { originalTextLength: 0, dictionaryEntries: [] };
}
if (text.length >= 4) {
// Simulates Yomitan normalization consuming punctuation/whitespace:
// the greedy match spans 平 (平 but no headword source equals it.
return {
originalTextLength: 4,
dictionaryEntries: [
{
headwords: [
{
term: '平々',
reading: 'へいへい',
sources: [{ originalText: '平平', isPrimary: true, matchType: 'exact' }],
},
],
},
],
};
}
return {
originalTextLength: 1,
dictionaryEntries: [
{
headwords: [
{
term: '平',
reading: 'たいら',
sources: [{ originalText: '平', isPrimary: true, matchType: 'exact' }],
},
],
},
],
};
});
assert.deepEqual(result, [
{
surface: '平',
reading: 'たいら',
headword: '平',
headwordReading: 'たいら',
startPos: 0,
endPos: 1,
isNameMatch: false,
frequencyRank: undefined,
},
{
surface: '平',
reading: 'たいら',
headword: '平',
headwordReading: 'たいら',
startPos: 3,
endPos: 4,
isNameMatch: false,
frequencyRank: undefined,
},
]);
});
test('requestYomitanScanTokens emits complete readings for kanji-kana compounds', async () => {
let scannerScript = '';
const deps = createDeps(async (script) => {
@@ -1662,6 +1890,364 @@ test('requestYomitanScanTokens accepts SubMiner character entries with structure
assert.equal((result as Array<{ isNameMatch?: boolean }>)[0]?.isNameMatch, true);
});
test('requestYomitanScanTokens greedily tokenizes character names before longer generic matches', async () => {
let scannerScript = '';
const deps = createDeps(async (script) => {
if (script.includes('termsFind')) {
scannerScript = script;
return [];
}
if (script.includes('optionsGetFull')) {
return {
profileCurrent: 0,
profiles: [
{
options: {
scanning: { length: 40 },
dictionaries: [
{ name: 'JMdict', enabled: true },
{ name: 'SubMiner Character Dictionary (AniList 130298)', enabled: true },
],
},
},
],
};
}
return null;
});
await requestYomitanScanTokens(
'美姫とヨータ',
deps,
{ error: () => undefined },
{ includeNameMatchMetadata: true },
);
assert.match(scannerScript, /const greedyNameScanEnabled = true;/);
const nameEntry = (term: string, reading: string) => ({
headwords: [
{
term,
reading,
sources: [{ originalText: term, isPrimary: true, matchType: 'exact' }],
},
],
definitions: [
{
dictionary: 'SubMiner Character Dictionary (AniList 130298)',
dictionaryAlias: 'SubMiner Character Dictionary (AniList 130298)',
},
],
});
const jmdictEntry = (term: string, reading: string, originalText: string) => ({
headwords: [
{
term,
reading,
sources: [{ originalText, isPrimary: true, matchType: 'exact' }],
},
],
definitions: [{ dictionary: 'JMdict', dictionaryAlias: 'JMdict' }],
});
const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
if (action !== 'termsFind') {
throw new Error(`unexpected action: ${action}`);
}
const text = (params as { text?: string } | undefined)?.text ?? '';
if (text.startsWith('美姫')) {
return { originalTextLength: 2, dictionaryEntries: [nameEntry('美姫', 'みき')] };
}
if (text.startsWith('とヨータ')) {
// Greedy generic match: とヨー normalizes to とよう (渡洋). Without the
// name pre-pass this consumes the ヨ of ヨータ.
return {
originalTextLength: 3,
dictionaryEntries: [jmdictEntry('渡洋', 'とよう', 'とヨー'), jmdictEntry('と', 'と', 'と')],
};
}
if (text.startsWith('ヨータ')) {
return { originalTextLength: 3, dictionaryEntries: [nameEntry('ヨータ', 'よーた')] };
}
if (text === 'と') {
return { originalTextLength: 1, dictionaryEntries: [jmdictEntry('と', 'と', 'と')] };
}
return { originalTextLength: 0, dictionaryEntries: [] };
});
assert.equal(Array.isArray(result), true);
assert.deepEqual(
(result as Array<Record<string, unknown>>).map(
({ surface, headword, startPos, endPos, isNameMatch }) => ({
surface,
headword,
startPos,
endPos,
isNameMatch,
}),
),
[
{ surface: '美姫', headword: '美姫', startPos: 0, endPos: 2, isNameMatch: true },
{ surface: 'と', headword: 'と', startPos: 2, endPos: 3, isNameMatch: false },
{ surface: 'ヨータ', headword: 'ヨータ', startPos: 3, endPos: 6, isNameMatch: true },
],
);
});
test('requestYomitanScanTokens lets a longer generic word beat a shorter name at the same position', async () => {
let scannerScript = '';
const deps = createDeps(async (script) => {
if (script.includes('termsFind')) {
scannerScript = script;
return [];
}
if (script.includes('optionsGetFull')) {
return {
profileCurrent: 0,
profiles: [
{
options: {
scanning: { length: 40 },
dictionaries: [
{ name: 'JMdict', enabled: true },
{ name: 'SubMiner Character Dictionary (AniList 130298)', enabled: true },
],
},
},
],
};
}
return null;
});
await requestYomitanScanTokens(
'空気変わって',
deps,
{ error: () => undefined },
{ includeNameMatchMetadata: true },
);
assert.match(scannerScript, /const greedyNameScanEnabled = true;/);
const nameEntry = (term: string, reading: string) => ({
headwords: [
{
term,
reading,
sources: [{ originalText: term, isPrimary: true, matchType: 'exact' }],
},
],
definitions: [
{
dictionary: 'SubMiner Character Dictionary (AniList 130298)',
dictionaryAlias: 'SubMiner Character Dictionary (AniList 130298)',
},
],
});
const jmdictEntry = (term: string, reading: string, originalText: string) => ({
headwords: [
{
term,
reading,
sources: [{ originalText, isPrimary: true, matchType: 'exact' }],
},
],
definitions: [{ dictionary: 'JMdict', dictionaryAlias: 'JMdict' }],
});
const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
if (action !== 'termsFind') {
throw new Error(`unexpected action: ${action}`);
}
const text = (params as { text?: string } | undefined)?.text ?? '';
if (text.startsWith('空気')) {
// A character named 空 matches here, but the generic 空気 is longer and
// must win the position.
return {
originalTextLength: 2,
dictionaryEntries: [nameEntry('空', 'くう'), jmdictEntry('空気', 'くうき', '空気')],
};
}
if (text.startsWith('変わって')) {
return {
originalTextLength: 4,
dictionaryEntries: [jmdictEntry('変わる', 'かわる', '変わって')],
};
}
return { originalTextLength: 0, dictionaryEntries: [] };
});
assert.equal(Array.isArray(result), true);
assert.deepEqual(
(result as Array<Record<string, unknown>>).map(
({ surface, headword, startPos, endPos, isNameMatch }) => ({
surface,
headword,
startPos,
endPos,
isNameMatch,
}),
),
[
{ surface: '空気', headword: '空気', startPos: 0, endPos: 2, isNameMatch: false },
{ surface: '変わって', headword: '変わる', startPos: 2, endPos: 6, isNameMatch: false },
],
);
});
test('requestYomitanScanTokens skips greedy name scan without an enabled character dictionary', async () => {
let scannerScript = '';
const deps = createDeps(async (script) => {
if (script.includes('termsFind')) {
scannerScript = script;
return [];
}
if (script.includes('optionsGetFull')) {
return {
profileCurrent: 0,
profiles: [
{
options: {
scanning: { length: 40 },
dictionaries: [{ name: 'JMdict', enabled: true }],
},
},
],
};
}
return null;
});
await requestYomitanScanTokens(
'アクア',
deps,
{ error: () => undefined },
{ includeNameMatchMetadata: true },
);
assert.match(scannerScript, /const greedyNameScanEnabled = false;/);
});
test('requestYomitanScanTokens replaces parseText segmentation where greedy name tokens re-segment', async () => {
const deps = createDeps(async (script) => {
if (script.includes('optionsGetFull')) {
return {
profileCurrent: 0,
profiles: [
{
options: {
scanning: { length: 40 },
dictionaries: [
{ name: 'JMdict', enabled: true },
{ name: 'SubMiner Character Dictionary (AniList 130298)', enabled: true },
],
},
},
],
};
}
if (script.includes('parseText')) {
// parseText walks greedily too, so it merges と with ヨー into 渡洋 and
// strands the タ.
return [
{
source: 'scanning-parser',
index: 0,
content: [
[
{
text: '美姫',
reading: 'みき',
headwords: [[{ term: '美姫' }]],
},
],
[
{
text: 'とヨー',
reading: 'とよう',
headwords: [[{ term: '渡洋' }]],
},
],
[
{
text: 'タ',
reading: '',
},
],
],
},
];
}
return [
{
surface: '美姫',
reading: 'みき',
headword: '美姫',
headwordReading: 'みき',
startPos: 0,
endPos: 2,
isNameMatch: true,
},
{
surface: 'と',
reading: 'と',
headword: 'と',
headwordReading: 'と',
startPos: 2,
endPos: 3,
isNameMatch: false,
},
{
surface: 'ヨータ',
reading: 'ヨータ',
headword: 'ヨータ',
headwordReading: 'よーた',
startPos: 3,
endPos: 6,
isNameMatch: true,
},
];
});
const result = await requestYomitanScanTokens(
'美姫とヨータ',
deps,
{ error: () => undefined },
{ includeNameMatchMetadata: true },
);
assert.deepEqual(result, [
{
surface: '美姫',
reading: 'みき',
headword: '美姫',
headwordReading: 'みき',
startPos: 0,
endPos: 2,
isNameMatch: true,
},
{
surface: 'と',
reading: 'と',
headword: 'と',
headwordReading: 'と',
startPos: 2,
endPos: 3,
isNameMatch: false,
},
{
surface: 'ヨータ',
reading: 'ヨータ',
headword: 'ヨータ',
headwordReading: 'よーた',
startPos: 3,
endPos: 6,
isNameMatch: true,
},
]);
});
test('requestYomitanScanTokens preserves matched headword word classes', async () => {
let scannerScript = '';
const deps = createDeps(async (script) => {
@@ -57,6 +57,7 @@ export interface YomitanScanToken {
isNameMatch?: boolean;
frequencyRank?: number;
wordClasses?: string[];
isUnparsedRun?: boolean;
}
interface YomitanProfileMetadata {
@@ -73,6 +74,7 @@ export interface YomitanAddNoteResult {
}
const DEFAULT_YOMITAN_SCAN_LENGTH = 40;
const CHARACTER_DICTIONARY_TITLE_PREFIX = 'SubMiner Character Dictionary';
const yomitanProfileMetadataByWindow = new WeakMap<BrowserWindow, YomitanProfileMetadata>();
const yomitanProfileDiagnosticsLoggedByWindow = new WeakSet<BrowserWindow>();
const yomitanFrequencyCacheByWindow = new WeakMap<
@@ -105,20 +107,89 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
);
}
function hasSameTokenSpans(left: YomitanScanToken[], right: YomitanScanToken[]): boolean {
if (left.length !== right.length) {
return false;
function scanTokenSpanKey(token: YomitanScanToken): string {
return `${token.startPos}:${token.endPos}:${token.surface}`;
}
// Maps a parse-selected token to the scanner-token shape carried out of the
// parser runtime. Shared by both selectYomitanParseTokens fallback paths so the
// projected fields stay in sync as the shape changes.
function toYomitanScanToken(token: {
surface: string;
reading: string;
headword: string;
startPos: number;
endPos: number;
isUnparsedRun?: boolean;
}): YomitanScanToken {
return {
surface: token.surface,
reading: token.reading,
headword: token.headword,
startPos: token.startPos,
endPos: token.endPos,
...(token.isUnparsedRun === true ? { isUnparsedRun: true } : {}),
};
}
// parseText segmentation is authoritative (it emits filler chunks for text the
// termsFind scanner skips), but only the termsFind scanner carries annotation
// metadata (isNameMatch, frequencyRank, headwordReading, wordClasses). Graft
// scanner tokens onto the parseText segmentation per matching span so one
// unmatched chunk degrades only itself instead of dropping the whole line's
// metadata.
//
// Exception: character-name tokens. The greedy name scan can re-segment text
// around a name (e.g. とヨータ → と + ヨータ instead of とヨー + タ), so
// parseText segmentation cannot be authoritative there. Each name span is
// expanded until it aligns with token boundaries in both segmentations, then
// the parse tokens inside are replaced with the scanner tokens.
function mergeScannerTokensIntoParseTokens(
parseScanTokens: YomitanScanToken[],
scannerTokens: YomitanScanToken[],
): YomitanScanToken[] {
const scannerTokensBySpan = new Map<string, YomitanScanToken>();
for (const token of scannerTokens) {
scannerTokensBySpan.set(scanTokenSpanKey(token), token);
}
const graftedTokens = parseScanTokens.map(
(token) => scannerTokensBySpan.get(scanTokenSpanKey(token)) ?? token,
);
const nameTokens = scannerTokens.filter((token) => token.isNameMatch === true);
if (nameTokens.length === 0) {
return graftedTokens;
}
return left.every((token, index) => {
const other = right[index];
return (
other !== undefined &&
token.surface === other.surface &&
token.startPos === other.startPos &&
token.endPos === other.endPos
);
});
const regions = nameTokens.map((token) => ({ start: token.startPos, end: token.endPos }));
const allTokens = [...parseScanTokens, ...scannerTokens];
let expanded = true;
while (expanded) {
expanded = false;
for (const region of regions) {
for (const token of allTokens) {
const overlaps = token.startPos < region.end && token.endPos > region.start;
const extendsBeyond = token.startPos < region.start || token.endPos > region.end;
if (overlaps && extendsBeyond) {
region.start = Math.min(region.start, token.startPos);
region.end = Math.max(region.end, token.endPos);
expanded = true;
}
}
}
}
const isInsideNameRegion = (token: YomitanScanToken): boolean =>
regions.some((region) => token.startPos >= region.start && token.endPos <= region.end);
const merged = graftedTokens.filter((token) => !isInsideNameRegion(token));
for (const token of scannerTokens) {
if (isInsideNameRegion(token)) {
merged.push(token);
}
}
merged.sort((a, b) => a.startPos - b.startPos || a.endPos - b.endPos);
return merged;
}
function makeTermReadingCacheKey(term: string, reading: string | null): string {
@@ -1104,8 +1175,7 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
}
return best;
}
function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
function normalizeWordClasses(headword) {
function normalizeWordClasses(headword) {
if (!Array.isArray(headword?.wordClasses)) { return undefined; }
const classes = headword.wordClasses.filter((wordClass) => typeof wordClass === "string" && wordClass.trim().length > 0);
return classes.length > 0 ? classes : undefined;
@@ -1146,7 +1216,7 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
if (!includeNameMatchMetadata || !entry || typeof entry !== 'object') {
return false;
}
return getDictionaryEntryNames(entry).some((name) => name.startsWith("SubMiner Character Dictionary"));
return getDictionaryEntryNames(entry).some((name) => name.startsWith(${JSON.stringify(CHARACTER_DICTIONARY_TITLE_PREFIX)}));
}
function parseSubMinerMediaIdFromString(value) {
const imageMatch = value.match(/\bimg\/m(\d+)-/i);
@@ -1154,7 +1224,7 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
const parsed = Number.parseInt(imageMatch[1], 10);
if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
}
const titleMatch = value.match(/SubMiner Character Dictionary[^\d]*(?:AniList\s*)?(\d+)/i);
const titleMatch = value.match(/${CHARACTER_DICTIONARY_TITLE_PREFIX}[^\d]*(?:AniList\s*)?(\d+)/i);
if (titleMatch) {
const parsed = Number.parseInt(titleMatch[1], 10);
if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
@@ -1214,7 +1284,43 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
}
const mediaIds = getSubMinerMediaIds(entry);
return mediaIds.size === 0 || mediaIds.has(currentCharacterDictionaryMediaId);
}
function findLongestNameMatch(dictionaryEntries, textWindow) {
let best = null;
for (const dictionaryEntry of dictionaryEntries || []) {
if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; }
const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
const headword = headwords[headwordIndex];
for (const src of headword?.sources || []) {
if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
const originalText = typeof src.originalText === 'string' ? src.originalText : '';
if (!originalText || !textWindow.startsWith(originalText)) { continue; }
if (best === null || originalText.length > best.sourceLength) {
best = { dictionaryEntry, headword, headwordIndex, sourceLength: originalText.length };
}
}
}
}
return best;
}
function findLongestGenericMatchLength(dictionaryEntries, textWindow) {
let best = 0;
for (const dictionaryEntry of dictionaryEntries || []) {
if (isNameDictionaryEntry(dictionaryEntry)) { continue; }
const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
for (const headword of headwords) {
for (const src of headword?.sources || []) {
if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
const originalText = typeof src.originalText === 'string' ? src.originalText : '';
if (!originalText || !textWindow.startsWith(originalText)) { continue; }
if (originalText.length > best) { best = originalText.length; }
}
}
}
return best;
}
function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
const currentMediaDictionaryEntries =
currentCharacterDictionaryMediaId === null
? (dictionaryEntries || [])
@@ -1261,6 +1367,7 @@ function buildYomitanScanningScript(
profileIndex: number,
scanLength: number,
includeNameMatchMetadata: boolean,
greedyNameScanEnabled: boolean,
currentCharacterDictionaryMediaId: number | null,
dictionaryPriorityByName: Record<string, number>,
dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>,
@@ -1287,6 +1394,7 @@ function buildYomitanScanningScript(
});
${YOMITAN_SCANNING_HELPERS}
const includeNameMatchMetadata = ${includeNameMatchMetadata ? 'true' : 'false'};
const greedyNameScanEnabled = ${greedyNameScanEnabled ? 'true' : 'false'};
const currentCharacterDictionaryMediaId = ${
currentCharacterDictionaryMediaId !== null
? String(currentCharacterDictionaryMediaId)
@@ -1297,47 +1405,136 @@ ${YOMITAN_SCANNING_HELPERS}
const text = ${JSON.stringify(text)};
const details = {matchType: "exact", deinflect: true};
const tokens = [];
let i = 0;
while (i < text.length) {
const codePoint = text.codePointAt(i);
const character = String.fromCodePoint(codePoint);
const substring = text.substring(i, i + ${scanLength});
const termsFindCache = new Map();
async function termsFindAt(position, windowLength) {
const cacheKey = position + ":" + windowLength;
const cached = termsFindCache.get(cacheKey);
if (cached) { return cached; }
const substring = text.substring(position, position + windowLength);
const result = await invoke("termsFind", { text: substring, details, optionsContext: { index: ${profileIndex} } });
termsFindCache.set(cacheKey, result);
return result;
}
function buildScanToken(position, source, preferredHeadword) {
const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
const tokenPayload = {
surface: segments.map((segment) => segment.text).join("") || source,
reading: segments.map(getSegmentReadingContribution).join(""),
headword: preferredHeadword.term,
headwordReading: reading || undefined,
startPos: position,
endPos: position + source.length,
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
frequencyRank:
typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
: undefined,
};
if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
tokenPayload.wordClasses = preferredHeadword.wordClasses;
}
return tokenPayload;
}
async function findTokenAt(position, windowLength) {
const codePoint = text.codePointAt(position);
const character = String.fromCodePoint(codePoint);
const result = await termsFindAt(position, windowLength);
const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
const originalTextLength = typeof result?.originalTextLength === "number" ? result.originalTextLength : 0;
if (dictionaryEntries.length > 0 && originalTextLength > 0 && (originalTextLength !== character.length || isCodePointJapanese(codePoint))) {
const source = substring.substring(0, originalTextLength);
const preferredHeadword = getPreferredHeadword(
dictionaryEntries,
source,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
);
if (preferredHeadword && typeof preferredHeadword.term === "string") {
const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
const tokenPayload = {
surface: segments.map((segment) => segment.text).join("") || source,
reading: segments.map(getSegmentReadingContribution).join(""),
headword: preferredHeadword.term,
headwordReading: reading || undefined,
startPos: i,
endPos: i + originalTextLength,
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
frequencyRank:
typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
: undefined,
};
if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
tokenPayload.wordClasses = preferredHeadword.wordClasses;
}
tokens.push(tokenPayload);
i += originalTextLength;
if (dictionaryEntries.length === 0 || originalTextLength <= 0 || (originalTextLength === character.length && !isCodePointJapanese(codePoint))) {
return { token: null, matchedLength: 0 };
}
const source = text.substring(position, position + originalTextLength);
const preferredHeadword = getPreferredHeadword(
dictionaryEntries,
source,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
);
if (!preferredHeadword || typeof preferredHeadword.term !== "string") {
return { token: null, matchedLength: originalTextLength };
}
return { token: buildScanToken(position, source, preferredHeadword), matchedLength: originalTextLength };
}
// Greedy name pre-pass: character-name matches claim their spans before
// the left-to-right walk, so a longer generic match starting earlier
// (e.g. とヨー → 渡洋) cannot swallow the start of a name (ヨータ).
const nameTokens = [];
if (greedyNameScanEnabled) {
let namePos = 0;
while (namePos < text.length) {
const codePoint = text.codePointAt(namePos);
if (!isCodePointJapanese(codePoint)) {
namePos += String.fromCodePoint(codePoint).length;
continue;
}
const result = await termsFindAt(namePos, ${scanLength});
const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
const textWindow = text.substring(namePos, namePos + ${scanLength});
const nameMatch = findLongestNameMatch(dictionaryEntries, textWindow);
// A name only claims its span when no strictly longer generic word
// starts at the same position (a character named 空 must not split
// 空気). Ties go to the name. Generic matches that start earlier and
// overlap the name are still blocked by the reservation.
if (
!nameMatch ||
findLongestGenericMatchLength(dictionaryEntries, textWindow) > nameMatch.sourceLength
) {
namePos += String.fromCodePoint(codePoint).length;
continue;
}
const source = text.substring(namePos, namePos + nameMatch.sourceLength);
nameTokens.push(buildScanToken(namePos, source, {
term: nameMatch.headword.term,
reading: nameMatch.headword.reading,
wordClasses: normalizeWordClasses(nameMatch.headword),
isNameMatch: true,
frequencyRank: getBestFrequencyRank(
nameMatch.dictionaryEntry,
nameMatch.headwordIndex,
dictionaryPriorityByName,
dictionaryFrequencyModeByName
)
}));
namePos += nameMatch.sourceLength;
}
i += character.length;
}
let i = 0;
let nameIndex = 0;
while (i < text.length) {
while (nameIndex < nameTokens.length && nameTokens[nameIndex].startPos < i) { nameIndex += 1; }
const nextNameToken = nameIndex < nameTokens.length ? nameTokens[nameIndex] : null;
if (nextNameToken && nextNameToken.startPos === i) {
tokens.push(nextNameToken);
i = nextNameToken.endPos;
nameIndex += 1;
continue;
}
// Cap the window at the next reserved name span so a generic match
// cannot consume into it.
const windowLength = nextNameToken ? Math.min(${scanLength}, nextNameToken.startPos - i) : ${scanLength};
let attempt = await findTokenAt(i, windowLength);
// Yomitan text normalization can consume characters (whitespace,
// punctuation) beyond the matched term, leaving no headword whose
// source equals the consumed text. Retry with shorter windows so a
// valid prefix term (e.g. a character name before a paren) still
// tokenizes instead of the position being skipped.
let retryLength = Math.min(attempt.matchedLength, windowLength) - 1;
while (!attempt.token && retryLength >= 1) {
const retry = await findTokenAt(i, retryLength);
if (retry.token) {
attempt = retry;
break;
}
retryLength = Math.min(retryLength - 1, retry.matchedLength - 1);
}
if (attempt.token) {
tokens.push(attempt.token);
i += attempt.matchedLength;
continue;
}
i += String.fromCodePoint(text.codePointAt(i)).length;
}
return tokens;
})();
@@ -1460,18 +1657,17 @@ export async function requestYomitanScanTokens(
const parseResults = await requestYomitanParseResults(text, deps, logger);
const selectedParseTokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
const parseScanTokens =
selectedParseTokens?.map((token) => ({
surface: token.surface,
reading: token.reading,
headword: token.headword,
startPos: token.startPos,
endPos: token.endPos,
})) ?? null;
const parseScanTokens = selectedParseTokens?.map(toYomitanScanToken) ?? null;
const metadata = await requestYomitanProfileMetadata(parserWindow, logger);
const profileIndex = metadata?.profileIndex ?? 0;
const scanLength = metadata?.scanLength ?? DEFAULT_YOMITAN_SCAN_LENGTH;
const includeNameMatchMetadata = options?.includeNameMatchMetadata === true;
const greedyNameScanEnabled =
includeNameMatchMetadata &&
(metadata?.dictionaries ?? []).some((name) =>
name.startsWith(CHARACTER_DICTIONARY_TITLE_PREFIX),
);
try {
const rawResult = await parserWindow.webContents.executeJavaScript(
@@ -1479,7 +1675,8 @@ export async function requestYomitanScanTokens(
text,
profileIndex,
scanLength,
options?.includeNameMatchMetadata === true,
includeNameMatchMetadata,
greedyNameScanEnabled,
typeof options?.currentCharacterDictionaryMediaId === 'number' &&
Number.isFinite(options.currentCharacterDictionaryMediaId) &&
options.currentCharacterDictionaryMediaId > 0
@@ -1492,21 +1689,13 @@ export async function requestYomitanScanTokens(
);
if (isScanTokenArray(rawResult)) {
if (parseScanTokens && parseScanTokens.length > 0) {
return hasSameTokenSpans(parseScanTokens, rawResult) ? rawResult : parseScanTokens;
return mergeScannerTokensIntoParseTokens(parseScanTokens, rawResult);
}
return rawResult;
}
if (Array.isArray(rawResult)) {
const selectedTokens = selectYomitanParseTokens(rawResult, () => false, 'headword');
return (
selectedTokens?.map((token) => ({
surface: token.surface,
reading: token.reading,
headword: token.headword,
startPos: token.startPos,
endPos: token.endPos,
})) ?? null
);
return selectedTokens?.map(toYomitanScanToken) ?? null;
}
if (parseScanTokens && parseScanTokens.length > 0) {
return parseScanTokens;
+7 -1
View File
@@ -2373,6 +2373,11 @@ const characterDictionaryRuntime = createCharacterDictionaryRuntimeService({
getNameMatchImagesEnabled: () => getResolvedConfig().subtitleStyle.nameMatchImagesEnabled,
getCollapsibleSectionOpenState: (section) =>
getResolvedConfig().anilist.characterDictionary.collapsibleSections[section],
tokenizeJapaneseName: async (text) => (await appState.mecabTokenizer?.tokenize(text)) ?? null,
getJapaneseNameTokenizerAvailable: () => {
const status = appState.mecabTokenizer?.getStatus();
return status?.available === true && status.enabled === true;
},
now: () => Date.now(),
logInfo: (message) => logger.info(message),
logWarn: (message) => logger.warn(message),
@@ -4647,7 +4652,8 @@ const {
setYomitanParserInitPromise: (promise) => {
appState.yomitanParserInitPromise = promise;
},
isKnownWord: (text, reading) => Boolean(appState.ankiIntegration?.isKnownWord(text, reading)),
isKnownWord: (text, reading, options) =>
Boolean(appState.ankiIntegration?.isKnownWord(text, reading, options)),
recordLookup: (hit) => {
ensureImmersionTrackerStarted();
appState.immersionTracker?.recordLookup(hit);
+25 -8
View File
@@ -38,6 +38,7 @@ import {
createCharacterDictionaryManualSelectionStore,
} from './character-dictionary-runtime/manual-selection';
import { snapshotHasCharacterNameImages } from './character-dictionary-runtime/image-lookup';
import { resolveJapaneseNameSplits } from './character-dictionary-runtime/name-split-resolver';
import type {
AniListMediaCandidate,
CharacterDictionaryBuildResult,
@@ -175,11 +176,19 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
userDataPath: deps.userDataPath,
});
const shouldRefreshCachedSnapshot = (snapshot: CharacterDictionarySnapshot): boolean => {
if (deps.getNameMatchImagesEnabled?.() !== true) {
return false;
const isNameSplitTokenizerAvailable = (): boolean =>
typeof deps.tokenizeJapaneseName === 'function' &&
deps.getJapaneseNameTokenizerAvailable?.() === true;
const getCachedSnapshotRefreshReason = (snapshot: CharacterDictionarySnapshot): string | null => {
if (deps.getNameMatchImagesEnabled?.() === true && !snapshotHasCharacterNameImages(snapshot)) {
return 'missing cached character images';
}
return !snapshotHasCharacterNameImages(snapshot);
// Heuristic name splits are upgraded once MeCab becomes available.
if (snapshot.nameSplitSource !== 'mecab' && isNameSplitTokenizerAvailable()) {
return 'name splits predate MeCab availability';
}
return null;
};
const createAniListRequestSlot = (): (() => Promise<void>) => {
@@ -323,7 +332,8 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
): Promise<CharacterDictionarySnapshotResult> => {
const snapshotPath = getSnapshotPath(outputDir, mediaId);
const cachedSnapshot = readSnapshot(snapshotPath);
if (cachedSnapshot && !shouldRefreshCachedSnapshot(cachedSnapshot)) {
const refreshReason = cachedSnapshot ? getCachedSnapshotRefreshReason(cachedSnapshot) : null;
if (cachedSnapshot && refreshReason === null) {
deps.logInfo?.(`[dictionary] snapshot hit for AniList ${mediaId}`);
return {
mediaId: cachedSnapshot.mediaId,
@@ -334,9 +344,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
};
}
if (cachedSnapshot) {
deps.logInfo?.(
`[dictionary] snapshot stale for AniList ${mediaId}: missing cached character images`,
);
deps.logInfo?.(`[dictionary] snapshot stale for AniList ${mediaId}: ${refreshReason}`);
}
progress?.onGenerating?.({
@@ -399,6 +407,13 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
}
}
const nameSplitTokenizerAvailable = isNameSplitTokenizerAvailable();
const resolvedNameSplits = nameSplitTokenizerAvailable
? await resolveJapaneseNameSplits(characters, deps.tokenizeJapaneseName!, deps.logWarn)
: undefined;
const nameSplitSource =
resolvedNameSplits && resolvedNameSplits.size > 0 ? 'mecab' : 'heuristic';
const snapshot = buildSnapshotFromCharacters(
mediaId,
fetchedMediaTitle || mediaTitleHint || `AniList ${mediaId}`,
@@ -407,6 +422,8 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
imagesByVaId,
deps.now(),
getCollapsibleSectionOpenState,
resolvedNameSplits,
nameSplitSource,
);
writeSnapshot(snapshotPath, snapshot);
deps.logInfo?.(
@@ -36,7 +36,20 @@ test('writeSnapshot persists and readSnapshot restores current-format snapshots'
writeSnapshot(snapshotPath, snapshot);
assert.deepEqual(readSnapshot(snapshotPath), snapshot);
assert.deepEqual(readSnapshot(snapshotPath), { ...snapshot, nameSplitSource: 'heuristic' });
});
test('readSnapshot preserves the mecab name-split source and defaults missing values to heuristic', () => {
const outputDir = makeTempDir();
const snapshotPath = getSnapshotPath(outputDir, 130298);
const snapshot: CharacterDictionarySnapshot = {
...createSnapshot(),
nameSplitSource: 'mecab',
};
writeSnapshot(snapshotPath, snapshot);
assert.equal(readSnapshot(snapshotPath)?.nameSplitSource, 'mecab');
});
test('readSnapshot ignores snapshots written with an older format version', () => {
@@ -141,6 +141,7 @@ export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot
mediaTitle: parsed.mediaTitle,
entryCount: parsed.entryCount,
updatedAt: parsed.updatedAt,
nameSplitSource: parsed.nameSplitSource === 'mecab' ? 'mecab' : 'heuristic',
termEntries: parsed.termEntries as CharacterDictionaryTermEntry[],
images: parsed.images as CharacterDictionarySnapshotImage[],
};
@@ -1,7 +1,7 @@
export const ANILIST_GRAPHQL_URL = 'https://graphql.anilist.co';
export const ANILIST_REQUEST_DELAY_MS = 2000;
export const CHARACTER_IMAGE_DOWNLOAD_DELAY_MS = 250;
export const CHARACTER_DICTIONARY_FORMAT_VERSION = 18;
export const CHARACTER_DICTIONARY_FORMAT_VERSION = 19;
export const CHARACTER_DICTIONARY_MERGED_TITLE = 'SubMiner Character Dictionary';
export const HONORIFIC_SUFFIXES = [
@@ -1,5 +1,5 @@
import { HONORIFIC_SUFFIXES } from './constants';
import type { JapaneseNameParts, NameReadings } from './types';
import type { JapaneseNameParts, NameReadings, ResolvedNameSplits } from './types';
export function hasKanaOnly(value: string): boolean {
return /^[\u3040-\u309f\u30a0-\u30ffー]+$/.test(value);
@@ -262,7 +262,7 @@ export function buildReadingFromRomanized(value: string): string {
return katakana ? katakanaToHiragana(katakana) : '';
}
function buildReadingFromHint(value: string): string {
export function buildReadingFromHint(value: string): string {
return buildReading(value) || buildReadingFromRomanized(value);
}
@@ -273,20 +273,22 @@ function scoreJapaneseNamePartLength(length: number): number {
return 0;
}
function inferJapaneseNameSplitIndex(
// Ranks every possible family/given boundary. Reading-length ratios cannot
// always identify the true boundary (あずま can be one kanji or two), so
// callers may take the top candidates rather than trusting only the best.
function inferJapaneseNameSplitIndices(
nameOriginal: string,
firstNameHint: string,
lastNameHint: string,
): number | null {
): number[] {
const chars = [...nameOriginal];
if (chars.length < 2) return null;
if (chars.length < 2) return [];
const familyHintLength = [...buildReadingFromHint(lastNameHint)].length;
const givenHintLength = [...buildReadingFromHint(firstNameHint)].length;
const totalHintLength = familyHintLength + givenHintLength;
const defaultBoundary = Math.round(chars.length / 2);
let bestIndex: number | null = null;
let bestScore = Number.NEGATIVE_INFINITY;
const scored: Array<{ index: number; score: number }> = [];
for (let index = 1; index < chars.length; index += 1) {
const familyLength = index;
@@ -309,13 +311,10 @@ function inferJapaneseNameSplitIndex(
score += 0.25;
}
if (score > bestScore) {
bestScore = score;
bestIndex = index;
}
scored.push({ index, score });
}
return bestIndex;
return scored.sort((left, right) => right.score - left.score).map((entry) => entry.index);
}
export function addRomanizedKanaAliases(values: Iterable<string>): string[] {
@@ -335,6 +334,7 @@ export function splitJapaneseName(
nameOriginal: string,
firstNameHint?: string,
lastNameHint?: string,
resolvedSplits?: ResolvedNameSplits,
): JapaneseNameParts {
const trimmed = nameOriginal.trim();
if (!trimmed) {
@@ -377,6 +377,22 @@ export function splitJapaneseName(
};
}
const resolvedSplit = resolvedSplits?.get(trimmed);
if (
resolvedSplit &&
resolvedSplit.family &&
resolvedSplit.given &&
`${resolvedSplit.family}${resolvedSplit.given}` === trimmed
) {
return {
hasSpace: true,
original: trimmed,
combined: trimmed,
family: resolvedSplit.family,
given: resolvedSplit.given,
};
}
const hintedFirst = firstNameHint?.trim() || '';
const hintedLast = lastNameHint?.trim() || '';
if (hintedFirst && hintedLast) {
@@ -404,7 +420,7 @@ export function splitJapaneseName(
}
if (hintedFirst && hintedLast && containsKanji(trimmed)) {
const splitIndex = inferJapaneseNameSplitIndex(trimmed, hintedFirst, hintedLast);
const splitIndex = inferJapaneseNameSplitIndices(trimmed, hintedFirst, hintedLast)[0] ?? null;
if (splitIndex != null) {
const chars = [...trimmed];
const family = chars.slice(0, splitIndex).join('');
@@ -430,11 +446,62 @@ export function splitJapaneseName(
};
}
const MAX_INFERRED_SPLIT_CANDIDATES = 2;
// Returns the possible family/given splits, best first. Only a boundary
// guessed by the length heuristic is ambiguous (あずま can be one kanji or
// two), so only that path yields a runner-up candidate; explicit separators,
// resolved (MeCab) splits, and exact hint matches are trusted as-is.
export function splitJapaneseNameCandidates(
nameOriginal: string,
firstNameHint?: string,
lastNameHint?: string,
resolvedSplits?: ResolvedNameSplits,
): JapaneseNameParts[] {
const primary = splitJapaneseName(nameOriginal, firstNameHint, lastNameHint, resolvedSplits);
if (!primary.family || !primary.given) {
return [primary];
}
const trimmed = nameOriginal.trim();
if (primary.combined !== trimmed) {
return [primary];
}
const resolvedSplit = resolvedSplits?.get(trimmed);
if (resolvedSplit && `${resolvedSplit.family}${resolvedSplit.given}` === trimmed) {
return [primary];
}
const hintedFirst = firstNameHint?.trim() || '';
const hintedLast = lastNameHint?.trim() || '';
if (`${hintedLast}${hintedFirst}` === trimmed || `${hintedFirst}${hintedLast}` === trimmed) {
return [primary];
}
const candidates = [primary];
const chars = [...trimmed];
const splitIndices = inferJapaneseNameSplitIndices(trimmed, hintedFirst, hintedLast);
for (const splitIndex of splitIndices.slice(1, MAX_INFERRED_SPLIT_CANDIDATES)) {
const family = chars.slice(0, splitIndex).join('');
const given = chars.slice(splitIndex).join('');
if (family && given) {
candidates.push({
hasSpace: true,
original: trimmed,
combined: trimmed,
family,
given,
});
}
}
return candidates;
}
export function generateNameReadings(
nameOriginal: string,
romanizedName: string,
firstNameHint?: string,
lastNameHint?: string,
resolvedSplits?: ResolvedNameSplits,
): NameReadings {
const trimmed = nameOriginal.trim();
if (!trimmed) {
@@ -447,7 +514,7 @@ export function generateNameReadings(
};
}
const nameParts = splitJapaneseName(trimmed, firstNameHint, lastNameHint);
const nameParts = splitJapaneseName(trimmed, firstNameHint, lastNameHint, resolvedSplits);
if (!nameParts.hasSpace || !nameParts.family || !nameParts.given) {
const full = containsKanji(trimmed)
? buildReadingFromRomanized(romanizedName)
@@ -0,0 +1,191 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { resolveJapaneseNameSplits } from './name-split-resolver';
import { splitJapaneseName, splitJapaneseNameCandidates } from './name-reading';
import { buildNameTerms } from './term-building';
import type { CharacterRecord, NameSplitToken } from './types';
function characterRecord(overrides: Partial<CharacterRecord>): CharacterRecord {
return {
id: 302626,
role: 'main',
firstNameHint: 'Shino',
fullName: 'Shino Azuma',
lastNameHint: 'Azuma',
nativeName: '東紫乃',
alternativeNames: [],
bloodType: '',
birthday: null,
description: '',
imageUrl: null,
age: '',
sex: '',
voiceActors: [],
...overrides,
};
}
function personNameToken(word: string, role: '姓' | '名', katakanaReading: string): NameSplitToken {
return { word, pos1: '名詞', pos2: '固有名詞', pos3: '人名', pos4: role, katakanaReading };
}
function tokenizerFor(
tokensByName: Record<string, NameSplitToken[]>,
): (text: string) => Promise<NameSplitToken[] | null> {
return async (text) => tokensByName[text] ?? null;
}
test('resolveJapaneseNameSplits splits a single-kanji surname via person-name POS tags', async () => {
const splits = await resolveJapaneseNameSplits(
[characterRecord({})],
tokenizerFor({
: [personNameToken('東', '姓', 'アズマ'), personNameToken('紫乃', '名', 'シノ')],
}),
);
assert.deepEqual(splits.get('東紫乃'), { family: '東', given: '紫乃' });
});
test('resolveJapaneseNameSplits corrects a hint-length-misleading surname boundary', async () => {
const splits = await resolveJapaneseNameSplits(
[
characterRecord({
nativeName: '渡辺真奈美',
fullName: 'Manami Watanabe',
firstNameHint: 'Manami',
lastNameHint: 'Watanabe',
}),
],
tokenizerFor({
: [
personNameToken('渡辺', '姓', 'ワタナベ'),
personNameToken('真奈美', '名', 'マナミ'),
],
}),
);
assert.deepEqual(splits.get('渡辺真奈美'), { family: '渡辺', given: '真奈美' });
});
test('resolveJapaneseNameSplits falls back to hint readings when POS tags are generic', async () => {
const splits = await resolveJapaneseNameSplits(
[
characterRecord({
nativeName: '鈴木みゆ',
fullName: 'Miyu Suzuki',
firstNameHint: 'Miyu',
lastNameHint: 'Suzuki',
}),
],
tokenizerFor({
: [
personNameToken('鈴木', '姓', 'スズキ'),
{ word: 'みゆ', pos1: '名詞', pos2: '一般', pos3: '*', pos4: '*', katakanaReading: 'ミユ' },
],
}),
);
assert.deepEqual(splits.get('鈴木みゆ'), { family: '鈴木', given: 'みゆ' });
});
test('resolveJapaneseNameSplits skips names whose tokens do not reconstruct the name', async () => {
const splits = await resolveJapaneseNameSplits(
[characterRecord({})],
tokenizerFor({
: [personNameToken('東', '姓', 'アズマ'), personNameToken('乃', '名', '')],
}),
);
assert.equal(splits.size, 0);
});
test('resolveJapaneseNameSplits skips ambiguous or untagged segmentations', async () => {
const splits = await resolveJapaneseNameSplits(
[
characterRecord({
nativeName: '担任',
fullName: 'Tannin',
firstNameHint: 'Tannin',
lastNameHint: '',
}),
],
tokenizerFor({
: [
{ word: '担', pos1: '名詞', pos2: '一般', pos3: '*', pos4: '*', katakanaReading: 'タン' },
{ word: '任', pos1: '名詞', pos2: '一般', pos3: '*', pos4: '*', katakanaReading: 'ニン' },
],
}),
);
assert.equal(splits.size, 0);
});
test('resolveJapaneseNameSplits survives tokenizer failures', async () => {
const warnings: string[] = [];
const splits = await resolveJapaneseNameSplits(
[characterRecord({})],
async () => {
throw new Error('mecab unavailable');
},
(message) => warnings.push(message),
);
assert.equal(splits.size, 0);
assert.equal(warnings.length, 1);
assert.match(warnings[0]!, /mecab unavailable/);
});
test('splitJapaneseName prefers a resolved split over hint-length inference', () => {
const resolved = new Map([['東紫乃', { family: '東', given: '紫乃' }]]);
const withResolved = splitJapaneseName('東紫乃', 'Shino', 'Azuma', resolved);
assert.equal(withResolved.family, '東');
assert.equal(withResolved.given, '紫乃');
const withoutResolved = splitJapaneseName('東紫乃', 'Shino', 'Azuma');
assert.notEqual(withoutResolved.family, '東');
});
test('splitJapaneseNameCandidates emits the runner-up boundary only for inferred splits', () => {
const inferred = splitJapaneseNameCandidates('東紫乃', 'Shino', 'Azuma');
assert.equal(inferred.length, 2);
assert.deepEqual(
inferred.map((parts) => `${parts.family}|${parts.given}`).sort(),
['東紫|乃', '東|紫乃'].sort(),
);
const resolved = new Map([['東紫乃', { family: '東', given: '紫乃' }]]);
const trusted = splitJapaneseNameCandidates('東紫乃', 'Shino', 'Azuma', resolved);
assert.equal(trusted.length, 1);
assert.equal(trusted[0]!.family, '東');
const spaced = splitJapaneseNameCandidates('須々木 心一', 'Shinichi', 'Susuki');
assert.equal(spaced.length, 1);
});
test('buildNameTerms without resolved splits still emits both candidate surnames', () => {
const terms = buildNameTerms(characterRecord({}));
assert.ok(terms.includes('東'));
assert.ok(terms.includes('紫乃'));
assert.ok(terms.includes('東紫'));
assert.ok(terms.includes('乃'));
});
test('buildNameTerms emits surname and given-name terms from resolved splits', () => {
const resolved = new Map([['渡辺真奈美', { family: '渡辺', given: '真奈美' }]]);
const terms = buildNameTerms(
characterRecord({
nativeName: '渡辺真奈美',
fullName: 'Manami Watanabe',
firstNameHint: 'Manami',
lastNameHint: 'Watanabe',
}),
resolved,
);
assert.ok(terms.includes('渡辺'));
assert.ok(terms.includes('真奈美'));
assert.ok(terms.includes('渡辺真奈美'));
assert.ok(!terms.includes('渡辺真'));
});
@@ -0,0 +1,118 @@
import { buildReading, buildReadingFromHint } from './name-reading';
import { expandRawNameVariants, isJapaneseNameSplitCandidate } from './term-building';
import type {
CharacterRecord,
NameSplitToken,
NameSplitTokenizer,
ResolvedNameSplit,
} from './types';
const NAME_SEPARATOR_PATTERN = /[\s ・・·•]/;
function joinSurfaces(tokens: NameSplitToken[]): string {
return tokens.map((token) => token.word).join('');
}
// MeCab tags dictionary-known person names as 名詞,固有名詞,人名,姓|名. A split is
// trusted only when every leading token is 姓 and every remaining token is 名.
function splitIndexFromPersonNamePos(tokens: NameSplitToken[]): number | null {
let familyEnd = 0;
while (
familyEnd < tokens.length &&
tokens[familyEnd]!.pos3 === '人名' &&
tokens[familyEnd]!.pos4 === '姓'
) {
familyEnd += 1;
}
if (familyEnd === 0 || familyEnd >= tokens.length) {
return null;
}
for (let index = familyEnd; index < tokens.length; index += 1) {
if (tokens[index]!.pos3 !== '人名' || tokens[index]!.pos4 !== '名') {
return null;
}
}
return familyEnd;
}
function splitIndexFromHintReadings(
tokens: NameSplitToken[],
familyHintReading: string,
givenHintReading: string,
): number | null {
if (!familyHintReading && !givenHintReading) {
return null;
}
const readings = tokens.map((token) => buildReading(token.katakanaReading || ''));
let matchedIndex: number | null = null;
for (let index = 1; index < tokens.length; index += 1) {
const familyReadings = readings.slice(0, index);
const givenReadings = readings.slice(index);
const familyMatches =
!!familyHintReading &&
familyReadings.every((reading) => reading.length > 0) &&
familyReadings.join('') === familyHintReading;
const givenMatches =
!!givenHintReading &&
givenReadings.every((reading) => reading.length > 0) &&
givenReadings.join('') === givenHintReading;
if (!familyMatches && !givenMatches) {
continue;
}
if (matchedIndex !== null && matchedIndex !== index) {
return null;
}
matchedIndex = index;
}
return matchedIndex;
}
function collectSplitCandidateNames(character: CharacterRecord): string[] {
const candidates = new Set<string>();
const rawNames = [character.nativeName, character.fullName, ...character.alternativeNames];
for (const rawName of rawNames) {
for (const name of expandRawNameVariants(rawName)) {
const trimmed = name.trim();
if (!trimmed || NAME_SEPARATOR_PATTERN.test(trimmed)) continue;
if (!isJapaneseNameSplitCandidate(trimmed)) continue;
if ([...trimmed].length < 2) continue;
candidates.add(trimmed);
}
}
return [...candidates];
}
export async function resolveJapaneseNameSplits(
characters: CharacterRecord[],
tokenize: NameSplitTokenizer,
logWarn?: (message: string) => void,
): Promise<Map<string, ResolvedNameSplit>> {
const splits = new Map<string, ResolvedNameSplit>();
for (const character of characters) {
const familyHintReading = buildReadingFromHint(character.lastNameHint?.trim() || '');
const givenHintReading = buildReadingFromHint(character.firstNameHint?.trim() || '');
for (const name of collectSplitCandidateNames(character)) {
if (splits.has(name)) continue;
let tokens: NameSplitToken[] | null = null;
try {
tokens = await tokenize(name);
} catch (err) {
logWarn?.(
`[dictionary] name split tokenization failed for "${name}": ${(err as Error).message}`,
);
continue;
}
if (!tokens || tokens.length < 2 || joinSurfaces(tokens) !== name) continue;
const splitIndex =
splitIndexFromPersonNamePos(tokens) ??
splitIndexFromHintReadings(tokens, familyHintReading, givenHintReading);
if (splitIndex === null) continue;
const family = joinSurfaces(tokens.slice(0, splitIndex));
const given = joinSurfaces(tokens.slice(splitIndex));
if (family && given) {
splits.set(name, { family, given });
}
}
}
return splits;
}
@@ -121,6 +121,175 @@ test('generateForCurrentMedia refreshes same-version snapshots missing images wh
}
});
test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable', async () => {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), {
...createSnapshotWithoutImages(),
nameSplitSource: 'heuristic',
});
const originalFetch = globalThis.fetch;
let characterPageRequests = 0;
globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
if (url === GRAPHQL_URL) {
const body = JSON.parse(String(init?.body ?? '{}')) as { query?: string };
if (body.query?.includes('characters(page: $page')) {
characterPageRequests += 1;
return new Response(
JSON.stringify({
data: {
Media: {
title: { english: 'The Eminence in Shadow' },
characters: {
pageInfo: { hasNextPage: false },
edges: [
{
role: 'SUPPORTING',
node: {
id: 123,
description: 'Alexia Midgar.',
image: { large: null, medium: null },
name: {
first: 'Taro',
last: 'Yamada',
full: 'Taro Yamada',
native: '山田太郎',
},
},
},
],
},
},
},
}),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
}
}
throw new Error(`Unexpected fetch URL: ${url}`);
}) as typeof globalThis.fetch;
try {
let tokenizerCalls = 0;
const runtime = createCharacterDictionaryRuntimeService({
userDataPath,
getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv',
getCurrentMediaTitle: () => 'The Eminence in Shadow - S01E05',
resolveMediaPathForJimaku: (mediaPath) => mediaPath,
guessAnilistMediaInfo: async () => ({
title: 'The Eminence in Shadow',
season: null,
episode: 5,
source: 'fallback',
}),
getNameMatchImagesEnabled: () => false,
tokenizeJapaneseName: async () => {
tokenizerCalls += 1;
return null;
},
getJapaneseNameTokenizerAvailable: () => true,
now: () => 1_700_000_000_500,
});
const result = await runtime.generateForCurrentMedia();
const refreshedSnapshot = JSON.parse(
fs.readFileSync(getSnapshotPath(outputDir, 130298), 'utf8'),
) as CharacterDictionarySnapshot;
assert.equal(result.fromCache, false);
assert.equal(refreshedSnapshot.nameSplitSource, 'heuristic');
const retriedResult = await runtime.generateForCurrentMedia();
assert.equal(retriedResult.fromCache, false);
assert.equal(characterPageRequests, 2);
assert.equal(tokenizerCalls, 2);
} finally {
globalThis.fetch = originalFetch;
}
});
test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is available', async () => {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), {
...createSnapshotWithoutImages(),
nameSplitSource: 'mecab',
});
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (input: string | URL | Request) => {
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
throw new Error(`Unexpected fetch URL: ${url}`);
}) as typeof globalThis.fetch;
try {
const runtime = createCharacterDictionaryRuntimeService({
userDataPath,
getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv',
getCurrentMediaTitle: () => 'The Eminence in Shadow - S01E05',
resolveMediaPathForJimaku: (mediaPath) => mediaPath,
guessAnilistMediaInfo: async () => ({
title: 'The Eminence in Shadow',
season: null,
episode: 5,
source: 'fallback',
}),
getNameMatchImagesEnabled: () => false,
tokenizeJapaneseName: async () => null,
getJapaneseNameTokenizerAvailable: () => true,
now: () => 1_700_000_000_500,
});
const result = await runtime.generateForCurrentMedia();
assert.equal(result.fromCache, true);
} finally {
globalThis.fetch = originalFetch;
}
});
test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is unavailable', async () => {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), {
...createSnapshotWithoutImages(),
nameSplitSource: 'heuristic',
});
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (input: string | URL | Request) => {
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
throw new Error(`Unexpected fetch URL: ${url}`);
}) as typeof globalThis.fetch;
try {
const runtime = createCharacterDictionaryRuntimeService({
userDataPath,
getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv',
getCurrentMediaTitle: () => 'The Eminence in Shadow - S01E05',
resolveMediaPathForJimaku: (mediaPath) => mediaPath,
guessAnilistMediaInfo: async () => ({
title: 'The Eminence in Shadow',
season: null,
episode: 5,
source: 'fallback',
}),
getNameMatchImagesEnabled: () => false,
tokenizeJapaneseName: async () => null,
getJapaneseNameTokenizerAvailable: () => false,
now: () => 1_700_000_000_500,
});
const result = await runtime.generateForCurrentMedia();
assert.equal(result.fromCache, true);
} finally {
globalThis.fetch = originalFetch;
}
});
test('generateForCurrentMedia keeps same-version snapshots without images when inline images are disabled', async () => {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
@@ -14,6 +14,8 @@ import type {
CharacterDictionarySnapshotImage,
CharacterDictionaryTermEntry,
CharacterRecord,
NameSplitSource,
ResolvedNameSplits,
} from './types';
export function buildSnapshotImagePath(mediaId: number, charId: number, ext: string): string {
@@ -34,6 +36,8 @@ export function buildSnapshotFromCharacters(
getCollapsibleSectionOpenState: (
section: AnilistCharacterDictionaryCollapsibleSectionKey,
) => boolean,
resolvedNameSplits?: ResolvedNameSplits,
nameSplitSource: NameSplitSource = 'heuristic',
): CharacterDictionarySnapshot {
const termEntries: CharacterDictionaryTermEntry[] = [];
@@ -45,7 +49,7 @@ export function buildSnapshotFromCharacters(
const vaImg = imagesByVaId.get(va.id);
if (vaImg) vaImagePaths.set(va.id, vaImg.path);
}
const candidateTerms = buildNameTerms(character);
const candidateTerms = buildNameTerms(character, resolvedNameSplits);
const glossary = createDefinitionGlossary(
character,
mediaId,
@@ -59,12 +63,14 @@ export function buildSnapshotFromCharacters(
character.nativeName,
character.firstNameHint,
character.lastNameHint,
resolvedNameSplits,
);
const readings = generateNameReadings(
character.nativeName,
character.fullName,
character.firstNameHint,
character.lastNameHint,
resolvedNameSplits,
);
for (const term of candidateTerms) {
if (seenTerms.has(term)) continue;
@@ -84,6 +90,7 @@ export function buildSnapshotFromCharacters(
mediaTitle,
entryCount: termEntries.length,
updatedAt,
nameSplitSource,
termEntries,
images: [...imagesByCharacterId.values(), ...imagesByVaId.values()],
};
@@ -7,6 +7,7 @@ import {
hasKanaOnly,
isRomanizedName,
splitJapaneseName,
splitJapaneseNameCandidates,
} from './name-reading';
import type {
CharacterDictionaryGlossaryEntry,
@@ -15,9 +16,10 @@ import type {
CharacterRecord,
JapaneseNameParts,
NameReadings,
ResolvedNameSplits,
} from './types';
function expandRawNameVariants(rawName: string): string[] {
export function expandRawNameVariants(rawName: string): string[] {
const trimmed = rawName.trim();
if (!trimmed) return [];
@@ -40,26 +42,41 @@ function expandRawNameVariants(rawName: string): string[] {
return [...variants];
}
function isJapaneseNameSplitCandidate(name: string): boolean {
export function isJapaneseNameSplitCandidate(name: string): boolean {
const compact = name.replace(/[\s\u3000・・·•]/g, '');
return (
containsKanji(compact) && /^[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff々〆ヵヶー]+$/.test(compact)
);
}
function addJapaneseNameParts(character: CharacterRecord, name: string, terms: Set<string>): void {
function addJapaneseNameParts(
character: CharacterRecord,
name: string,
terms: Set<string>,
resolvedSplits?: ResolvedNameSplits,
): void {
if (!isJapaneseNameSplitCandidate(name)) return;
const nameParts = splitJapaneseName(name, character.firstNameHint, character.lastNameHint);
if (nameParts.family) {
terms.add(nameParts.family);
}
if (nameParts.given) {
terms.add(nameParts.given);
const candidates = splitJapaneseNameCandidates(
name,
character.firstNameHint,
character.lastNameHint,
resolvedSplits,
);
for (const nameParts of candidates) {
if (nameParts.family) {
terms.add(nameParts.family);
}
if (nameParts.given) {
terms.add(nameParts.given);
}
}
}
export function buildNameTerms(character: CharacterRecord): string[] {
export function buildNameTerms(
character: CharacterRecord,
resolvedSplits?: ResolvedNameSplits,
): string[] {
const base = new Set<string>();
const romanizedBase = new Set<string>();
const rawNames = [character.nativeName, character.fullName, ...character.alternativeNames];
@@ -95,7 +112,7 @@ export function buildNameTerms(character: CharacterRecord): string[] {
}
if (target === base) {
addJapaneseNameParts(character, name, base);
addJapaneseNameParts(character, name, base, resolvedSplits);
}
}
}
@@ -108,6 +125,7 @@ export function buildNameTerms(character: CharacterRecord): string[] {
character.nativeName,
character.firstNameHint,
character.lastNameHint,
resolvedSplits,
);
if (nativeParts.family) {
base.add(nativeParts.family);
@@ -31,6 +31,26 @@ export type JapaneseNameParts = {
given: string | null;
};
export type ResolvedNameSplit = {
family: string;
given: string;
};
export type ResolvedNameSplits = ReadonlyMap<string, ResolvedNameSplit>;
export type NameSplitToken = {
word: string;
pos1?: string;
pos2?: string;
pos3?: string;
pos4?: string;
katakanaReading?: string;
};
export type NameSplitTokenizer = (text: string) => Promise<NameSplitToken[] | null>;
export type NameSplitSource = 'mecab' | 'heuristic';
export type NameReadings = {
hasSpace: boolean;
original: string;
@@ -45,6 +65,7 @@ export type CharacterDictionarySnapshot = {
mediaTitle: string;
entryCount: number;
updatedAt: number;
nameSplitSource?: NameSplitSource;
termEntries: CharacterDictionaryTermEntry[];
images: CharacterDictionarySnapshotImage[];
};
@@ -152,6 +173,8 @@ export interface CharacterDictionaryRuntimeDeps {
getCollapsibleSectionOpenState?: (
section: AnilistCharacterDictionaryCollapsibleSectionKey,
) => boolean;
tokenizeJapaneseName?: NameSplitTokenizer;
getJapaneseNameTokenizerAvailable?: () => boolean;
}
export type ResolvedAniListMedia = {
+4 -2
View File
@@ -1,6 +1,7 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { appendClipboardVideoToQueueRuntime } from './clipboard-queue';
@@ -27,7 +28,8 @@ test('appendClipboardVideoToQueueRuntime rejects unsupported clipboard path', ()
});
test('appendClipboardVideoToQueueRuntime queues readable media file', () => {
const tempPath = path.join(process.cwd(), 'dist', 'clipboard-queue-test-video.mkv');
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'clipboard-queue-'));
const tempPath = path.join(tempDir, 'clipboard-queue-test-video.mkv');
fs.writeFileSync(tempPath, 'stub');
const commands: Array<(string | number)[]> = [];
@@ -43,5 +45,5 @@ test('appendClipboardVideoToQueueRuntime queues readable media file', () => {
assert.deepEqual(commands[0], ['loadfile', tempPath, 'append']);
assert.equal(osdMessages[0], `Queued from clipboard: ${path.basename(tempPath)}`);
fs.unlinkSync(tempPath);
fs.rmSync(tempDir, { recursive: true, force: true });
});
@@ -37,8 +37,8 @@ export function createBuildTokenizerDepsMainHandler(deps: TokenizerMainDeps) {
getYomitanParserInitPromise: () => deps.getYomitanParserInitPromise(),
setYomitanParserInitPromise: (promise: Promise<boolean> | null) =>
deps.setYomitanParserInitPromise(promise),
isKnownWord: (text: string, reading?: string) => {
const hit = deps.isKnownWord(text, reading);
isKnownWord: (text, reading, options) => {
const hit = deps.isKnownWord(text, reading, options);
deps.recordLookup(hit);
return hit;
},
+1
View File
@@ -3,6 +3,7 @@ import { normalizePos1ExclusionList } from './token-pos1-exclusions';
export const DEFAULT_ANNOTATION_POS2_EXCLUSION_DEFAULTS = Object.freeze([
'非自立',
'接尾',
]) as readonly string[];
export const DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG: ResolvedTokenPos2ExclusionConfig = {
+6
View File
@@ -40,6 +40,12 @@ export interface MergedToken {
isMerged: boolean;
isKnown: boolean;
isNPlusOneTarget: boolean;
/**
* Text Yomitan had no dictionary entry for (e.g. elongation runs,
* truncated inflections). Kept as a token so it stays hoverable, but
* ignored by annotation, N+1, and vocabulary-stats logic.
*/
isUnparsedRun?: boolean;
isNameMatch?: boolean;
characterImage?: CharacterNameImage;
jlptLevel?: JlptLevel;
Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.5 KiB

After

Width:  |  Height:  |  Size: 6.2 KiB