From ab5e50adac7233815e88d86235b0595ec9835491 Mon Sep 17 00:00:00 2001 From: sudacode Date: Tue, 22 Sep 2026 15:01:23 -0700 Subject: [PATCH] fix(dictionary): complete Hachidori imports and mining integration --- changes/hachidori-backend.md | 10 +- config.example.jsonc | 9 ++ docs-site/configuration.md | 2 + docs-site/immersion-tracking.md | 4 +- docs-site/public/config.example.jsonc | 9 ++ docs-site/usage.md | 4 +- launcher/config.ts | 5 +- launcher/main.test.ts | 51 +++++++++ package.json | 2 +- scripts/check-dictionary-backends.cjs | 21 +++- scripts/run-dictionary-settings-smoke.mjs | 26 +++++ scripts/smoke-package.cjs | 6 +- .../anki-connect-proxy.test.ts | 57 ++++++++++ src/anki-integration/anki-connect-proxy.ts | 15 ++- .../note-update-workflow.test.ts | 22 ++++ src/anki-integration/note-update-workflow.ts | 6 ++ src/config/definitions.ts | 2 + src/config/definitions/defaults-core.ts | 2 + src/config/definitions/options-core.ts | 7 ++ src/config/definitions/template-sections.ts | 8 ++ src/config/resolve/core-domains.ts | 16 +++ src/config/resolve/dictionary-backend.test.ts | 21 ++++ src/config/settings/registry.ts | 2 +- .../services/__tests__/stats-server.test.ts | 2 + .../services/stats-server/mining-routes.ts | 3 +- .../hachidori-dictionary-import.test.ts | 100 ++++++++++++++++++ .../tokenizer/hachidori-dictionary-import.ts | 59 +++++++++++ .../tokenizer/hachidori-parser-bridge.test.ts | 41 ++++++- .../tokenizer/hachidori-parser-bridge.ts | 15 ++- .../tokenizer/yomitan-parser-runtime.test.ts | 38 +++++++ .../tokenizer/yomitan-parser-runtime.ts | 44 +++++++- .../tokenizer/yomitan-scan-test-harness.ts | 9 ++ src/core/services/yomitan-extension-loader.ts | 5 + src/main.ts | 17 ++- .../character-dictionary-auto-sync.test.ts | 39 +++++++ .../runtime/character-dictionary-auto-sync.ts | 3 +- .../runtime/first-run-setup-service.test.ts | 30 ++++++ src/main/runtime/first-run-setup-service.ts | 7 +- src/main/runtime/stats-server-runtime.test.ts | 47 ++++++++ src/main/runtime/stats-server-runtime.ts | 7 +- src/shared/anki-source.ts | 2 + src/shared/hachidori-sharing.ts | 17 +++ src/types/config.ts | 2 + vendor/subminer-yomitan | 2 +- 44 files changed, 756 insertions(+), 40 deletions(-) create mode 100644 scripts/run-dictionary-settings-smoke.mjs create mode 100644 src/core/services/tokenizer/hachidori-dictionary-import.test.ts create mode 100644 src/core/services/tokenizer/hachidori-dictionary-import.ts create mode 100644 src/shared/anki-source.ts diff --git a/changes/hachidori-backend.md b/changes/hachidori-backend.md index 91343982..f0af50fe 100644 --- a/changes/hachidori-backend.md +++ b/changes/hachidori-backend.md @@ -2,13 +2,13 @@ type: added area: dictionary - Added a bundled Hachidori dictionary backend alongside the default Yomitan backend. Select it with `dictionaryBackend` and restart SubMiner. -- The tray and dictionary-settings shortcut follow the selected backend. `--hachidori` opens Hachidori settings, while `--yomitan` continues to open Yomitan settings. -- Hachidori integrates with subtitle scanning, popup controls, lookup tracking, character dictionaries, and Anki media enrichment, with separate dictionaries and settings for each backend. +- The tray and dictionary-settings shortcut follow the selected backend. `--hachidori` opens Hachidori settings, while `--yomitan` continues to open Yomitan settings. With Hachidori active, an inactive external Yomitan profile does not block the bundled Yomitan settings. +- Hachidori integrates with subtitle scanning, popup controls, lookup tracking, character dictionaries, and Anki media enrichment, with separate dictionaries and settings for each backend. Linked Docker hosts receive character dictionary uploads through `hachidori.externalHostManagementUrl`, retry busy imports, and replace the previous dictionary only after a successful import. - Hachidori auto-populates its first Anki template from SubMiner's deck, tags, and field mappings, detects an unambiguous matching note type, and preserves existing custom templates. Anki discovery retries after an unavailable connection. - Hachidori saves downloadable word audio before sending a note through SubMiner's Anki proxy, so freshly mined animated cards include the word-audio delay. -- Hachidori reloads its background code on startup so extension updates take effect while preserving installed dictionaries and settings. -- First-run setup remembers each backend that finished it, so switching back does not repeat setup, and the launcher gates playback on the backend the running app started with. Current incomplete or cancelled setup takes precedence over stale completion history. -- Stats dashboard mining and deck lookup use the selected backend. Settings labels for popup pause and the dictionary deck no longer name Yomitan, and the backend selector sits with the other dictionary settings. +- Both bundled dictionary backends reload their background code on startup so extension updates take effect while preserving installed dictionaries and settings. +- First-run setup remembers each backend that finished it, including when reopened for legacy plugin cleanup, so switching back does not repeat setup, and the launcher gates playback on the backend the running app started with. Current incomplete or cancelled setup takes precedence over stale completion history. A running Yomitan session keeps using its external profile until it restarts into Hachidori. +- Stats dashboard mining and deck lookup use the selected backend, with the Anki proxy enabled or disabled. Dashboard cards retain the selected history line and media instead of being processed again with the current mpv subtitle, with both Yomitan and Hachidori. Stats cards carry a `SubMiner::Stats` tag so polling also preserves their context. Hachidori word mining supplies dictionary aliases, IDs, and frequency metadata to its native Anki renderer. Settings labels for popup pause and the dictionary deck no longer name Yomitan, and the backend selector sits with the other dictionary settings. - Hachidori scans, dictionary counts, and settings reads wait for the dictionary engine to finish loading or importing instead of caching empty results. - First-run setup can link an external Hachidori dictionary host in an app, browser, or Docker container, verify its library, and unlink back to local dictionaries. The optional host controls are collapsed by default and explain which apps or containers must stay running. Unresponsive connection checks time out so setup remains usable. Anki mining, media enrichment, and custom toolbar buttons stay in SubMiner, including when editing buttons while linked. - Hachidori frequency highlighting reuses dictionary-entry ranks and fills missing ranks through its existing API. Entries without a matching definition may remain unranked. The bundled integration is maintained in a pinned fork submodule using upstream HoshiDicts and WASM binaries. diff --git a/config.example.jsonc b/config.example.jsonc index fd3ad54e..d1704a9b 100644 --- a/config.example.jsonc +++ b/config.example.jsonc @@ -13,6 +13,15 @@ // ========================================== "dictionaryBackend": "yomitan", // Dictionary lookup backend. Restart SubMiner after changing this setting. Values: yomitan | hachidori + // ========================================== + // Hachidori External Dictionary Imports + // Configure the linked Docker host management URL, for example http://127.0.0.1:8780. + // Used only while Hachidori is linked to an external host. + // ========================================== + "hachidori": { + "externalHostManagementUrl": "" // Docker host management URL for automatic character dictionary uploads and replacement. Empty disables external uploads. + }, // Configure the linked Docker host management URL, for example http://127.0.0.1:8780. + // ========================================== // Japanese Subtitle Generation // Generate timed Japanese subtitles from local audio using whisper.cpp. diff --git a/docs-site/configuration.md b/docs-site/configuration.md index 37ed57df..fc1aba2a 100644 --- a/docs-site/configuration.md +++ b/docs-site/configuration.md @@ -177,6 +177,8 @@ The configuration file includes several main sections: Each backend stores its own dictionaries and mining settings. `yomitan.externalProfilePath` applies only to Yomitan. See [Hachidori setup](./usage.md#hachidori-setup) before switching an existing installation. +`hachidori.externalHostManagementUrl` specifies the linked Docker host's HTTP(S) management origin for automatic character dictionary uploads and replacement. Use the management port, not the sharing or dictionary API port. See [Hachidori setup](./usage.md#hachidori-setup) for an example and [the generated configuration example](/config.example.jsonc) for the default. + ### Logging Control the minimum log level for runtime output: diff --git a/docs-site/immersion-tracking.md b/docs-site/immersion-tracking.md index de40a036..5c9a7d5a 100644 --- a/docs-site/immersion-tracking.md +++ b/docs-site/immersion-tracking.md @@ -152,12 +152,14 @@ Stats server config lives under `stats`: The Search tab and the Vocabulary tab's word detail panel both mine from subtitle lines in your viewing history. Search matches sentence text and media titles, and **Search by headword** is enabled by default so dictionary-form searches such as `知らない` can find tracked subtitle lines with inflected variants. Turn that toggle off for exact text/title matching only. Each line with a valid source file offers sentence-card mining; word/audio mining is available when the selected word or searched word appears in the sentence: -- **Mine Word** - performs a full Yomitan dictionary lookup for the word (definition, reading, pitch accent, etc.) via a short-lived hidden helper, then enriches the card with sentence audio, a screenshot or animated AVIF clip, the highlighted sentence, and metadata extracted from the source video file. Requires Anki and Yomitan dictionaries to be loaded. +- **Mine Word** - looks up the word with the selected dictionary backend, Yomitan or Hachidori, then enriches the card with sentence audio, a screenshot or animated AVIF clip, the highlighted sentence, and metadata extracted from the source video file. The selected history line supplies the card's context even while another subtitle is playing in mpv. Hachidori uses its configured Anki template, dictionary aliases, and frequency metadata. Requires Anki and the selected backend's dictionaries to be loaded. - **Mine Sentence** - creates a sentence card directly with the `IsSentenceCard` flag set (for Lapis/Kiku workflows), along with audio and image from the source video. - **Mine Audio** - creates an audio-only card with the `IsAudioCard` flag, attaching only the sentence audio clip. All three modes respect your `ankiConnect` config: deck, model, field mappings, media settings (static vs AVIF, quality, dimensions), audio padding, metadata pattern, and tags. Media generation runs in parallel for faster card creation. +Stats cards also receive the `SubMiner::Stats` tag. SubMiner uses it to preserve their selected history context when detecting new cards through polling with the Anki proxy disabled. + Secondary subtitle text is stored alongside primary subtitles during playback, but the Search tab does not use it for display or matching. ### Word exclusion list diff --git a/docs-site/public/config.example.jsonc b/docs-site/public/config.example.jsonc index fd3ad54e..d1704a9b 100644 --- a/docs-site/public/config.example.jsonc +++ b/docs-site/public/config.example.jsonc @@ -13,6 +13,15 @@ // ========================================== "dictionaryBackend": "yomitan", // Dictionary lookup backend. Restart SubMiner after changing this setting. Values: yomitan | hachidori + // ========================================== + // Hachidori External Dictionary Imports + // Configure the linked Docker host management URL, for example http://127.0.0.1:8780. + // Used only while Hachidori is linked to an external host. + // ========================================== + "hachidori": { + "externalHostManagementUrl": "" // Docker host management URL for automatic character dictionary uploads and replacement. Empty disables external uploads. + }, // Configure the linked Docker host management URL, for example http://127.0.0.1:8780. + // ========================================== // Japanese Subtitle Generation // Generate timed Japanese subtitles from local audio using whisper.cpp. diff --git a/docs-site/usage.md b/docs-site/usage.md index 88850891..fb6f1b3e 100644 --- a/docs-site/usage.md +++ b/docs-site/usage.md @@ -329,9 +329,11 @@ Both named settings flags work independently of the selected backend. Opening se Hachidori uses SubMiner's subtitle scanning, lookup counter, popup pause behavior, controller commands, character dictionaries, and Anki media enrichment. Keep SubMiner's AnkiConnect proxy enabled for screenshots and sentence audio. SubMiner routes Hachidori to that proxy when it is active; Hachidori's own screen recorder and screenshot capture are disabled in the embedded app. +For automatic character dictionary sync with a Docker host, set `hachidori.externalHostManagementUrl` to that same host's management origin, for example `"http://127.0.0.1:8780"`. This is separate from the WebSocket sharing address. SubMiner uploads the generated ZIP directly and replaces its previous dictionary after a successful import; busy imports are retried. Keep this URL pointed at the currently linked Docker host if you change hosts. An empty value disables external uploads and reports a configuration error when sync is attempted. Local Hachidori dictionaries do not need this setting. External browser/app hosts without the Docker management API do not support this automatic upload path. + Existing controls such as `startupWarmups.yomitanExtension` and `subtitleStyle.autoPauseVideoOnYomitanPopup` apply to the selected backend. Hachidori has one dictionary configuration, so character-dictionary profile scope applies to that configuration. -First-run setup remembers each backend that finished it. Switching to a backend for the first time asks for that backend's dictionaries; switching back to one that already finished does not repeat setup. Until SubMiner restarts, it keeps running the backend it started with, and the launcher gates playback on that running backend and logs a restart reminder. +First-run setup remembers each backend that finished it, including when setup is reopened for legacy plugin cleanup. Switching to a backend for the first time asks for that backend's dictionaries; switching back to one that already finished does not repeat setup. Until SubMiner restarts, it keeps running the backend it started with, and the launcher gates playback on that running backend and logs a restart reminder. A running Yomitan session continues using its external profile until the restart. While Hachidori is active, `--yomitan` opens the bundled Yomitan settings even if an external Yomitan profile is configured. Hachidori's own duplicate handling differs from Yomitan's. Choosing **Overwrite** in the Hachidori popup updates the existing note and SubMiner enriches its media, while **Add anyway** creates a new note and runs SubMiner's Kiku/Senren [field grouping](./anki-integration.md#field-grouping-kiku-senren). Mining from the stats dashboard uses the selected backend as well. diff --git a/launcher/config.ts b/launcher/config.ts index 96f0e867..f6556447 100644 --- a/launcher/config.ts +++ b/launcher/config.ts @@ -108,10 +108,7 @@ export function loadLauncherDictionaryBackend(): DictionaryBackend { } export function hasLauncherExternalYomitanProfileConfig(): boolean { - const config = readLauncherMainConfigObject(); - return ( - config?.dictionaryBackend !== 'hachidori' && readExternalYomitanProfilePath(config) !== null - ); + return readExternalYomitanProfilePath(readLauncherMainConfigObject()) !== null; } export function readPluginRuntimeConfig(logLevel: LogLevel): PluginRuntimeConfig { diff --git a/launcher/main.test.ts b/launcher/main.test.ts index 1c9805db..c635d073 100644 --- a/launcher/main.test.ts +++ b/launcher/main.test.ts @@ -1112,3 +1112,54 @@ test('classifyJellyfinChildSelection keeps container drilldown state instead of id: 'season-2', }); }); + +test('external Yomitan profile remains available while a running app awaits a backend switch', () => { + withTempDir((dir) => { + const env = makeTestEnv(dir, path.join(dir, 'config')); + const configPath = resolveConfigFilePath({ + appDataDir: env.APPDATA, + xdgConfigHome: env.XDG_CONFIG_HOME, + homeDir: dir, + existsSync: () => false, + }); + fs.mkdirSync(path.dirname(configPath), { recursive: true }); + fs.writeFileSync( + configPath, + JSON.stringify({ + dictionaryBackend: 'hachidori', + yomitan: { externalProfilePath: '/external/yomitan-profile' }, + }), + ); + const result = spawnSync( + process.execPath, + [ + '--eval', + ` + import assert from 'node:assert/strict'; + import { hasLauncherExternalYomitanProfileConfig } from './launcher/config.ts'; + import { ensureLauncherSetupReady } from './launcher/setup-gate.ts'; + import { createDefaultSetupState } from './src/shared/setup-state.ts'; + for (const running of [true, false]) { + let launches = 0; + let tick = 0; + const ready = await ensureLauncherSetupReady({ + dictionaryBackend: 'hachidori', + isAppRunning: async () => running, + readSetupState: () => ({ ...createDefaultSetupState(), dictionaryBackend: 'yomitan' }), + isExternalYomitanConfigured: hasLauncherExternalYomitanProfileConfig, + launchSetupApp: () => { launches += 1; }, + sleep: async () => {}, + now: () => tick++, + timeoutMs: 2, + pollIntervalMs: 1, + }); + assert.equal(ready, running); + assert.equal(launches, running ? 0 : 1); + } + `, + ], + { cwd: process.cwd(), env, encoding: 'utf8', timeout: LAUNCHER_RUN_TIMEOUT_MS }, + ); + assert.equal(result.status, 0, result.stderr); + }); +}); diff --git a/package.json b/package.json index 7b35782e..a63dbbb9 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "get-frequency": "bun run scripts/get_frequency.ts --pretty --color-top-x 10000 --yomitan-user-data ~/.config/SubMiner --colorized-line", "get-frequency:electron": "bun run build:yomitan && bun build scripts/get_frequency.ts --format=cjs --target=node --outfile dist/scripts/get_frequency.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/get_frequency.js --pretty --color-top-x 10000 --yomitan-user-data ~/.config/SubMiner --colorized-line", "test-yomitan-parser": "bun run scripts/test-yomitan-parser.ts", - "test:dictionary:electron": "env -u ELECTRON_RUN_AS_NODE electron --no-sandbox scripts/check-dictionary-backends.cjs && env -u ELECTRON_RUN_AS_NODE electron --no-sandbox scripts/check-dictionary-backends.cjs --backend=yomitan && env -u ELECTRON_RUN_AS_NODE electron --no-sandbox --ozone-platform=x11 scripts/check-hachidori-parser.cjs", + "test:dictionary:electron": "bun scripts/run-dictionary-settings-smoke.mjs && bun scripts/run-dictionary-settings-smoke.mjs --backend=yomitan && env -u ELECTRON_RUN_AS_NODE electron --no-sandbox --ozone-platform=x11 scripts/check-hachidori-parser.cjs", "test-yomitan-parser:electron": "bun run build:yomitan && bun build scripts/test-yomitan-parser.ts --format=cjs --target=node --outfile dist/scripts/test-yomitan-parser.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/test-yomitan-parser.js", "verify-known-word-highlights:electron": "bun run build:yomitan && bun build scripts/verify-known-word-highlights.ts --format=cjs --target=node --outfile dist/scripts/verify-known-word-highlights.js --packages=external && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/verify-known-word-highlights.js", "record-tokenizer-fixture:electron": "bun run build:yomitan && bun build scripts/record-tokenizer-fixture.ts --format=cjs --target=node --outfile dist/scripts/record-tokenizer-fixture.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/record-tokenizer-fixture.js", diff --git a/scripts/check-dictionary-backends.cjs b/scripts/check-dictionary-backends.cjs index f5d43862..65465d9b 100644 --- a/scripts/check-dictionary-backends.cjs +++ b/scripts/check-dictionary-backends.cjs @@ -7,7 +7,11 @@ if (process.platform !== 'linux') throw new Error('This app-entry smoke requires Linux XDG isolation.'); const root = process.cwd(); const backend = process.argv.includes('--backend=yomitan') ? 'yomitan' : 'hachidori'; -const profile = fs.mkdtempSync('/tmp/subminer-hachi-settings-'); +const profile = process.env.SUBMINER_DICTIONARY_SMOKE_DATA; +assert( + profile && fs.existsSync(profile), + 'Use bun run test:dictionary:electron for profile isolation', +); process.env.XDG_CONFIG_HOME = profile; process.env.XDG_DATA_HOME = path.join(profile, 'data'); fs.mkdirSync(path.join(profile, 'SubMiner')); @@ -15,6 +19,10 @@ fs.writeFileSync( path.join(profile, 'SubMiner', 'config.json'), JSON.stringify({ dictionaryBackend: backend, + // An inactive external Yomitan profile must not block bundled settings in Hachidori mode. + yomitan: { + externalProfilePath: backend === 'hachidori' ? path.join(profile, 'external-yomitan') : '', + }, mpv: { socketPath: path.join(profile, 'missing-mpv.sock') }, ankiConnect: { enabled: false }, startupWarmups: { lowPowerMode: true }, @@ -39,8 +47,12 @@ const deadline = setTimeout(() => { 'FAIL timeout', BrowserWindow.getAllWindows().map((w) => w.webContents.getURL()), ); - app.exit(1); + finish(1); }, 60000); +function finish(exitCode) { + clearTimeout(deadline); + app.exit(exitCode); +} (async () => { await app.whenReady(); let window; @@ -102,9 +114,8 @@ const deadline = setTimeout(() => { `PASS ${backend} overlay session, independent settings windows, real preload external link bridge`, ); - clearTimeout(deadline); - app.exit(0); + finish(0); })().catch((error) => { console.error(error); - app.exit(1); + finish(1); }); diff --git a/scripts/run-dictionary-settings-smoke.mjs b/scripts/run-dictionary-settings-smoke.mjs new file mode 100644 index 00000000..c2f84db7 --- /dev/null +++ b/scripts/run-dictionary-settings-smoke.mjs @@ -0,0 +1,26 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; + +const require = createRequire(import.meta.url); +const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-hachi-settings-')); +const env = { ...process.env, SUBMINER_DICTIONARY_SMOKE_DATA: profile }; +delete env.ELECTRON_RUN_AS_NODE; +try { + const result = spawnSync( + require('electron'), + [ + '--no-sandbox', + fileURLToPath(new URL('./check-dictionary-backends.cjs', import.meta.url)), + ...process.argv.slice(2), + ], + { env, stdio: 'inherit', timeout: 75_000 }, + ); + if (result.error) throw result.error; + process.exitCode = result.status ?? 1; +} finally { + fs.rmSync(profile, { recursive: true, force: true, maxRetries: 3 }); +} diff --git a/scripts/smoke-package.cjs b/scripts/smoke-package.cjs index 4fd0476d..1bcfacee 100644 --- a/scripts/smoke-package.cjs +++ b/scripts/smoke-package.cjs @@ -99,7 +99,11 @@ async function smoke() { if (details.statusCode >= 400) failedRequests.push(`${details.url}: ${details.statusCode}`); }); await statsWindow.loadURL(url); - assert.equal((await fetch(`${url}/api/stats/overview`)).status, 200); + for (const endpoint of ['overview', 'sessions']) { + const response = await fetch(`${url}/api/stats/${endpoint}`); + assert.equal(response.status, 200, `Stats ${endpoint} request failed`); + await response.json(); + } } finally { statsWindow.destroy(); await statsServer.close(); diff --git a/src/anki-integration/anki-connect-proxy.test.ts b/src/anki-integration/anki-connect-proxy.test.ts index 6721c884..8f811fae 100644 --- a/src/anki-integration/anki-connect-proxy.test.ts +++ b/src/anki-integration/anki-connect-proxy.test.ts @@ -505,6 +505,63 @@ test('proxy enriches confirmed Hachidori overwrites without counting a new card } }); +test('stats-owned notes bypass overlay enrichment while popup notes still enqueue', async () => { + const processed: number[] = []; + const added: number[] = []; + const received: unknown[] = []; + let noteId = 70; + const upstream = http.createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + received.push(JSON.parse(Buffer.concat(chunks).toString())); + res.end(JSON.stringify({ result: ++noteId, error: null })); + }); + upstream.listen(0, '127.0.0.1'); + await once(upstream, 'listening'); + const address = upstream.address(); + assert.ok(address && typeof address === 'object'); + const proxy = new AnkiConnectProxyServer({ + shouldAutoUpdateNewCards: () => true, + processNewCard: async (id) => { + processed.push(id); + }, + recordCardsAdded: (_count, ids) => { + added.push(...ids); + }, + logInfo: () => {}, + logWarn: () => {}, + logError: () => {}, + }); + try { + proxy.start({ host: '127.0.0.1', port: 0, upstreamUrl: `http://127.0.0.1:${address.port}` }); + await proxy.waitUntilReady(); + const server: unknown = Reflect.get(proxy, 'server'); + assert.ok(server instanceof http.Server); + const bound = server.address(); + assert.ok(bound && typeof bound === 'object'); + for (const metadata of [{ subminerEnrich: false }, {}]) { + const response: Response = await fetch(`http://127.0.0.1:${bound.port}`, { + method: 'POST', + body: JSON.stringify({ + action: 'addNote', + version: 6, + params: { note: { fields: { Expression: '猫' } }, ...metadata }, + }), + }); + assert.equal(response.status, 200); + await response.json(); + } + await waitForCondition(() => processed.includes(72)); + assert.deepEqual(processed, [72]); + assert.deepEqual(added, [71, 72]); + assert.equal(JSON.stringify(received).includes('subminerEnrich'), false); + } finally { + proxy.stop(); + upstream.close(); + await once(upstream, 'close'); + } +}); + test('proxy returns addNote response without waiting for background enrichment', async () => { const processed: number[] = []; let releaseProcessing: (() => void) | undefined; diff --git a/src/anki-integration/anki-connect-proxy.ts b/src/anki-integration/anki-connect-proxy.ts index 60b02bee..91c25c1d 100644 --- a/src/anki-integration/anki-connect-proxy.ts +++ b/src/anki-integration/anki-connect-proxy.ts @@ -264,12 +264,23 @@ export class AnkiConnectProxyServer { return; } - this.maybeTrackDuplicateNoteIds(requestJson, action, responseResult); - const noteIds = action === 'multi' ? this.collectMultiResultIds(requestJson, responseResult) : this.collectNoteIdsForAction(action, responseResult); + const params = requestJson.params; + if ( + action === 'addNote' && + params && + typeof params === 'object' && + 'subminerEnrich' in params && + params.subminerEnrich === false + ) { + // Stats owns the saved sentence and media; the live mpv context is unrelated. + if (noteIds.length > 0) this.deps.recordCardsAdded?.(noteIds.length, noteIds); + return; + } + this.maybeTrackDuplicateNoteIds(requestJson, action, responseResult); if (noteIds.length === 0 && shouldFallbackToLatestAdded) { void this.enqueueMostRecentAddedNote(); return; diff --git a/src/anki-integration/note-update-workflow.test.ts b/src/anki-integration/note-update-workflow.test.ts index 99019886..846d37c0 100644 --- a/src/anki-integration/note-update-workflow.test.ts +++ b/src/anki-integration/note-update-workflow.test.ts @@ -8,6 +8,7 @@ import { import type { SubtitleMiningContext } from '../types/subtitle'; import type { CardKind } from '../types/anki'; import { applyCardKindFlagFields } from './card-kinds'; +import { STATS_MINING_TAG } from '../shared/anki-source'; function setCardTypeFields( updatedFields: Record, @@ -110,6 +111,27 @@ function createWorkflowHarness() { }; } +test('NoteUpdateWorkflow preserves stats cards discovered by polling', async () => { + const { workflow, deps, updates } = createWorkflowHarness(); + const note = { + noteId: 42, + tags: [STATS_MINING_TAG], + fields: { Expression: { value: '猫' }, Sentence: { value: '猫がいる。' } }, + }; + deps.client.notesInfo = async () => [note]; + deps.captureSubtitleMediaContext = () => assert.fail('Must not capture current playback'); + deps.findDuplicateNote = async () => assert.fail('Must not regroup a stats card'); + let cachedNote: NoteUpdateWorkflowNoteInfo | undefined; + deps.appendKnownWordsFromNoteInfo = (value) => { + cachedNote = value; + }; + + await workflow.execute(42); + + assert.deepEqual(updates, []); + assert.equal(cachedNote, note); +}); + test('NoteUpdateWorkflow updates sentence field and emits notification', async () => { const harness = createWorkflowHarness(); diff --git a/src/anki-integration/note-update-workflow.ts b/src/anki-integration/note-update-workflow.ts index 7d1468b1..aeb2d587 100644 --- a/src/anki-integration/note-update-workflow.ts +++ b/src/anki-integration/note-update-workflow.ts @@ -8,9 +8,11 @@ import type { WordCardKind, } from '../types/anki'; import { resolveWordCardKind } from './note-field-utils'; +import { STATS_MINING_TAG } from '../shared/anki-source'; export interface NoteUpdateWorkflowNoteInfo { noteId: number; + tags?: string[]; fields: Record; } @@ -181,6 +183,10 @@ export class NoteUpdateWorkflow { } const noteInfo = notesInfo[0]!; + if (noteInfo.tags?.includes(STATS_MINING_TAG)) { + this.deps.appendKnownWordsFromNoteInfo(noteInfo); + return; + } const fields = this.deps.extractFields(noteInfo.fields); const config = this.deps.getConfig(); diff --git a/src/config/definitions.ts b/src/config/definitions.ts index c76c962f..ea68de54 100644 --- a/src/config/definitions.ts +++ b/src/config/definitions.ts @@ -23,6 +23,7 @@ export type { const { dictionaryBackend, + hachidori, subtitlePosition, keybindings, websocket, @@ -59,6 +60,7 @@ const { stats } = STATS_DEFAULT_CONFIG; export const DEFAULT_CONFIG: ResolvedConfig = { subtitleGeneration: { ...DEFAULT_SUBTITLE_GENERATION_CONFIG }, dictionaryBackend, + hachidori, subtitlePosition, keybindings, websocket, diff --git a/src/config/definitions/defaults-core.ts b/src/config/definitions/defaults-core.ts index 15229e37..23edc3b4 100644 --- a/src/config/definitions/defaults-core.ts +++ b/src/config/definitions/defaults-core.ts @@ -3,6 +3,7 @@ import { ResolvedConfig } from '../../types/config'; export const CORE_DEFAULT_CONFIG: Pick< ResolvedConfig, | 'dictionaryBackend' + | 'hachidori' | 'subtitlePosition' | 'keybindings' | 'websocket' @@ -20,6 +21,7 @@ export const CORE_DEFAULT_CONFIG: Pick< | 'auto_start_overlay' > = { dictionaryBackend: 'yomitan', + hachidori: { externalHostManagementUrl: '' }, subtitlePosition: { yPercent: 10 }, keybindings: [], websocket: { diff --git a/src/config/definitions/options-core.ts b/src/config/definitions/options-core.ts index 5d091676..48b300ff 100644 --- a/src/config/definitions/options-core.ts +++ b/src/config/definitions/options-core.ts @@ -81,6 +81,13 @@ export function buildCoreConfigOptionRegistry( ] as const; return [ + { + path: 'hachidori.externalHostManagementUrl', + kind: 'string', + defaultValue: defaultConfig.hachidori.externalHostManagementUrl, + description: + 'Docker host management URL for automatic character dictionary uploads and replacement. Empty disables external uploads.', + }, { path: 'dictionaryBackend', kind: 'enum', diff --git a/src/config/definitions/template-sections.ts b/src/config/definitions/template-sections.ts index c9322a11..60956d79 100644 --- a/src/config/definitions/template-sections.ts +++ b/src/config/definitions/template-sections.ts @@ -9,6 +9,14 @@ const CORE_TEMPLATE_SECTIONS: ConfigTemplateSection[] = [ ], key: 'dictionaryBackend', }, + { + title: 'Hachidori External Dictionary Imports', + description: [ + 'Configure the linked Docker host management URL, for example http://127.0.0.1:8780.', + ], + notes: ['Used only while Hachidori is linked to an external host.'], + key: 'hachidori', + }, { title: 'Japanese Subtitle Generation', description: [ diff --git a/src/config/resolve/core-domains.ts b/src/config/resolve/core-domains.ts index 51567497..95806b62 100644 --- a/src/config/resolve/core-domains.ts +++ b/src/config/resolve/core-domains.ts @@ -2,10 +2,26 @@ import { ResolveContext } from './context'; import { applyControllerConfig } from './controller'; import { isNotificationType, isOverlayNotificationPosition } from '../../types/notification'; import { asBoolean, asNumber, asString, isObject } from './shared'; +import { parseHachidoriManagementUrl } from '../../shared/hachidori-sharing'; export function applyCoreDomainConfig(context: ResolveContext): void { const { src, resolved, warn } = context; + if (isObject(src.hachidori) && src.hachidori.externalHostManagementUrl !== undefined) { + try { + resolved.hachidori.externalHostManagementUrl = parseHachidoriManagementUrl( + src.hachidori.externalHostManagementUrl, + ); + } catch { + warn( + 'hachidori.externalHostManagementUrl', + src.hachidori.externalHostManagementUrl, + resolved.hachidori.externalHostManagementUrl, + 'Expected an HTTP(S) origin or an empty string.', + ); + } + } + if (src.dictionaryBackend === 'yomitan' || src.dictionaryBackend === 'hachidori') { resolved.dictionaryBackend = src.dictionaryBackend; } else if (src.dictionaryBackend !== undefined) { diff --git a/src/config/resolve/dictionary-backend.test.ts b/src/config/resolve/dictionary-backend.test.ts index b23307dc..312198a7 100644 --- a/src/config/resolve/dictionary-backend.test.ts +++ b/src/config/resolve/dictionary-backend.test.ts @@ -28,3 +28,24 @@ test('unknown dictionary backend values warn and preserve the default', () => { assert.equal(warnings[0]?.path, 'dictionaryBackend'); } }); + +test('Hachidori external import URL accepts HTTP origins and rejects invalid targets', () => { + assert.equal(resolveConfig({}).resolved.hachidori.externalHostManagementUrl, ''); + const result = resolveConfig({ + hachidori: { externalHostManagementUrl: 'http://127.0.0.1:8780/' }, + }); + assert.equal(result.resolved.hachidori.externalHostManagementUrl, 'http://127.0.0.1:8780'); + assert.deepEqual(result.warnings, []); + for (const value of [ + 'file:///tmp/dict', + 'http://host/import', + 'http://user:password@host', + true, + ]) { + const { context, warnings } = createResolveContext({}); + context.src.hachidori = { externalHostManagementUrl: value }; + applyCoreDomainConfig(context); + assert.equal(context.resolved.hachidori.externalHostManagementUrl, ''); + assert.equal(warnings[0]?.path, 'hachidori.externalHostManagementUrl'); + } +}); diff --git a/src/config/settings/registry.ts b/src/config/settings/registry.ts index ea5d8f7a..bd6ede71 100644 --- a/src/config/settings/registry.ts +++ b/src/config/settings/registry.ts @@ -341,7 +341,7 @@ function humanizePath(path: string): string { } function categoryAndSection(path: string): { category: ConfigSettingsCategory; section: string } { - if (path === 'dictionaryBackend') { + if (path === 'dictionaryBackend' || path.startsWith('hachidori.')) { return { category: 'integrations', section: 'Dictionary Lookup' }; } if ( diff --git a/src/core/services/__tests__/stats-server.test.ts b/src/core/services/__tests__/stats-server.test.ts index eefc9aff..9ba104e3 100644 --- a/src/core/services/__tests__/stats-server.test.ts +++ b/src/core/services/__tests__/stats-server.test.ts @@ -2525,6 +2525,8 @@ Aligned English subtitle await new Promise((resolve) => setTimeout(resolve, 1)); } const addedBeforeMediaFinished = requests.some((request) => request.action === 'addNote'); + const addRequest = requests.find((request) => request.action === 'addNote'); + assert.deepEqual(addRequest?.params?.note?.tags, ['SubMiner', 'SubMiner::Stats']); mediaRelease.audio?.(); mediaRelease.image?.(); diff --git a/src/core/services/stats-server/mining-routes.ts b/src/core/services/stats-server/mining-routes.ts index a2e2d7ba..68043a05 100644 --- a/src/core/services/stats-server/mining-routes.ts +++ b/src/core/services/stats-server/mining-routes.ts @@ -7,6 +7,7 @@ import { resolveAnimatedImageLeadInSeconds } from '../../../anki-integration/ani import { clampMediaEndTime } from '../../../anki-integration/media-duration.js'; import { MediaGenerator } from '../../../media-generator.js'; import { statsJson } from '../../../types/stats-http-contract.js'; +import { STATS_MINING_TAG } from '../../../shared/anki-source.js'; import { resolveRetimedSecondarySubtitleTextFromSidecar, resolveSecondarySubtitleTextFromSidecar, @@ -359,7 +360,7 @@ export function registerStatsMiningRoutes(app: Hono, options?: StatsMiningRouteO } const model = ankiConfig.isLapis?.sentenceCardModel || 'Basic'; - const tags = ankiConfig.tags ?? ['SubMiner']; + const tags = [...new Set([...(ankiConfig.tags ?? ['SubMiner']), STATS_MINING_TAG])]; const addNotePromise = timeMiningPhase( mode, diff --git a/src/core/services/tokenizer/hachidori-dictionary-import.test.ts b/src/core/services/tokenizer/hachidori-dictionary-import.test.ts new file mode 100644 index 00000000..3f932051 --- /dev/null +++ b/src/core/services/tokenizer/hachidori-dictionary-import.test.ts @@ -0,0 +1,100 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { uploadHachidoriDictionary } from './hachidori-dictionary-import'; +import { importYomitanDictionaryFromZip } from './yomitan-parser-runtime'; +import { createDeps } from './yomitan-scan-test-harness'; + +test('linked Hachidori uploads replacement bytes, retries a busy host, and never invokes local import', async () => { + const zipPath = path.join(await mkdtemp(path.join(os.tmpdir(), 'hachi-import-')), 'merged.zip'); + const archive = Buffer.from('PK-test-archive'); + await writeFile(zipPath, archive); + let attempts = 0; + const server = createServer(async (request, response) => { + attempts += 1; + assert.equal(request.url, '/import?name=merged.zip&replace=true'); + assert.equal(request.method, 'POST'); + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + assert.deepEqual(Buffer.concat(chunks), archive); + response.writeHead(attempts === 1 ? 409 : 200, { 'Content-Type': 'application/json' }); + response.end( + JSON.stringify(attempts === 1 ? { error: 'busy' } : { ok: true, report: { success: true } }), + ); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const address = server.address(); + assert.ok(address && typeof address === 'object'); + const managementUrl = `http://127.0.0.1:${address.port}`; + let connected = true; + const deps = { + ...createDeps(async (script) => { + assert.ok(script.includes('hd_sharing_status'), 'must not invoke local ZIP automation'); + return { + ok: true, + sharing: { + client: { + linked: true, + connected, + address: 'ws://127.0.0.1:8771/link', + host: { dictionaryCount: 8 }, + }, + }, + }; + }), + getYomitanExt: () => ({ + id: 'hachi', + name: 'Hachidori', + version: '1', + path: '', + url: '', + manifest: {}, + }), + }; + try { + assert.equal( + await importYomitanDictionaryFromZip(zipPath, deps, { error: assert.fail }, managementUrl), + true, + ); + assert.equal(attempts, 2); + const errors: string[] = []; + const logger = { error: (...args: unknown[]) => errors.push(args.join(' ')) }; + assert.equal(await importYomitanDictionaryFromZip(zipPath, deps, logger), false); + assert.match(errors.pop() ?? '', /externalHostManagementUrl/); + connected = false; + assert.equal(await importYomitanDictionaryFromZip(zipPath, deps, logger, managementUrl), false); + assert.equal(attempts, 2); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); + +test('Hachidori upload requires a successful import report, not just HTTP success', async () => { + const zipPath = path.join(await mkdtemp(path.join(os.tmpdir(), 'hachi-import-')), 'merged.zip'); + await writeFile(zipPath, 'bad archive'); + const server = createServer((_request, response) => { + response.writeHead(200, { 'Content-Type': 'application/json' }); + response.end( + JSON.stringify({ ok: true, report: { success: false, error: 'Invalid archive' } }), + ); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const address = server.address(); + assert.ok(address && typeof address === 'object'); + try { + await assert.rejects( + uploadHachidoriDictionary(zipPath, `http://127.0.0.1:${address.port}`), + /Invalid archive/, + ); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/src/core/services/tokenizer/hachidori-dictionary-import.ts b/src/core/services/tokenizer/hachidori-dictionary-import.ts new file mode 100644 index 00000000..11285af4 --- /dev/null +++ b/src/core/services/tokenizer/hachidori-dictionary-import.ts @@ -0,0 +1,59 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { parseHachidoriManagementUrl } from '../../../shared/hachidori-sharing'; + +// Upload from the main process: extension blob URLs cannot cross the sharing link, +// and the Docker management API deliberately rejects browser cross-origin writes. +export async function uploadHachidoriDictionary( + zipPath: string, + managementUrl: string, +): Promise { + const origin = parseHachidoriManagementUrl(managementUrl); + if (!origin) { + throw new Error( + 'Set hachidori.externalHostManagementUrl to the linked Docker host management URL to sync character dictionaries.', + ); + } + const url = new URL('/import', origin); + url.searchParams.set('name', path.basename(zipPath)); + url.searchParams.set('replace', 'true'); + const bytes = await readFile(zipPath); + const signal = AbortSignal.timeout(300_000); + for (;;) { + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/zip' }, + body: bytes, + signal, + redirect: 'error', + }); + if (response.status === 409) { + await response.body?.cancel(); + await delay(500, undefined, { signal }); + continue; + } + const result: unknown = await response.json(); + if ( + response.ok && + typeof result === 'object' && + result !== null && + 'ok' in result && + result.ok === true && + 'report' in result && + typeof result.report === 'object' && + result.report !== null && + 'success' in result.report && + result.report.success === true + ) + return; + const detail = + typeof result === 'object' && + result !== null && + 'error' in result && + typeof result.error === 'string' + ? result.error + : JSON.stringify(result); + throw new Error(`Hachidori dictionary upload failed (${response.status}): ${detail}`); + } +} diff --git a/src/core/services/tokenizer/hachidori-parser-bridge.test.ts b/src/core/services/tokenizer/hachidori-parser-bridge.test.ts index ae2b46b2..f34883df 100644 --- a/src/core/services/tokenizer/hachidori-parser-bridge.test.ts +++ b/src/core/services/tokenizer/hachidori-parser-bridge.test.ts @@ -47,7 +47,7 @@ async function createHarness(emptyLibrary = false) { ], }; let dictionaries = [ - { id: 'terms', title: 'JMdict', enabled: true, revision: '1' }, + { id: 'terms', title: 'JMdict', displayName: 'Main dictionary', enabled: true, revision: '1' }, { id: 'names', title: characterDictionary, enabled: true, revision: '1' }, { id: 'frequency', @@ -55,6 +55,7 @@ async function createHarness(emptyLibrary = false) { enabled: true, revision: '1', frequencyMode: 'rank-based', + frequencyCount: 1, }, ]; let duplicate = false; @@ -425,6 +426,44 @@ test('Hachidori stats mining returns note IDs and prevents duplicate submissions assert.equal(harness.messages.filter((message) => message.type === 'hd_anki_submit').length, 1); }); +test('Hachidori mining requests render native dictionary aliases and frequency markers', async () => { + const harness = await createHarness(); + await addYomitanNoteViaSearch('食べる', harness.deps, { error: assert.fail }); + const request = harness.messages.find((message) => message.type === 'hd_anki_preflight')?.request; + assert.ok(request && typeof request === 'object'); + assert.ok('subminerEnrich' in request && request.subminerEnrich === false); + const native: unknown = await import( + pathToFileURL(path.join(extensionPath, 'anki-values.js')).href + ); + assert.ok(native && typeof native === 'object' && 'buildAnkiFields' in native); + assert.equal(typeof native.buildAnkiFields, 'function'); + if (typeof native.buildAnkiFields !== 'function') assert.fail('Native renderer is unavailable'); + const fields: unknown = await native.buildAnkiFields( + request, + { + Dictionary: { value: '{dictionary-alias}' }, + Frequency: { value: '{single-frequency-frequency}' }, + Rank: { value: '{frequency-harmonic-rank}' }, + }, + {}, + ); + assert.deepEqual(fields, { + Dictionary: 'Main dictionary', + Frequency: '
  • Frequency:
', + Rank: '42', + }); + assert.ok('dictionaryIds' in request); + assert.deepEqual(request.dictionaryIds, { + JMdict: 'terms', + [characterDictionary]: 'names', + Frequency: 'frequency', + }); + assert.deepEqual( + harness.messages.find((message) => message.type === 'hd_anki_submit')?.request, + request, + ); +}); + test('Hachidori settings automation imports ZIP bytes and removes the matching dictionary ID', async () => { const harness = await createHarness(); await harness.run( diff --git a/src/core/services/tokenizer/hachidori-parser-bridge.ts b/src/core/services/tokenizer/hachidori-parser-bridge.ts index 66d89f84..95c30d2b 100644 --- a/src/core/services/tokenizer/hachidori-parser-bridge.ts +++ b/src/core/services/tokenizer/hachidori-parser-bridge.ts @@ -236,11 +236,22 @@ export const HACHIDORI_PARSER_BRIDGE_SCRIPT = String.raw` const lookup = await engine('hd_lookup', { text: word, maxResults: 1 }); const result = lookup.results[0]; if (!result) return { noteId: null, duplicateNoteIds: [] }; + // Match the dictionary context supplied by Hachidori's popup to its + // native glossary, alias, and frequency template renderers. + const { dictionaries } = await readState(); + const frequencyModes = new Map(dictionaries.map(entry => [entry.title, entry.frequencyMode])); + const term = { ...result.term, frequencies: result.term.frequencies.map(group => + ({ ...group, frequencyMode: frequencyModes.get(group.dictionary) })) }; const status = await send('hd_anki_status', {}, 'hachidori-anki'); const request = { - ...result, generation: lookup.generation, sentence: word, searchQuery: word, + ...result, term, generation: lookup.generation, sentence: word, searchQuery: word, matchOffset: 0, documentTitle: 'SubMiner', popupSelectionText: '', - configKey: status.configKey, + configKey: status.configKey, subminerEnrich: false, + dictionaryAliases: Object.fromEntries(dictionaries.filter(entry => entry.displayName) + .map(entry => [entry.title, entry.displayName])), + dictionaryIds: Object.fromEntries(dictionaries.map(entry => [entry.title, entry.id])), + frequencyDictionaries: dictionaries.filter(entry => entry.enabled !== false && entry.frequencyCount > 0) + .map(entry => entry.title), captureUnavailable: ['screenshot', 'animation', 'audio'], }; const preflight = await send('hd_anki_preflight', { request }, 'hachidori-anki'); diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts index ee7d8fa2..61bed0c7 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts @@ -3,6 +3,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import test from 'node:test'; +import * as vm from 'node:vm'; import { countTermsFindLookups, createDeps, @@ -23,6 +24,43 @@ import { upsertYomitanDictionarySettings, } from './yomitan-parser-runtime'; +test('Yomitan restores direct Anki after disabling its managed proxy without a page helper', async () => { + const options = { profiles: [{ options: { anki: { server: 'http://127.0.0.1:8765' } } }] }; + let managedUrl: string | null = null; + const context = vm.createContext({ + chrome: { + storage: { + local: { + get: async () => ({ subminerAnkiProxyUrl: managedUrl }), + set: async (value: { subminerAnkiProxyUrl: string | null }) => { + managedUrl = value.subminerAnkiProxyUrl; + }, + }, + }, + runtime: { + sendMessage: ( + message: { action: string }, + callback: (response: { result: unknown }) => void, + ) => callback({ result: message.action === 'optionsGetFull' ? options : null }), + }, + }, + }); + const deps = createDeps(async (script) => + structuredClone(await vm.runInContext(script, context)), + ); + const logger = { error: assert.fail }; + assert.equal( + await syncYomitanDefaultAnkiServer('http://127.0.0.1:8766', deps, logger, { + forceOverride: true, + }), + true, + ); + assert.equal(managedUrl, 'http://127.0.0.1:8766'); + assert.equal(await syncYomitanDefaultAnkiServer('http://127.0.0.1:8765', deps, logger), true); + assert.equal(options.profiles[0]?.options.anki.server, 'http://127.0.0.1:8765'); + assert.equal(managedUrl, null); +}); + test('syncYomitanDefaultAnkiServer updates default profile server when script reports update', async () => { let scriptValue = ''; const deps = createDeps(async (script) => { diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.ts b/src/core/services/tokenizer/yomitan-parser-runtime.ts index 55b9e81f..6cb39ca1 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.ts @@ -1,6 +1,7 @@ import type { BrowserWindow, Extension, Session } from 'electron'; import type { AnkiConnectConfig } from '../../../types'; import { buildHachidoriAnkiHints } from './hachidori-anki-settings'; +import { uploadHachidoriDictionary } from './hachidori-dictionary-import'; import { buildHachidoriSharingScript, parseHachidoriHostStatus, @@ -681,6 +682,18 @@ async function ensureYomitanParserWindow( } if (isHachidoriExtension(yomitanExt)) { await parserWindow.webContents.executeJavaScript(HACHIDORI_PARSER_BRIDGE_SCRIPT, true); + } else { + // did-finish-load precedes the search page's asynchronous backend initialization. + await parserWindow.webContents.executeJavaScript( + `(async () => { + const deadline = Date.now() + 10000; + while (typeof window.__subminerAddNote !== 'function') { + if (Date.now() >= deadline) throw new Error('Yomitan search page initialization timed out'); + await new Promise(resolve => setTimeout(resolve, 50)); + } + })()`, + true, + ); } // Eagerly install the scan runtime so the first subtitle line does not // pay the install round trip; failures fall back to the per-request @@ -1428,10 +1441,8 @@ export async function syncYomitanDefaultAnkiServer( server: targetServer, deck: targetDeck, forceOverride, hints: hachidoriHints, }); } - let previousManagedProxy = null; - if (typeof globalThis.__subminerSetAnkiProxyUrl === 'function') { - previousManagedProxy = await globalThis.__subminerSetAnkiProxyUrl(forceOverride ? targetServer : null); - } + const { subminerAnkiProxyUrl: previousManagedProxy } = await chrome.storage.local.get('subminerAnkiProxyUrl'); + await chrome.storage.local.set({ subminerAnkiProxyUrl: forceOverride ? targetServer : null }); const optionsFull = await invoke("optionsGetFull", undefined); const profiles = Array.isArray(optionsFull.profiles) ? optionsFull.profiles : []; if (profiles.length === 0) { @@ -1783,6 +1794,7 @@ export async function importYomitanDictionaryFromZip( zipPath: string, deps: YomitanParserRuntimeDeps, logger: LoggerLike, + hachidoriManagementUrl = '', ): Promise { const normalizedZipPath = zipPath.trim(); if (!normalizedZipPath || !fs.existsSync(normalizedZipPath)) { @@ -1790,6 +1802,30 @@ export async function importYomitanDictionaryFromZip( return false; } + const extension = deps.getYomitanExt(); + if (extension && isHachidoriExtension(extension)) { + try { + const host = await requestHachidoriSharing({ type: 'hd_sharing_status' }, deps, logger); + if (host.kind === 'disconnected' || host.kind === 'unavailable') + throw new Error(host.message); + if (host.kind === 'connected') { + await uploadHachidoriDictionary(normalizedZipPath, hachidoriManagementUrl); + const window = deps.getYomitanParserWindow(); + if (window) clearYomitanParserCachesForWindow(window); + logger.info?.( + `Uploaded character dictionary to Hachidori host: ${path.basename(normalizedZipPath)}`, + ); + return true; + } + } catch (error) { + logger.error( + 'Hachidori character dictionary import failed:', + error instanceof Error ? error.message : String(error), + ); + return false; + } + } + const supportsUrlImport = await invokeYomitanSettingsAutomation( ` (() => typeof globalThis.__subminerYomitanSettingsAutomation.importDictionaryArchiveUrl === "function")(); diff --git a/src/core/services/tokenizer/yomitan-scan-test-harness.ts b/src/core/services/tokenizer/yomitan-scan-test-harness.ts index 8c9efeaa..05d03970 100644 --- a/src/core/services/tokenizer/yomitan-scan-test-harness.ts +++ b/src/core/services/tokenizer/yomitan-scan-test-harness.ts @@ -30,8 +30,17 @@ export function createDeps( } function createYomitanScriptSandbox(handler: (action: string, params: unknown) => unknown) { + const storage: Record = {}; return { chrome: { + storage: { + local: { + get: async () => ({ ...storage }), + set: async (value: Record) => { + Object.assign(storage, value); + }, + }, + }, runtime: { lastError: null, sendMessage: ( diff --git a/src/core/services/yomitan-extension-loader.ts b/src/core/services/yomitan-extension-loader.ts index 864792de..a05e479a 100644 --- a/src/core/services/yomitan-extension-loader.ts +++ b/src/core/services/yomitan-extension-loader.ts @@ -188,6 +188,11 @@ export async function loadYomitanExtension( deps.setYomitanSession(targetSession); try { + if (!externalProfilePath) { + // Electron may retain old extension scripts after an update, across app restarts. + // Keep dictionaries/settings while ensuring the bundled worker loads current code. + await targetSession.clearStorageData({ storages: ['serviceworkers'] }); + } const extensions = targetSession.extensions; const extension = await withSuppressedYomitanExtensionWarnings(() => extensions diff --git a/src/main.ts b/src/main.ts index 61aea9b1..cd09fe3b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2668,11 +2668,18 @@ const characterDictionaryAutoSyncRuntime = createCharacterDictionaryAutoSyncRunt return false; } await ensureYomitanExtensionLoaded(); - return await importYomitanDictionaryFromZip(zipPath, getYomitanParserRuntimeDeps(), { - error: (message, ...args) => logger.error(message, ...args), - info: (message, ...args) => logger.info(message, ...args), - }); + return await importYomitanDictionaryFromZip( + zipPath, + getYomitanParserRuntimeDeps(), + { + error: (message, ...args) => logger.error(message, ...args), + info: (message, ...args) => logger.info(message, ...args), + }, + configService.getConfig().hachidori.externalHostManagementUrl, + ); }, + dictionaryImportReplacesExisting: () => + getYomitanParserRuntimeDeps().getYomitanExt()?.name === 'Hachidori', deleteYomitanDictionary: async (dictionaryTitle) => { if (yomitanProfilePolicy.isExternalReadOnlyMode()) { yomitanProfilePolicy.logSkippedWrite( @@ -5232,7 +5239,7 @@ function initializeOverlayRuntime(): void { function openYomitanSettings(): boolean { if (activeDictionaryBackend === 'hachidori') { - if (configService.getConfig().yomitan.externalProfilePath.trim()) { + if (yomitanProfilePolicy.isExternalReadOnlyMode()) { logger.warn('Yomitan settings unavailable while using read-only external-profile mode.'); return false; } diff --git a/src/main/runtime/character-dictionary-auto-sync.test.ts b/src/main/runtime/character-dictionary-auto-sync.test.ts index 35b34ace..3a654b51 100644 --- a/src/main/runtime/character-dictionary-auto-sync.test.ts +++ b/src/main/runtime/character-dictionary-auto-sync.test.ts @@ -31,6 +31,45 @@ function createDeferred(): { promise: Promise; resolve: (value: T) => void return { promise, resolve }; } +test('replacement-capable imports retain the installed dictionary when an update fails', async () => { + for (const succeeds of [true, false]) { + let imported = false; + const runtime = createCharacterDictionaryAutoSyncRuntimeService({ + userDataPath: makeTempDir(), + getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }), + getOrCreateCurrentSnapshot: async () => ({ + mediaId: 7, + mediaTitle: 'Frieren', + entryCount: 100, + fromCache: true, + updatedAt: 1000, + }), + buildMergedDictionary: async () => ({ + zipPath: '/tmp/replacement.zip', + revision: 'new', + dictionaryTitle: 'SubMiner Character Dictionary', + entryCount: 100, + }), + getYomitanDictionaryInfo: async () => [ + { title: 'SubMiner Character Dictionary', revision: 'old' }, + ], + dictionaryImportReplacesExisting: () => true, + importYomitanDictionary: async () => { + imported = true; + return succeeds; + }, + deleteYomitanDictionary: async () => { + assert.fail('must keep the old dictionary until replacement succeeds'); + }, + upsertYomitanDictionarySettings: async () => true, + now: () => 1000, + }); + if (succeeds) await runtime.runSyncNow(); + else await assert.rejects(runtime.runSyncNow(), /Failed to import dictionary ZIP/); + assert.equal(imported, true); + } +}); + test('character dictionary manager snapshots, reorders, and removes MRU entries', () => { const userDataPath = makeTempDir(); const statePath = path.join(userDataPath, 'character-dictionaries', 'auto-sync-state.json'); diff --git a/src/main/runtime/character-dictionary-auto-sync.ts b/src/main/runtime/character-dictionary-auto-sync.ts index 82e27ec6..4bb7ad0c 100644 --- a/src/main/runtime/character-dictionary-auto-sync.ts +++ b/src/main/runtime/character-dictionary-auto-sync.ts @@ -80,6 +80,7 @@ export interface CharacterDictionaryAutoSyncRuntimeDeps { waitForYomitanMutationReady?: () => Promise; getYomitanDictionaryInfo: () => Promise; importYomitanDictionary: (zipPath: string) => Promise; + dictionaryImportReplacesExisting?: () => boolean; deleteYomitanDictionary: (dictionaryTitle: string) => Promise; upsertYomitanDictionarySettings: ( dictionaryTitle: string, @@ -669,7 +670,7 @@ export function createCharacterDictionaryAutoSyncRuntimeService( const importTimeoutMs = resolveImportTimeoutMs( merged?.zipPath ?? path.join(dictionariesDir, 'merged.zip'), ); - if (existing !== null) { + if (existing !== null && deps.dictionaryImportReplacesExisting?.() !== true) { await withTimeout( `deleteYomitanDictionary(${dictionaryTitle})`, deps.deleteYomitanDictionary(dictionaryTitle), diff --git a/src/main/runtime/first-run-setup-service.test.ts b/src/main/runtime/first-run-setup-service.test.ts index 0209b23b..db84a203 100644 --- a/src/main/runtime/first-run-setup-service.test.ts +++ b/src/main/runtime/first-run-setup-service.test.ts @@ -815,6 +815,36 @@ test('switching to Hachidori requires its own dictionaries and persists backend }); }); +test('reopening setup for legacy plugin cleanup preserves both backend completions', async () => { + await withTempDir(async (configDir) => { + fs.writeFileSync(path.join(configDir, 'config.jsonc'), '{}'); + const yomitan = createFirstRunSetupService({ + configDir, + getYomitanDictionaryCount: async () => 1, + detectPluginInstalled: () => false, + detectLegacyMpvPluginCandidates: () => [ + { path: '/tmp/mpv/scripts/subminer.lua', kind: 'file' }, + ], + }); + await yomitan.ensureSetupStateInitialized(); + const reopened = await yomitan.markSetupInProgress(); + assert.equal(reopened.state.status, 'in_progress'); + assert.deepEqual(reopened.state.completedDictionaryBackends, ['yomitan']); + assert.equal(yomitan.isSetupCompleted(), false); + + const hachidori = createFirstRunSetupService({ + configDir, + getDictionaryBackend: () => 'hachidori', + getYomitanDictionaryCount: async () => 1, + detectPluginInstalled: () => false, + }); + const switched = await hachidori.ensureSetupStateInitialized(); + assert.deepEqual(switched.state.completedDictionaryBackends, ['yomitan', 'hachidori']); + const restored = await yomitan.getSetupStatus(); + assert.equal(restored.state.status, 'completed'); + }); +}); + test('a legacy completed Yomitan state file survives a first Hachidori run', async () => { await withTempDir(async (configDir) => { fs.writeFileSync(path.join(configDir, 'config.jsonc'), '{}'); diff --git a/src/main/runtime/first-run-setup-service.ts b/src/main/runtime/first-run-setup-service.ts index 0dfd107b..80fb9669 100644 --- a/src/main/runtime/first-run-setup-service.ts +++ b/src/main/runtime/first-run-setup-service.ts @@ -336,7 +336,6 @@ export function createFirstRunSetupService(deps: { // incomplete until its own dictionaries are ready. const projectState = (stored: SetupState): SetupState => { const backend = getDictionaryBackend(); - if (getSetupStateDictionaryBackend(stored) === backend) return stored; const finishedBefore = hasCompletedSetupForBackend(stored, backend); // Legacy files carry their completion only as the recorded status; keep it. const storedBackend = getSetupStateDictionaryBackend(stored); @@ -346,6 +345,7 @@ export function createFirstRunSetupService(deps: { ...(stored.status === 'completed' ? [storedBackend] : []), ]), ]; + if (storedBackend === backend) return { ...stored, completedDictionaryBackends }; return { ...stored, dictionaryBackend: backend, @@ -364,7 +364,10 @@ export function createFirstRunSetupService(deps: { state = { ...state, dictionaryBackend: backend, - completedDictionaryBackends: state.status === 'completed' ? [...others, backend] : others, + completedDictionaryBackends: + state.status === 'completed' + ? [...others, backend] + : (state.completedDictionaryBackends ?? []), }; writeSetupState(setupStatePath, state); completed = state.status === 'completed'; diff --git a/src/main/runtime/stats-server-runtime.test.ts b/src/main/runtime/stats-server-runtime.test.ts index 92f4dee0..82b06431 100644 --- a/src/main/runtime/stats-server-runtime.test.ts +++ b/src/main/runtime/stats-server-runtime.test.ts @@ -1,5 +1,7 @@ import assert from 'node:assert/strict'; import test, { after } from 'node:test'; +import * as vm from 'node:vm'; +import { createDeps } from '../../core/services/tokenizer/yomitan-scan-test-harness'; import { DEFAULT_CONFIG } from '../../config'; import { ImmersionTrackerService } from '../../core/services/immersion-tracker-service'; import { createAnilistRateLimiter } from '../../core/services/anilist/rate-limiter'; @@ -34,6 +36,7 @@ function createDeferred() { function createRuntimeHarness( startServer: NonNullable, backgroundState: BackgroundStatsServerState | null = null, + overrides: Partial = {}, ) { const appStateValues: Array = []; const tracker = new ImmersionTrackerService({ dbPath: ':memory:' }); @@ -69,10 +72,54 @@ function createRuntimeHarness( removeBackgroundStatsServerState: () => {}, isBackgroundStatsServerProcessAlive: () => false, startServer, + ...overrides, }); return { runtime, appStateValues }; } +test('dashboard word mining keeps the configured proxy as Hachidori Anki endpoint', async () => { + const settings: unknown[] = []; + const context = vm.createContext({ + chrome: {}, + __subminerSyncAnkiSettings: async (value: unknown) => { + settings.push(value); + return { updated: true, matched: true }; + }, + __subminerAddNote: async () => ({ noteId: 123, duplicateNoteIds: [] }), + }); + vm.runInContext('window = globalThis', context); + const configs: Parameters>[0][] = []; + const { runtime } = createRuntimeHarness( + async (config) => { + configs.push(config); + return { close: async () => {} }; + }, + null, + { + ...createDeps(async (script) => structuredClone(await vm.runInContext(script, context))), + getResolvedConfig: () => ({ + ...DEFAULT_CONFIG, + ankiConnect: { + ...DEFAULT_CONFIG.ankiConnect, + enabled: true, + url: 'http://127.0.0.1:8765', + proxy: { + ...DEFAULT_CONFIG.ankiConnect.proxy, + enabled: true, + host: '127.0.0.1', + port: 8766, + }, + }, + }), + }, + ); + await runtime.ensureStatsServerStarted(); + assert.equal(await configs[0]?.addYomitanNote?.('入れる'), 123); + assert.ok(settings[0] && typeof settings[0] === 'object' && 'server' in settings[0]); + assert.equal(settings[0].server, 'http://127.0.0.1:8766'); + await runtime.stopStatsServer(); +}); + test('detects self-owned background stats daemon state', () => { assert.equal( isSelfOwnedBackgroundStatsDaemonState({ pid: process.pid, port: 6969, startedAtMs: 1 }), diff --git a/src/main/runtime/stats-server-runtime.ts b/src/main/runtime/stats-server-runtime.ts index 1125411c..6d16902c 100644 --- a/src/main/runtime/stats-server-runtime.ts +++ b/src/main/runtime/stats-server-runtime.ts @@ -18,7 +18,10 @@ import { writeBackgroundStatsServerState, } from './stats-daemon'; import { createEnsureStatsServerUrlHandler } from './stats-server-routing'; -import { shouldForceOverrideYomitanAnkiServer } from './yomitan-anki-server'; +import { + getPreferredYomitanAnkiServerUrl, + shouldForceOverrideYomitanAnkiServer, +} from './yomitan-anki-server'; export function isSelfOwnedBackgroundStatsDaemonState(state: { pid: number; @@ -168,7 +171,7 @@ export function createStatsServerRuntime(deps: StatsServerRuntimeDeps): { resolveSentenceSearchHeadwords: (term: string) => deps.resolveSentenceSearchHeadwords(term), addYomitanNote: async (word: string) => { const ankiConnectConfig = deps.getResolvedConfig().ankiConnect; - const ankiUrl = ankiConnectConfig.url || 'http://127.0.0.1:8765'; + const ankiUrl = getPreferredYomitanAnkiServerUrl(ankiConnectConfig); await syncYomitanDefaultAnkiServerCore(ankiUrl, yomitanDeps, yomitanLogger, { forceOverride: shouldForceOverrideYomitanAnkiServer(ankiConnectConfig), deck: ankiConnectConfig.deck, diff --git a/src/shared/anki-source.ts b/src/shared/anki-source.ts new file mode 100644 index 00000000..b41b3f55 --- /dev/null +++ b/src/shared/anki-source.ts @@ -0,0 +1,2 @@ +// Stats cards already own their sentence and media, including while polling Anki directly. +export const STATS_MINING_TAG = 'SubMiner::Stats'; diff --git a/src/shared/hachidori-sharing.ts b/src/shared/hachidori-sharing.ts index c269d336..4dab6c21 100644 --- a/src/shared/hachidori-sharing.ts +++ b/src/shared/hachidori-sharing.ts @@ -9,6 +9,23 @@ export type HachidoriSharingRequest = | { type: 'hd_sharing_client_link'; address: string } | { type: 'hd_sharing_client_unlink' }; +export function parseHachidoriManagementUrl(value: unknown): string { + if (typeof value !== 'string') throw new Error('Expected an HTTP(S) origin or an empty string.'); + if (!value.trim()) return ''; + const url = new URL(value.trim()); + if ( + !['http:', 'https:'].includes(url.protocol) || + url.username || + url.password || + url.pathname !== '/' || + url.search || + url.hash + ) { + throw new Error('Expected an HTTP(S) origin without credentials, a path, query, or fragment.'); + } + return url.origin; +} + function object(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/src/types/config.ts b/src/types/config.ts index 3682d864..b48c6263 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -144,6 +144,7 @@ export type DictionaryBackend = 'yomitan' | 'hachidori'; export interface Config { dictionaryBackend?: DictionaryBackend; + hachidori?: { externalHostManagementUrl?: string }; subtitlePosition?: SubtitlePosition; keybindings?: Keybinding[]; websocket?: WebSocketConfig; @@ -187,6 +188,7 @@ export type RawConfig = Config; export interface ResolvedConfig { dictionaryBackend: DictionaryBackend; + hachidori: { externalHostManagementUrl: string }; subtitlePosition: SubtitlePosition; keybindings: Keybinding[]; websocket: Required; diff --git a/vendor/subminer-yomitan b/vendor/subminer-yomitan index 57516d3b..e95b0b66 160000 --- a/vendor/subminer-yomitan +++ b/vendor/subminer-yomitan @@ -1 +1 @@ -Subproject commit 57516d3b7f3bffa604f575026cd39390067137ce +Subproject commit e95b0b6615dfd8e720af98596093fc90fb495e41