fix(dictionary): complete Hachidori imports and mining integration

This commit is contained in:
2026-09-22 15:01:23 -07:00
parent 15c74ef502
commit ab5e50adac
44 changed files with 756 additions and 40 deletions
+5 -5
View File
@@ -2,13 +2,13 @@ type: added
area: dictionary area: dictionary
- Added a bundled Hachidori dictionary backend alongside the default Yomitan backend. Select it with `dictionaryBackend` and restart SubMiner. - Added a bundled Hachidori dictionary backend alongside the default Yomitan backend. Select it with `dictionaryBackend` and restart SubMiner.
- The tray and dictionary-settings shortcut follow the selected backend. `--hachidori` opens Hachidori settings, while `--yomitan` continues to open Yomitan settings. - The tray and dictionary-settings shortcut follow the selected backend. `--hachidori` opens Hachidori settings, while `--yomitan` continues to open Yomitan settings. With Hachidori active, an inactive external Yomitan profile does not block the bundled Yomitan settings.
- Hachidori integrates with subtitle scanning, popup controls, lookup tracking, character dictionaries, and Anki media enrichment, with separate dictionaries and settings for each backend. - Hachidori integrates with subtitle scanning, popup controls, lookup tracking, character dictionaries, and Anki media enrichment, with separate dictionaries and settings for each backend. Linked Docker hosts receive character dictionary uploads through `hachidori.externalHostManagementUrl`, retry busy imports, and replace the previous dictionary only after a successful import.
- Hachidori auto-populates its first Anki template from SubMiner's deck, tags, and field mappings, detects an unambiguous matching note type, and preserves existing custom templates. Anki discovery retries after an unavailable connection. - Hachidori auto-populates its first Anki template from SubMiner's deck, tags, and field mappings, detects an unambiguous matching note type, and preserves existing custom templates. Anki discovery retries after an unavailable connection.
- Hachidori saves downloadable word audio before sending a note through SubMiner's Anki proxy, so freshly mined animated cards include the word-audio delay. - Hachidori saves downloadable word audio before sending a note through SubMiner's Anki proxy, so freshly mined animated cards include the word-audio delay.
- Hachidori reloads its background code on startup so extension updates take effect while preserving installed dictionaries and settings. - Both bundled dictionary backends reload their background code on startup so extension updates take effect while preserving installed dictionaries and settings.
- First-run setup remembers each backend that finished it, so switching back does not repeat setup, and the launcher gates playback on the backend the running app started with. Current incomplete or cancelled setup takes precedence over stale completion history. - First-run setup remembers each backend that finished it, including when reopened for legacy plugin cleanup, so switching back does not repeat setup, and the launcher gates playback on the backend the running app started with. Current incomplete or cancelled setup takes precedence over stale completion history. A running Yomitan session keeps using its external profile until it restarts into Hachidori.
- Stats dashboard mining and deck lookup use the selected backend. Settings labels for popup pause and the dictionary deck no longer name Yomitan, and the backend selector sits with the other dictionary settings. - Stats dashboard mining and deck lookup use the selected backend, with the Anki proxy enabled or disabled. Dashboard cards retain the selected history line and media instead of being processed again with the current mpv subtitle, with both Yomitan and Hachidori. Stats cards carry a `SubMiner::Stats` tag so polling also preserves their context. Hachidori word mining supplies dictionary aliases, IDs, and frequency metadata to its native Anki renderer. Settings labels for popup pause and the dictionary deck no longer name Yomitan, and the backend selector sits with the other dictionary settings.
- Hachidori scans, dictionary counts, and settings reads wait for the dictionary engine to finish loading or importing instead of caching empty results. - Hachidori scans, dictionary counts, and settings reads wait for the dictionary engine to finish loading or importing instead of caching empty results.
- First-run setup can link an external Hachidori dictionary host in an app, browser, or Docker container, verify its library, and unlink back to local dictionaries. The optional host controls are collapsed by default and explain which apps or containers must stay running. Unresponsive connection checks time out so setup remains usable. Anki mining, media enrichment, and custom toolbar buttons stay in SubMiner, including when editing buttons while linked. - First-run setup can link an external Hachidori dictionary host in an app, browser, or Docker container, verify its library, and unlink back to local dictionaries. The optional host controls are collapsed by default and explain which apps or containers must stay running. Unresponsive connection checks time out so setup remains usable. Anki mining, media enrichment, and custom toolbar buttons stay in SubMiner, including when editing buttons while linked.
- Hachidori frequency highlighting reuses dictionary-entry ranks and fills missing ranks through its existing API. Entries without a matching definition may remain unranked. The bundled integration is maintained in a pinned fork submodule using upstream HoshiDicts and WASM binaries. - Hachidori frequency highlighting reuses dictionary-entry ranks and fills missing ranks through its existing API. Entries without a matching definition may remain unranked. The bundled integration is maintained in a pinned fork submodule using upstream HoshiDicts and WASM binaries.
+9
View File
@@ -13,6 +13,15 @@
// ========================================== // ==========================================
"dictionaryBackend": "yomitan", // Dictionary lookup backend. Restart SubMiner after changing this setting. Values: yomitan | hachidori "dictionaryBackend": "yomitan", // Dictionary lookup backend. Restart SubMiner after changing this setting. Values: yomitan | hachidori
// ==========================================
// Hachidori External Dictionary Imports
// Configure the linked Docker host management URL, for example http://127.0.0.1:8780.
// Used only while Hachidori is linked to an external host.
// ==========================================
"hachidori": {
"externalHostManagementUrl": "" // Docker host management URL for automatic character dictionary uploads and replacement. Empty disables external uploads.
}, // Configure the linked Docker host management URL, for example http://127.0.0.1:8780.
// ========================================== // ==========================================
// Japanese Subtitle Generation // Japanese Subtitle Generation
// Generate timed Japanese subtitles from local audio using whisper.cpp. // Generate timed Japanese subtitles from local audio using whisper.cpp.
+2
View File
@@ -177,6 +177,8 @@ The configuration file includes several main sections:
Each backend stores its own dictionaries and mining settings. `yomitan.externalProfilePath` applies only to Yomitan. See [Hachidori setup](./usage.md#hachidori-setup) before switching an existing installation. Each backend stores its own dictionaries and mining settings. `yomitan.externalProfilePath` applies only to Yomitan. See [Hachidori setup](./usage.md#hachidori-setup) before switching an existing installation.
`hachidori.externalHostManagementUrl` specifies the linked Docker host's HTTP(S) management origin for automatic character dictionary uploads and replacement. Use the management port, not the sharing or dictionary API port. See [Hachidori setup](./usage.md#hachidori-setup) for an example and [the generated configuration example](/config.example.jsonc) for the default.
### Logging ### Logging
Control the minimum log level for runtime output: Control the minimum log level for runtime output:
+3 -1
View File
@@ -152,12 +152,14 @@ Stats server config lives under `stats`:
The Search tab and the Vocabulary tab's word detail panel both mine from subtitle lines in your viewing history. Search matches sentence text and media titles, and **Search by headword** is enabled by default so dictionary-form searches such as `知らない` can find tracked subtitle lines with inflected variants. Turn that toggle off for exact text/title matching only. Each line with a valid source file offers sentence-card mining; word/audio mining is available when the selected word or searched word appears in the sentence: The Search tab and the Vocabulary tab's word detail panel both mine from subtitle lines in your viewing history. Search matches sentence text and media titles, and **Search by headword** is enabled by default so dictionary-form searches such as `知らない` can find tracked subtitle lines with inflected variants. Turn that toggle off for exact text/title matching only. Each line with a valid source file offers sentence-card mining; word/audio mining is available when the selected word or searched word appears in the sentence:
- **Mine Word** - performs a full Yomitan dictionary lookup for the word (definition, reading, pitch accent, etc.) via a short-lived hidden helper, then enriches the card with sentence audio, a screenshot or animated AVIF clip, the highlighted sentence, and metadata extracted from the source video file. Requires Anki and Yomitan dictionaries to be loaded. - **Mine Word** - looks up the word with the selected dictionary backend, Yomitan or Hachidori, then enriches the card with sentence audio, a screenshot or animated AVIF clip, the highlighted sentence, and metadata extracted from the source video file. The selected history line supplies the card's context even while another subtitle is playing in mpv. Hachidori uses its configured Anki template, dictionary aliases, and frequency metadata. Requires Anki and the selected backend's dictionaries to be loaded.
- **Mine Sentence** - creates a sentence card directly with the `IsSentenceCard` flag set (for Lapis/Kiku workflows), along with audio and image from the source video. - **Mine Sentence** - creates a sentence card directly with the `IsSentenceCard` flag set (for Lapis/Kiku workflows), along with audio and image from the source video.
- **Mine Audio** - creates an audio-only card with the `IsAudioCard` flag, attaching only the sentence audio clip. - **Mine Audio** - creates an audio-only card with the `IsAudioCard` flag, attaching only the sentence audio clip.
All three modes respect your `ankiConnect` config: deck, model, field mappings, media settings (static vs AVIF, quality, dimensions), audio padding, metadata pattern, and tags. Media generation runs in parallel for faster card creation. All three modes respect your `ankiConnect` config: deck, model, field mappings, media settings (static vs AVIF, quality, dimensions), audio padding, metadata pattern, and tags. Media generation runs in parallel for faster card creation.
Stats cards also receive the `SubMiner::Stats` tag. SubMiner uses it to preserve their selected history context when detecting new cards through polling with the Anki proxy disabled.
Secondary subtitle text is stored alongside primary subtitles during playback, but the Search tab does not use it for display or matching. Secondary subtitle text is stored alongside primary subtitles during playback, but the Search tab does not use it for display or matching.
### Word exclusion list ### Word exclusion list
+9
View File
@@ -13,6 +13,15 @@
// ========================================== // ==========================================
"dictionaryBackend": "yomitan", // Dictionary lookup backend. Restart SubMiner after changing this setting. Values: yomitan | hachidori "dictionaryBackend": "yomitan", // Dictionary lookup backend. Restart SubMiner after changing this setting. Values: yomitan | hachidori
// ==========================================
// Hachidori External Dictionary Imports
// Configure the linked Docker host management URL, for example http://127.0.0.1:8780.
// Used only while Hachidori is linked to an external host.
// ==========================================
"hachidori": {
"externalHostManagementUrl": "" // Docker host management URL for automatic character dictionary uploads and replacement. Empty disables external uploads.
}, // Configure the linked Docker host management URL, for example http://127.0.0.1:8780.
// ========================================== // ==========================================
// Japanese Subtitle Generation // Japanese Subtitle Generation
// Generate timed Japanese subtitles from local audio using whisper.cpp. // Generate timed Japanese subtitles from local audio using whisper.cpp.
+3 -1
View File
@@ -329,9 +329,11 @@ Both named settings flags work independently of the selected backend. Opening se
Hachidori uses SubMiner's subtitle scanning, lookup counter, popup pause behavior, controller commands, character dictionaries, and Anki media enrichment. Keep SubMiner's AnkiConnect proxy enabled for screenshots and sentence audio. SubMiner routes Hachidori to that proxy when it is active; Hachidori's own screen recorder and screenshot capture are disabled in the embedded app. Hachidori uses SubMiner's subtitle scanning, lookup counter, popup pause behavior, controller commands, character dictionaries, and Anki media enrichment. Keep SubMiner's AnkiConnect proxy enabled for screenshots and sentence audio. SubMiner routes Hachidori to that proxy when it is active; Hachidori's own screen recorder and screenshot capture are disabled in the embedded app.
For automatic character dictionary sync with a Docker host, set `hachidori.externalHostManagementUrl` to that same host's management origin, for example `"http://127.0.0.1:8780"`. This is separate from the WebSocket sharing address. SubMiner uploads the generated ZIP directly and replaces its previous dictionary after a successful import; busy imports are retried. Keep this URL pointed at the currently linked Docker host if you change hosts. An empty value disables external uploads and reports a configuration error when sync is attempted. Local Hachidori dictionaries do not need this setting. External browser/app hosts without the Docker management API do not support this automatic upload path.
Existing controls such as `startupWarmups.yomitanExtension` and `subtitleStyle.autoPauseVideoOnYomitanPopup` apply to the selected backend. Hachidori has one dictionary configuration, so character-dictionary profile scope applies to that configuration. Existing controls such as `startupWarmups.yomitanExtension` and `subtitleStyle.autoPauseVideoOnYomitanPopup` apply to the selected backend. Hachidori has one dictionary configuration, so character-dictionary profile scope applies to that configuration.
First-run setup remembers each backend that finished it. Switching to a backend for the first time asks for that backend's dictionaries; switching back to one that already finished does not repeat setup. Until SubMiner restarts, it keeps running the backend it started with, and the launcher gates playback on that running backend and logs a restart reminder. First-run setup remembers each backend that finished it, including when setup is reopened for legacy plugin cleanup. Switching to a backend for the first time asks for that backend's dictionaries; switching back to one that already finished does not repeat setup. Until SubMiner restarts, it keeps running the backend it started with, and the launcher gates playback on that running backend and logs a restart reminder. A running Yomitan session continues using its external profile until the restart. While Hachidori is active, `--yomitan` opens the bundled Yomitan settings even if an external Yomitan profile is configured.
Hachidori's own duplicate handling differs from Yomitan's. Choosing **Overwrite** in the Hachidori popup updates the existing note and SubMiner enriches its media, while **Add anyway** creates a new note and runs SubMiner's Kiku/Senren [field grouping](./anki-integration.md#field-grouping-kiku-senren). Mining from the stats dashboard uses the selected backend as well. Hachidori's own duplicate handling differs from Yomitan's. Choosing **Overwrite** in the Hachidori popup updates the existing note and SubMiner enriches its media, while **Add anyway** creates a new note and runs SubMiner's Kiku/Senren [field grouping](./anki-integration.md#field-grouping-kiku-senren). Mining from the stats dashboard uses the selected backend as well.
+1 -4
View File
@@ -108,10 +108,7 @@ export function loadLauncherDictionaryBackend(): DictionaryBackend {
} }
export function hasLauncherExternalYomitanProfileConfig(): boolean { export function hasLauncherExternalYomitanProfileConfig(): boolean {
const config = readLauncherMainConfigObject(); return readExternalYomitanProfilePath(readLauncherMainConfigObject()) !== null;
return (
config?.dictionaryBackend !== 'hachidori' && readExternalYomitanProfilePath(config) !== null
);
} }
export function readPluginRuntimeConfig(logLevel: LogLevel): PluginRuntimeConfig { export function readPluginRuntimeConfig(logLevel: LogLevel): PluginRuntimeConfig {
+51
View File
@@ -1112,3 +1112,54 @@ test('classifyJellyfinChildSelection keeps container drilldown state instead of
id: 'season-2', id: 'season-2',
}); });
}); });
test('external Yomitan profile remains available while a running app awaits a backend switch', () => {
withTempDir((dir) => {
const env = makeTestEnv(dir, path.join(dir, 'config'));
const configPath = resolveConfigFilePath({
appDataDir: env.APPDATA,
xdgConfigHome: env.XDG_CONFIG_HOME,
homeDir: dir,
existsSync: () => false,
});
fs.mkdirSync(path.dirname(configPath), { recursive: true });
fs.writeFileSync(
configPath,
JSON.stringify({
dictionaryBackend: 'hachidori',
yomitan: { externalProfilePath: '/external/yomitan-profile' },
}),
);
const result = spawnSync(
process.execPath,
[
'--eval',
`
import assert from 'node:assert/strict';
import { hasLauncherExternalYomitanProfileConfig } from './launcher/config.ts';
import { ensureLauncherSetupReady } from './launcher/setup-gate.ts';
import { createDefaultSetupState } from './src/shared/setup-state.ts';
for (const running of [true, false]) {
let launches = 0;
let tick = 0;
const ready = await ensureLauncherSetupReady({
dictionaryBackend: 'hachidori',
isAppRunning: async () => running,
readSetupState: () => ({ ...createDefaultSetupState(), dictionaryBackend: 'yomitan' }),
isExternalYomitanConfigured: hasLauncherExternalYomitanProfileConfig,
launchSetupApp: () => { launches += 1; },
sleep: async () => {},
now: () => tick++,
timeoutMs: 2,
pollIntervalMs: 1,
});
assert.equal(ready, running);
assert.equal(launches, running ? 0 : 1);
}
`,
],
{ cwd: process.cwd(), env, encoding: 'utf8', timeout: LAUNCHER_RUN_TIMEOUT_MS },
);
assert.equal(result.status, 0, result.stderr);
});
});
+1 -1
View File
@@ -12,7 +12,7 @@
"get-frequency": "bun run scripts/get_frequency.ts --pretty --color-top-x 10000 --yomitan-user-data ~/.config/SubMiner --colorized-line", "get-frequency": "bun run scripts/get_frequency.ts --pretty --color-top-x 10000 --yomitan-user-data ~/.config/SubMiner --colorized-line",
"get-frequency:electron": "bun run build:yomitan && bun build scripts/get_frequency.ts --format=cjs --target=node --outfile dist/scripts/get_frequency.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/get_frequency.js --pretty --color-top-x 10000 --yomitan-user-data ~/.config/SubMiner --colorized-line", "get-frequency:electron": "bun run build:yomitan && bun build scripts/get_frequency.ts --format=cjs --target=node --outfile dist/scripts/get_frequency.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/get_frequency.js --pretty --color-top-x 10000 --yomitan-user-data ~/.config/SubMiner --colorized-line",
"test-yomitan-parser": "bun run scripts/test-yomitan-parser.ts", "test-yomitan-parser": "bun run scripts/test-yomitan-parser.ts",
"test:dictionary:electron": "env -u ELECTRON_RUN_AS_NODE electron --no-sandbox scripts/check-dictionary-backends.cjs && env -u ELECTRON_RUN_AS_NODE electron --no-sandbox scripts/check-dictionary-backends.cjs --backend=yomitan && env -u ELECTRON_RUN_AS_NODE electron --no-sandbox --ozone-platform=x11 scripts/check-hachidori-parser.cjs", "test:dictionary:electron": "bun scripts/run-dictionary-settings-smoke.mjs && bun scripts/run-dictionary-settings-smoke.mjs --backend=yomitan && env -u ELECTRON_RUN_AS_NODE electron --no-sandbox --ozone-platform=x11 scripts/check-hachidori-parser.cjs",
"test-yomitan-parser:electron": "bun run build:yomitan && bun build scripts/test-yomitan-parser.ts --format=cjs --target=node --outfile dist/scripts/test-yomitan-parser.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/test-yomitan-parser.js", "test-yomitan-parser:electron": "bun run build:yomitan && bun build scripts/test-yomitan-parser.ts --format=cjs --target=node --outfile dist/scripts/test-yomitan-parser.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/test-yomitan-parser.js",
"verify-known-word-highlights:electron": "bun run build:yomitan && bun build scripts/verify-known-word-highlights.ts --format=cjs --target=node --outfile dist/scripts/verify-known-word-highlights.js --packages=external && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/verify-known-word-highlights.js", "verify-known-word-highlights:electron": "bun run build:yomitan && bun build scripts/verify-known-word-highlights.ts --format=cjs --target=node --outfile dist/scripts/verify-known-word-highlights.js --packages=external && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/verify-known-word-highlights.js",
"record-tokenizer-fixture:electron": "bun run build:yomitan && bun build scripts/record-tokenizer-fixture.ts --format=cjs --target=node --outfile dist/scripts/record-tokenizer-fixture.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/record-tokenizer-fixture.js", "record-tokenizer-fixture:electron": "bun run build:yomitan && bun build scripts/record-tokenizer-fixture.ts --format=cjs --target=node --outfile dist/scripts/record-tokenizer-fixture.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/record-tokenizer-fixture.js",
+16 -5
View File
@@ -7,7 +7,11 @@ if (process.platform !== 'linux')
throw new Error('This app-entry smoke requires Linux XDG isolation.'); throw new Error('This app-entry smoke requires Linux XDG isolation.');
const root = process.cwd(); const root = process.cwd();
const backend = process.argv.includes('--backend=yomitan') ? 'yomitan' : 'hachidori'; const backend = process.argv.includes('--backend=yomitan') ? 'yomitan' : 'hachidori';
const profile = fs.mkdtempSync('/tmp/subminer-hachi-settings-'); const profile = process.env.SUBMINER_DICTIONARY_SMOKE_DATA;
assert(
profile && fs.existsSync(profile),
'Use bun run test:dictionary:electron for profile isolation',
);
process.env.XDG_CONFIG_HOME = profile; process.env.XDG_CONFIG_HOME = profile;
process.env.XDG_DATA_HOME = path.join(profile, 'data'); process.env.XDG_DATA_HOME = path.join(profile, 'data');
fs.mkdirSync(path.join(profile, 'SubMiner')); fs.mkdirSync(path.join(profile, 'SubMiner'));
@@ -15,6 +19,10 @@ fs.writeFileSync(
path.join(profile, 'SubMiner', 'config.json'), path.join(profile, 'SubMiner', 'config.json'),
JSON.stringify({ JSON.stringify({
dictionaryBackend: backend, dictionaryBackend: backend,
// An inactive external Yomitan profile must not block bundled settings in Hachidori mode.
yomitan: {
externalProfilePath: backend === 'hachidori' ? path.join(profile, 'external-yomitan') : '',
},
mpv: { socketPath: path.join(profile, 'missing-mpv.sock') }, mpv: { socketPath: path.join(profile, 'missing-mpv.sock') },
ankiConnect: { enabled: false }, ankiConnect: { enabled: false },
startupWarmups: { lowPowerMode: true }, startupWarmups: { lowPowerMode: true },
@@ -39,8 +47,12 @@ const deadline = setTimeout(() => {
'FAIL timeout', 'FAIL timeout',
BrowserWindow.getAllWindows().map((w) => w.webContents.getURL()), BrowserWindow.getAllWindows().map((w) => w.webContents.getURL()),
); );
app.exit(1); finish(1);
}, 60000); }, 60000);
function finish(exitCode) {
clearTimeout(deadline);
app.exit(exitCode);
}
(async () => { (async () => {
await app.whenReady(); await app.whenReady();
let window; let window;
@@ -102,9 +114,8 @@ const deadline = setTimeout(() => {
`PASS ${backend} overlay session, independent settings windows, real preload external link bridge`, `PASS ${backend} overlay session, independent settings windows, real preload external link bridge`,
); );
clearTimeout(deadline); finish(0);
app.exit(0);
})().catch((error) => { })().catch((error) => {
console.error(error); console.error(error);
app.exit(1); finish(1);
}); });
+26
View File
@@ -0,0 +1,26 @@
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawnSync } from 'node:child_process';
import { createRequire } from 'node:module';
import { fileURLToPath } from 'node:url';
const require = createRequire(import.meta.url);
const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-hachi-settings-'));
const env = { ...process.env, SUBMINER_DICTIONARY_SMOKE_DATA: profile };
delete env.ELECTRON_RUN_AS_NODE;
try {
const result = spawnSync(
require('electron'),
[
'--no-sandbox',
fileURLToPath(new URL('./check-dictionary-backends.cjs', import.meta.url)),
...process.argv.slice(2),
],
{ env, stdio: 'inherit', timeout: 75_000 },
);
if (result.error) throw result.error;
process.exitCode = result.status ?? 1;
} finally {
fs.rmSync(profile, { recursive: true, force: true, maxRetries: 3 });
}
+5 -1
View File
@@ -99,7 +99,11 @@ async function smoke() {
if (details.statusCode >= 400) failedRequests.push(`${details.url}: ${details.statusCode}`); if (details.statusCode >= 400) failedRequests.push(`${details.url}: ${details.statusCode}`);
}); });
await statsWindow.loadURL(url); await statsWindow.loadURL(url);
assert.equal((await fetch(`${url}/api/stats/overview`)).status, 200); for (const endpoint of ['overview', 'sessions']) {
const response = await fetch(`${url}/api/stats/${endpoint}`);
assert.equal(response.status, 200, `Stats ${endpoint} request failed`);
await response.json();
}
} finally { } finally {
statsWindow.destroy(); statsWindow.destroy();
await statsServer.close(); await statsServer.close();
@@ -505,6 +505,63 @@ test('proxy enriches confirmed Hachidori overwrites without counting a new card
} }
}); });
test('stats-owned notes bypass overlay enrichment while popup notes still enqueue', async () => {
const processed: number[] = [];
const added: number[] = [];
const received: unknown[] = [];
let noteId = 70;
const upstream = http.createServer(async (req, res) => {
const chunks: Buffer[] = [];
for await (const chunk of req) chunks.push(Buffer.from(chunk));
received.push(JSON.parse(Buffer.concat(chunks).toString()));
res.end(JSON.stringify({ result: ++noteId, error: null }));
});
upstream.listen(0, '127.0.0.1');
await once(upstream, 'listening');
const address = upstream.address();
assert.ok(address && typeof address === 'object');
const proxy = new AnkiConnectProxyServer({
shouldAutoUpdateNewCards: () => true,
processNewCard: async (id) => {
processed.push(id);
},
recordCardsAdded: (_count, ids) => {
added.push(...ids);
},
logInfo: () => {},
logWarn: () => {},
logError: () => {},
});
try {
proxy.start({ host: '127.0.0.1', port: 0, upstreamUrl: `http://127.0.0.1:${address.port}` });
await proxy.waitUntilReady();
const server: unknown = Reflect.get(proxy, 'server');
assert.ok(server instanceof http.Server);
const bound = server.address();
assert.ok(bound && typeof bound === 'object');
for (const metadata of [{ subminerEnrich: false }, {}]) {
const response: Response = await fetch(`http://127.0.0.1:${bound.port}`, {
method: 'POST',
body: JSON.stringify({
action: 'addNote',
version: 6,
params: { note: { fields: { Expression: '猫' } }, ...metadata },
}),
});
assert.equal(response.status, 200);
await response.json();
}
await waitForCondition(() => processed.includes(72));
assert.deepEqual(processed, [72]);
assert.deepEqual(added, [71, 72]);
assert.equal(JSON.stringify(received).includes('subminerEnrich'), false);
} finally {
proxy.stop();
upstream.close();
await once(upstream, 'close');
}
});
test('proxy returns addNote response without waiting for background enrichment', async () => { test('proxy returns addNote response without waiting for background enrichment', async () => {
const processed: number[] = []; const processed: number[] = [];
let releaseProcessing: (() => void) | undefined; let releaseProcessing: (() => void) | undefined;
+13 -2
View File
@@ -264,12 +264,23 @@ export class AnkiConnectProxyServer {
return; return;
} }
this.maybeTrackDuplicateNoteIds(requestJson, action, responseResult);
const noteIds = const noteIds =
action === 'multi' action === 'multi'
? this.collectMultiResultIds(requestJson, responseResult) ? this.collectMultiResultIds(requestJson, responseResult)
: this.collectNoteIdsForAction(action, responseResult); : this.collectNoteIdsForAction(action, responseResult);
const params = requestJson.params;
if (
action === 'addNote' &&
params &&
typeof params === 'object' &&
'subminerEnrich' in params &&
params.subminerEnrich === false
) {
// Stats owns the saved sentence and media; the live mpv context is unrelated.
if (noteIds.length > 0) this.deps.recordCardsAdded?.(noteIds.length, noteIds);
return;
}
this.maybeTrackDuplicateNoteIds(requestJson, action, responseResult);
if (noteIds.length === 0 && shouldFallbackToLatestAdded) { if (noteIds.length === 0 && shouldFallbackToLatestAdded) {
void this.enqueueMostRecentAddedNote(); void this.enqueueMostRecentAddedNote();
return; return;
@@ -8,6 +8,7 @@ import {
import type { SubtitleMiningContext } from '../types/subtitle'; import type { SubtitleMiningContext } from '../types/subtitle';
import type { CardKind } from '../types/anki'; import type { CardKind } from '../types/anki';
import { applyCardKindFlagFields } from './card-kinds'; import { applyCardKindFlagFields } from './card-kinds';
import { STATS_MINING_TAG } from '../shared/anki-source';
function setCardTypeFields( function setCardTypeFields(
updatedFields: Record<string, string>, updatedFields: Record<string, string>,
@@ -110,6 +111,27 @@ function createWorkflowHarness() {
}; };
} }
test('NoteUpdateWorkflow preserves stats cards discovered by polling', async () => {
const { workflow, deps, updates } = createWorkflowHarness();
const note = {
noteId: 42,
tags: [STATS_MINING_TAG],
fields: { Expression: { value: '猫' }, Sentence: { value: '猫がいる。' } },
};
deps.client.notesInfo = async () => [note];
deps.captureSubtitleMediaContext = () => assert.fail('Must not capture current playback');
deps.findDuplicateNote = async () => assert.fail('Must not regroup a stats card');
let cachedNote: NoteUpdateWorkflowNoteInfo | undefined;
deps.appendKnownWordsFromNoteInfo = (value) => {
cachedNote = value;
};
await workflow.execute(42);
assert.deepEqual(updates, []);
assert.equal(cachedNote, note);
});
test('NoteUpdateWorkflow updates sentence field and emits notification', async () => { test('NoteUpdateWorkflow updates sentence field and emits notification', async () => {
const harness = createWorkflowHarness(); const harness = createWorkflowHarness();
@@ -8,9 +8,11 @@ import type {
WordCardKind, WordCardKind,
} from '../types/anki'; } from '../types/anki';
import { resolveWordCardKind } from './note-field-utils'; import { resolveWordCardKind } from './note-field-utils';
import { STATS_MINING_TAG } from '../shared/anki-source';
export interface NoteUpdateWorkflowNoteInfo { export interface NoteUpdateWorkflowNoteInfo {
noteId: number; noteId: number;
tags?: string[];
fields: Record<string, { value: string }>; fields: Record<string, { value: string }>;
} }
@@ -181,6 +183,10 @@ export class NoteUpdateWorkflow {
} }
const noteInfo = notesInfo[0]!; const noteInfo = notesInfo[0]!;
if (noteInfo.tags?.includes(STATS_MINING_TAG)) {
this.deps.appendKnownWordsFromNoteInfo(noteInfo);
return;
}
const fields = this.deps.extractFields(noteInfo.fields); const fields = this.deps.extractFields(noteInfo.fields);
const config = this.deps.getConfig(); const config = this.deps.getConfig();
+2
View File
@@ -23,6 +23,7 @@ export type {
const { const {
dictionaryBackend, dictionaryBackend,
hachidori,
subtitlePosition, subtitlePosition,
keybindings, keybindings,
websocket, websocket,
@@ -59,6 +60,7 @@ const { stats } = STATS_DEFAULT_CONFIG;
export const DEFAULT_CONFIG: ResolvedConfig = { export const DEFAULT_CONFIG: ResolvedConfig = {
subtitleGeneration: { ...DEFAULT_SUBTITLE_GENERATION_CONFIG }, subtitleGeneration: { ...DEFAULT_SUBTITLE_GENERATION_CONFIG },
dictionaryBackend, dictionaryBackend,
hachidori,
subtitlePosition, subtitlePosition,
keybindings, keybindings,
websocket, websocket,
+2
View File
@@ -3,6 +3,7 @@ import { ResolvedConfig } from '../../types/config';
export const CORE_DEFAULT_CONFIG: Pick< export const CORE_DEFAULT_CONFIG: Pick<
ResolvedConfig, ResolvedConfig,
| 'dictionaryBackend' | 'dictionaryBackend'
| 'hachidori'
| 'subtitlePosition' | 'subtitlePosition'
| 'keybindings' | 'keybindings'
| 'websocket' | 'websocket'
@@ -20,6 +21,7 @@ export const CORE_DEFAULT_CONFIG: Pick<
| 'auto_start_overlay' | 'auto_start_overlay'
> = { > = {
dictionaryBackend: 'yomitan', dictionaryBackend: 'yomitan',
hachidori: { externalHostManagementUrl: '' },
subtitlePosition: { yPercent: 10 }, subtitlePosition: { yPercent: 10 },
keybindings: [], keybindings: [],
websocket: { websocket: {
+7
View File
@@ -81,6 +81,13 @@ export function buildCoreConfigOptionRegistry(
] as const; ] as const;
return [ return [
{
path: 'hachidori.externalHostManagementUrl',
kind: 'string',
defaultValue: defaultConfig.hachidori.externalHostManagementUrl,
description:
'Docker host management URL for automatic character dictionary uploads and replacement. Empty disables external uploads.',
},
{ {
path: 'dictionaryBackend', path: 'dictionaryBackend',
kind: 'enum', kind: 'enum',
@@ -9,6 +9,14 @@ const CORE_TEMPLATE_SECTIONS: ConfigTemplateSection[] = [
], ],
key: 'dictionaryBackend', key: 'dictionaryBackend',
}, },
{
title: 'Hachidori External Dictionary Imports',
description: [
'Configure the linked Docker host management URL, for example http://127.0.0.1:8780.',
],
notes: ['Used only while Hachidori is linked to an external host.'],
key: 'hachidori',
},
{ {
title: 'Japanese Subtitle Generation', title: 'Japanese Subtitle Generation',
description: [ description: [
+16
View File
@@ -2,10 +2,26 @@ import { ResolveContext } from './context';
import { applyControllerConfig } from './controller'; import { applyControllerConfig } from './controller';
import { isNotificationType, isOverlayNotificationPosition } from '../../types/notification'; import { isNotificationType, isOverlayNotificationPosition } from '../../types/notification';
import { asBoolean, asNumber, asString, isObject } from './shared'; import { asBoolean, asNumber, asString, isObject } from './shared';
import { parseHachidoriManagementUrl } from '../../shared/hachidori-sharing';
export function applyCoreDomainConfig(context: ResolveContext): void { export function applyCoreDomainConfig(context: ResolveContext): void {
const { src, resolved, warn } = context; const { src, resolved, warn } = context;
if (isObject(src.hachidori) && src.hachidori.externalHostManagementUrl !== undefined) {
try {
resolved.hachidori.externalHostManagementUrl = parseHachidoriManagementUrl(
src.hachidori.externalHostManagementUrl,
);
} catch {
warn(
'hachidori.externalHostManagementUrl',
src.hachidori.externalHostManagementUrl,
resolved.hachidori.externalHostManagementUrl,
'Expected an HTTP(S) origin or an empty string.',
);
}
}
if (src.dictionaryBackend === 'yomitan' || src.dictionaryBackend === 'hachidori') { if (src.dictionaryBackend === 'yomitan' || src.dictionaryBackend === 'hachidori') {
resolved.dictionaryBackend = src.dictionaryBackend; resolved.dictionaryBackend = src.dictionaryBackend;
} else if (src.dictionaryBackend !== undefined) { } else if (src.dictionaryBackend !== undefined) {
@@ -28,3 +28,24 @@ test('unknown dictionary backend values warn and preserve the default', () => {
assert.equal(warnings[0]?.path, 'dictionaryBackend'); assert.equal(warnings[0]?.path, 'dictionaryBackend');
} }
}); });
test('Hachidori external import URL accepts HTTP origins and rejects invalid targets', () => {
assert.equal(resolveConfig({}).resolved.hachidori.externalHostManagementUrl, '');
const result = resolveConfig({
hachidori: { externalHostManagementUrl: 'http://127.0.0.1:8780/' },
});
assert.equal(result.resolved.hachidori.externalHostManagementUrl, 'http://127.0.0.1:8780');
assert.deepEqual(result.warnings, []);
for (const value of [
'file:///tmp/dict',
'http://host/import',
'http://user:password@host',
true,
]) {
const { context, warnings } = createResolveContext({});
context.src.hachidori = { externalHostManagementUrl: value };
applyCoreDomainConfig(context);
assert.equal(context.resolved.hachidori.externalHostManagementUrl, '');
assert.equal(warnings[0]?.path, 'hachidori.externalHostManagementUrl');
}
});
+1 -1
View File
@@ -341,7 +341,7 @@ function humanizePath(path: string): string {
} }
function categoryAndSection(path: string): { category: ConfigSettingsCategory; section: string } { function categoryAndSection(path: string): { category: ConfigSettingsCategory; section: string } {
if (path === 'dictionaryBackend') { if (path === 'dictionaryBackend' || path.startsWith('hachidori.')) {
return { category: 'integrations', section: 'Dictionary Lookup' }; return { category: 'integrations', section: 'Dictionary Lookup' };
} }
if ( if (
@@ -2525,6 +2525,8 @@ Aligned English subtitle
await new Promise((resolve) => setTimeout(resolve, 1)); await new Promise((resolve) => setTimeout(resolve, 1));
} }
const addedBeforeMediaFinished = requests.some((request) => request.action === 'addNote'); const addedBeforeMediaFinished = requests.some((request) => request.action === 'addNote');
const addRequest = requests.find((request) => request.action === 'addNote');
assert.deepEqual(addRequest?.params?.note?.tags, ['SubMiner', 'SubMiner::Stats']);
mediaRelease.audio?.(); mediaRelease.audio?.();
mediaRelease.image?.(); mediaRelease.image?.();
@@ -7,6 +7,7 @@ import { resolveAnimatedImageLeadInSeconds } from '../../../anki-integration/ani
import { clampMediaEndTime } from '../../../anki-integration/media-duration.js'; import { clampMediaEndTime } from '../../../anki-integration/media-duration.js';
import { MediaGenerator } from '../../../media-generator.js'; import { MediaGenerator } from '../../../media-generator.js';
import { statsJson } from '../../../types/stats-http-contract.js'; import { statsJson } from '../../../types/stats-http-contract.js';
import { STATS_MINING_TAG } from '../../../shared/anki-source.js';
import { import {
resolveRetimedSecondarySubtitleTextFromSidecar, resolveRetimedSecondarySubtitleTextFromSidecar,
resolveSecondarySubtitleTextFromSidecar, resolveSecondarySubtitleTextFromSidecar,
@@ -359,7 +360,7 @@ export function registerStatsMiningRoutes(app: Hono, options?: StatsMiningRouteO
} }
const model = ankiConfig.isLapis?.sentenceCardModel || 'Basic'; const model = ankiConfig.isLapis?.sentenceCardModel || 'Basic';
const tags = ankiConfig.tags ?? ['SubMiner']; const tags = [...new Set([...(ankiConfig.tags ?? ['SubMiner']), STATS_MINING_TAG])];
const addNotePromise = timeMiningPhase( const addNotePromise = timeMiningPhase(
mode, mode,
@@ -0,0 +1,100 @@
import assert from 'node:assert/strict';
import { mkdtemp, writeFile } from 'node:fs/promises';
import { createServer } from 'node:http';
import { once } from 'node:events';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { uploadHachidoriDictionary } from './hachidori-dictionary-import';
import { importYomitanDictionaryFromZip } from './yomitan-parser-runtime';
import { createDeps } from './yomitan-scan-test-harness';
test('linked Hachidori uploads replacement bytes, retries a busy host, and never invokes local import', async () => {
const zipPath = path.join(await mkdtemp(path.join(os.tmpdir(), 'hachi-import-')), 'merged.zip');
const archive = Buffer.from('PK-test-archive');
await writeFile(zipPath, archive);
let attempts = 0;
const server = createServer(async (request, response) => {
attempts += 1;
assert.equal(request.url, '/import?name=merged.zip&replace=true');
assert.equal(request.method, 'POST');
const chunks: Buffer[] = [];
for await (const chunk of request) chunks.push(Buffer.from(chunk));
assert.deepEqual(Buffer.concat(chunks), archive);
response.writeHead(attempts === 1 ? 409 : 200, { 'Content-Type': 'application/json' });
response.end(
JSON.stringify(attempts === 1 ? { error: 'busy' } : { ok: true, report: { success: true } }),
);
});
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const address = server.address();
assert.ok(address && typeof address === 'object');
const managementUrl = `http://127.0.0.1:${address.port}`;
let connected = true;
const deps = {
...createDeps(async (script) => {
assert.ok(script.includes('hd_sharing_status'), 'must not invoke local ZIP automation');
return {
ok: true,
sharing: {
client: {
linked: true,
connected,
address: 'ws://127.0.0.1:8771/link',
host: { dictionaryCount: 8 },
},
},
};
}),
getYomitanExt: () => ({
id: 'hachi',
name: 'Hachidori',
version: '1',
path: '',
url: '',
manifest: {},
}),
};
try {
assert.equal(
await importYomitanDictionaryFromZip(zipPath, deps, { error: assert.fail }, managementUrl),
true,
);
assert.equal(attempts, 2);
const errors: string[] = [];
const logger = { error: (...args: unknown[]) => errors.push(args.join(' ')) };
assert.equal(await importYomitanDictionaryFromZip(zipPath, deps, logger), false);
assert.match(errors.pop() ?? '', /externalHostManagementUrl/);
connected = false;
assert.equal(await importYomitanDictionaryFromZip(zipPath, deps, logger, managementUrl), false);
assert.equal(attempts, 2);
} finally {
server.closeAllConnections();
await new Promise<void>((resolve) => server.close(() => resolve()));
}
});
test('Hachidori upload requires a successful import report, not just HTTP success', async () => {
const zipPath = path.join(await mkdtemp(path.join(os.tmpdir(), 'hachi-import-')), 'merged.zip');
await writeFile(zipPath, 'bad archive');
const server = createServer((_request, response) => {
response.writeHead(200, { 'Content-Type': 'application/json' });
response.end(
JSON.stringify({ ok: true, report: { success: false, error: 'Invalid archive' } }),
);
});
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const address = server.address();
assert.ok(address && typeof address === 'object');
try {
await assert.rejects(
uploadHachidoriDictionary(zipPath, `http://127.0.0.1:${address.port}`),
/Invalid archive/,
);
} finally {
server.closeAllConnections();
await new Promise<void>((resolve) => server.close(() => resolve()));
}
});
@@ -0,0 +1,59 @@
import { readFile } from 'node:fs/promises';
import path from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { parseHachidoriManagementUrl } from '../../../shared/hachidori-sharing';
// Upload from the main process: extension blob URLs cannot cross the sharing link,
// and the Docker management API deliberately rejects browser cross-origin writes.
export async function uploadHachidoriDictionary(
zipPath: string,
managementUrl: string,
): Promise<void> {
const origin = parseHachidoriManagementUrl(managementUrl);
if (!origin) {
throw new Error(
'Set hachidori.externalHostManagementUrl to the linked Docker host management URL to sync character dictionaries.',
);
}
const url = new URL('/import', origin);
url.searchParams.set('name', path.basename(zipPath));
url.searchParams.set('replace', 'true');
const bytes = await readFile(zipPath);
const signal = AbortSignal.timeout(300_000);
for (;;) {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/zip' },
body: bytes,
signal,
redirect: 'error',
});
if (response.status === 409) {
await response.body?.cancel();
await delay(500, undefined, { signal });
continue;
}
const result: unknown = await response.json();
if (
response.ok &&
typeof result === 'object' &&
result !== null &&
'ok' in result &&
result.ok === true &&
'report' in result &&
typeof result.report === 'object' &&
result.report !== null &&
'success' in result.report &&
result.report.success === true
)
return;
const detail =
typeof result === 'object' &&
result !== null &&
'error' in result &&
typeof result.error === 'string'
? result.error
: JSON.stringify(result);
throw new Error(`Hachidori dictionary upload failed (${response.status}): ${detail}`);
}
}
@@ -47,7 +47,7 @@ async function createHarness(emptyLibrary = false) {
], ],
}; };
let dictionaries = [ let dictionaries = [
{ id: 'terms', title: 'JMdict', enabled: true, revision: '1' }, { id: 'terms', title: 'JMdict', displayName: 'Main dictionary', enabled: true, revision: '1' },
{ id: 'names', title: characterDictionary, enabled: true, revision: '1' }, { id: 'names', title: characterDictionary, enabled: true, revision: '1' },
{ {
id: 'frequency', id: 'frequency',
@@ -55,6 +55,7 @@ async function createHarness(emptyLibrary = false) {
enabled: true, enabled: true,
revision: '1', revision: '1',
frequencyMode: 'rank-based', frequencyMode: 'rank-based',
frequencyCount: 1,
}, },
]; ];
let duplicate = false; let duplicate = false;
@@ -425,6 +426,44 @@ test('Hachidori stats mining returns note IDs and prevents duplicate submissions
assert.equal(harness.messages.filter((message) => message.type === 'hd_anki_submit').length, 1); assert.equal(harness.messages.filter((message) => message.type === 'hd_anki_submit').length, 1);
}); });
test('Hachidori mining requests render native dictionary aliases and frequency markers', async () => {
const harness = await createHarness();
await addYomitanNoteViaSearch('食べる', harness.deps, { error: assert.fail });
const request = harness.messages.find((message) => message.type === 'hd_anki_preflight')?.request;
assert.ok(request && typeof request === 'object');
assert.ok('subminerEnrich' in request && request.subminerEnrich === false);
const native: unknown = await import(
pathToFileURL(path.join(extensionPath, 'anki-values.js')).href
);
assert.ok(native && typeof native === 'object' && 'buildAnkiFields' in native);
assert.equal(typeof native.buildAnkiFields, 'function');
if (typeof native.buildAnkiFields !== 'function') assert.fail('Native renderer is unavailable');
const fields: unknown = await native.buildAnkiFields(
request,
{
Dictionary: { value: '{dictionary-alias}' },
Frequency: { value: '{single-frequency-frequency}' },
Rank: { value: '{frequency-harmonic-rank}' },
},
{},
);
assert.deepEqual(fields, {
Dictionary: 'Main dictionary',
Frequency: '<ul style="text-align: left;"><li>Frequency: </li></ul>',
Rank: '42',
});
assert.ok('dictionaryIds' in request);
assert.deepEqual(request.dictionaryIds, {
JMdict: 'terms',
[characterDictionary]: 'names',
Frequency: 'frequency',
});
assert.deepEqual(
harness.messages.find((message) => message.type === 'hd_anki_submit')?.request,
request,
);
});
test('Hachidori settings automation imports ZIP bytes and removes the matching dictionary ID', async () => { test('Hachidori settings automation imports ZIP bytes and removes the matching dictionary ID', async () => {
const harness = await createHarness(); const harness = await createHarness();
await harness.run( await harness.run(
@@ -236,11 +236,22 @@ export const HACHIDORI_PARSER_BRIDGE_SCRIPT = String.raw`
const lookup = await engine('hd_lookup', { text: word, maxResults: 1 }); const lookup = await engine('hd_lookup', { text: word, maxResults: 1 });
const result = lookup.results[0]; const result = lookup.results[0];
if (!result) return { noteId: null, duplicateNoteIds: [] }; if (!result) return { noteId: null, duplicateNoteIds: [] };
// Match the dictionary context supplied by Hachidori's popup to its
// native glossary, alias, and frequency template renderers.
const { dictionaries } = await readState();
const frequencyModes = new Map(dictionaries.map(entry => [entry.title, entry.frequencyMode]));
const term = { ...result.term, frequencies: result.term.frequencies.map(group =>
({ ...group, frequencyMode: frequencyModes.get(group.dictionary) })) };
const status = await send('hd_anki_status', {}, 'hachidori-anki'); const status = await send('hd_anki_status', {}, 'hachidori-anki');
const request = { const request = {
...result, generation: lookup.generation, sentence: word, searchQuery: word, ...result, term, generation: lookup.generation, sentence: word, searchQuery: word,
matchOffset: 0, documentTitle: 'SubMiner', popupSelectionText: '', matchOffset: 0, documentTitle: 'SubMiner', popupSelectionText: '',
configKey: status.configKey, configKey: status.configKey, subminerEnrich: false,
dictionaryAliases: Object.fromEntries(dictionaries.filter(entry => entry.displayName)
.map(entry => [entry.title, entry.displayName])),
dictionaryIds: Object.fromEntries(dictionaries.map(entry => [entry.title, entry.id])),
frequencyDictionaries: dictionaries.filter(entry => entry.enabled !== false && entry.frequencyCount > 0)
.map(entry => entry.title),
captureUnavailable: ['screenshot', 'animation', 'audio'], captureUnavailable: ['screenshot', 'animation', 'audio'],
}; };
const preflight = await send('hd_anki_preflight', { request }, 'hachidori-anki'); const preflight = await send('hd_anki_preflight', { request }, 'hachidori-anki');
@@ -3,6 +3,7 @@ import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
import * as path from 'path'; import * as path from 'path';
import test from 'node:test'; import test from 'node:test';
import * as vm from 'node:vm';
import { import {
countTermsFindLookups, countTermsFindLookups,
createDeps, createDeps,
@@ -23,6 +24,43 @@ import {
upsertYomitanDictionarySettings, upsertYomitanDictionarySettings,
} from './yomitan-parser-runtime'; } from './yomitan-parser-runtime';
test('Yomitan restores direct Anki after disabling its managed proxy without a page helper', async () => {
const options = { profiles: [{ options: { anki: { server: 'http://127.0.0.1:8765' } } }] };
let managedUrl: string | null = null;
const context = vm.createContext({
chrome: {
storage: {
local: {
get: async () => ({ subminerAnkiProxyUrl: managedUrl }),
set: async (value: { subminerAnkiProxyUrl: string | null }) => {
managedUrl = value.subminerAnkiProxyUrl;
},
},
},
runtime: {
sendMessage: (
message: { action: string },
callback: (response: { result: unknown }) => void,
) => callback({ result: message.action === 'optionsGetFull' ? options : null }),
},
},
});
const deps = createDeps(async (script) =>
structuredClone(await vm.runInContext(script, context)),
);
const logger = { error: assert.fail };
assert.equal(
await syncYomitanDefaultAnkiServer('http://127.0.0.1:8766', deps, logger, {
forceOverride: true,
}),
true,
);
assert.equal(managedUrl, 'http://127.0.0.1:8766');
assert.equal(await syncYomitanDefaultAnkiServer('http://127.0.0.1:8765', deps, logger), true);
assert.equal(options.profiles[0]?.options.anki.server, 'http://127.0.0.1:8765');
assert.equal(managedUrl, null);
});
test('syncYomitanDefaultAnkiServer updates default profile server when script reports update', async () => { test('syncYomitanDefaultAnkiServer updates default profile server when script reports update', async () => {
let scriptValue = ''; let scriptValue = '';
const deps = createDeps(async (script) => { const deps = createDeps(async (script) => {
@@ -1,6 +1,7 @@
import type { BrowserWindow, Extension, Session } from 'electron'; import type { BrowserWindow, Extension, Session } from 'electron';
import type { AnkiConnectConfig } from '../../../types'; import type { AnkiConnectConfig } from '../../../types';
import { buildHachidoriAnkiHints } from './hachidori-anki-settings'; import { buildHachidoriAnkiHints } from './hachidori-anki-settings';
import { uploadHachidoriDictionary } from './hachidori-dictionary-import';
import { import {
buildHachidoriSharingScript, buildHachidoriSharingScript,
parseHachidoriHostStatus, parseHachidoriHostStatus,
@@ -681,6 +682,18 @@ async function ensureYomitanParserWindow(
} }
if (isHachidoriExtension(yomitanExt)) { if (isHachidoriExtension(yomitanExt)) {
await parserWindow.webContents.executeJavaScript(HACHIDORI_PARSER_BRIDGE_SCRIPT, true); await parserWindow.webContents.executeJavaScript(HACHIDORI_PARSER_BRIDGE_SCRIPT, true);
} else {
// did-finish-load precedes the search page's asynchronous backend initialization.
await parserWindow.webContents.executeJavaScript(
`(async () => {
const deadline = Date.now() + 10000;
while (typeof window.__subminerAddNote !== 'function') {
if (Date.now() >= deadline) throw new Error('Yomitan search page initialization timed out');
await new Promise(resolve => setTimeout(resolve, 50));
}
})()`,
true,
);
} }
// Eagerly install the scan runtime so the first subtitle line does not // Eagerly install the scan runtime so the first subtitle line does not
// pay the install round trip; failures fall back to the per-request // pay the install round trip; failures fall back to the per-request
@@ -1428,10 +1441,8 @@ export async function syncYomitanDefaultAnkiServer(
server: targetServer, deck: targetDeck, forceOverride, hints: hachidoriHints, server: targetServer, deck: targetDeck, forceOverride, hints: hachidoriHints,
}); });
} }
let previousManagedProxy = null; const { subminerAnkiProxyUrl: previousManagedProxy } = await chrome.storage.local.get('subminerAnkiProxyUrl');
if (typeof globalThis.__subminerSetAnkiProxyUrl === 'function') { await chrome.storage.local.set({ subminerAnkiProxyUrl: forceOverride ? targetServer : null });
previousManagedProxy = await globalThis.__subminerSetAnkiProxyUrl(forceOverride ? targetServer : null);
}
const optionsFull = await invoke("optionsGetFull", undefined); const optionsFull = await invoke("optionsGetFull", undefined);
const profiles = Array.isArray(optionsFull.profiles) ? optionsFull.profiles : []; const profiles = Array.isArray(optionsFull.profiles) ? optionsFull.profiles : [];
if (profiles.length === 0) { if (profiles.length === 0) {
@@ -1783,6 +1794,7 @@ export async function importYomitanDictionaryFromZip(
zipPath: string, zipPath: string,
deps: YomitanParserRuntimeDeps, deps: YomitanParserRuntimeDeps,
logger: LoggerLike, logger: LoggerLike,
hachidoriManagementUrl = '',
): Promise<boolean> { ): Promise<boolean> {
const normalizedZipPath = zipPath.trim(); const normalizedZipPath = zipPath.trim();
if (!normalizedZipPath || !fs.existsSync(normalizedZipPath)) { if (!normalizedZipPath || !fs.existsSync(normalizedZipPath)) {
@@ -1790,6 +1802,30 @@ export async function importYomitanDictionaryFromZip(
return false; return false;
} }
const extension = deps.getYomitanExt();
if (extension && isHachidoriExtension(extension)) {
try {
const host = await requestHachidoriSharing({ type: 'hd_sharing_status' }, deps, logger);
if (host.kind === 'disconnected' || host.kind === 'unavailable')
throw new Error(host.message);
if (host.kind === 'connected') {
await uploadHachidoriDictionary(normalizedZipPath, hachidoriManagementUrl);
const window = deps.getYomitanParserWindow();
if (window) clearYomitanParserCachesForWindow(window);
logger.info?.(
`Uploaded character dictionary to Hachidori host: ${path.basename(normalizedZipPath)}`,
);
return true;
}
} catch (error) {
logger.error(
'Hachidori character dictionary import failed:',
error instanceof Error ? error.message : String(error),
);
return false;
}
}
const supportsUrlImport = await invokeYomitanSettingsAutomation<boolean>( const supportsUrlImport = await invokeYomitanSettingsAutomation<boolean>(
` `
(() => typeof globalThis.__subminerYomitanSettingsAutomation.importDictionaryArchiveUrl === "function")(); (() => typeof globalThis.__subminerYomitanSettingsAutomation.importDictionaryArchiveUrl === "function")();
@@ -30,8 +30,17 @@ export function createDeps(
} }
function createYomitanScriptSandbox(handler: (action: string, params: unknown) => unknown) { function createYomitanScriptSandbox(handler: (action: string, params: unknown) => unknown) {
const storage: Record<string, unknown> = {};
return { return {
chrome: { chrome: {
storage: {
local: {
get: async () => ({ ...storage }),
set: async (value: Record<string, unknown>) => {
Object.assign(storage, value);
},
},
},
runtime: { runtime: {
lastError: null, lastError: null,
sendMessage: ( sendMessage: (
@@ -188,6 +188,11 @@ export async function loadYomitanExtension(
deps.setYomitanSession(targetSession); deps.setYomitanSession(targetSession);
try { try {
if (!externalProfilePath) {
// Electron may retain old extension scripts after an update, across app restarts.
// Keep dictionaries/settings while ensuring the bundled worker loads current code.
await targetSession.clearStorageData({ storages: ['serviceworkers'] });
}
const extensions = targetSession.extensions; const extensions = targetSession.extensions;
const extension = await withSuppressedYomitanExtensionWarnings(() => const extension = await withSuppressedYomitanExtensionWarnings(() =>
extensions extensions
+10 -3
View File
@@ -2668,11 +2668,18 @@ const characterDictionaryAutoSyncRuntime = createCharacterDictionaryAutoSyncRunt
return false; return false;
} }
await ensureYomitanExtensionLoaded(); await ensureYomitanExtensionLoaded();
return await importYomitanDictionaryFromZip(zipPath, getYomitanParserRuntimeDeps(), { return await importYomitanDictionaryFromZip(
zipPath,
getYomitanParserRuntimeDeps(),
{
error: (message, ...args) => logger.error(message, ...args), error: (message, ...args) => logger.error(message, ...args),
info: (message, ...args) => logger.info(message, ...args), info: (message, ...args) => logger.info(message, ...args),
});
}, },
configService.getConfig().hachidori.externalHostManagementUrl,
);
},
dictionaryImportReplacesExisting: () =>
getYomitanParserRuntimeDeps().getYomitanExt()?.name === 'Hachidori',
deleteYomitanDictionary: async (dictionaryTitle) => { deleteYomitanDictionary: async (dictionaryTitle) => {
if (yomitanProfilePolicy.isExternalReadOnlyMode()) { if (yomitanProfilePolicy.isExternalReadOnlyMode()) {
yomitanProfilePolicy.logSkippedWrite( yomitanProfilePolicy.logSkippedWrite(
@@ -5232,7 +5239,7 @@ function initializeOverlayRuntime(): void {
function openYomitanSettings(): boolean { function openYomitanSettings(): boolean {
if (activeDictionaryBackend === 'hachidori') { if (activeDictionaryBackend === 'hachidori') {
if (configService.getConfig().yomitan.externalProfilePath.trim()) { if (yomitanProfilePolicy.isExternalReadOnlyMode()) {
logger.warn('Yomitan settings unavailable while using read-only external-profile mode.'); logger.warn('Yomitan settings unavailable while using read-only external-profile mode.');
return false; return false;
} }
@@ -31,6 +31,45 @@ function createDeferred<T>(): { promise: Promise<T>; resolve: (value: T) => void
return { promise, resolve }; return { promise, resolve };
} }
test('replacement-capable imports retain the installed dictionary when an update fails', async () => {
for (const succeeds of [true, false]) {
let imported = false;
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath: makeTempDir(),
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async () => ({
mediaId: 7,
mediaTitle: 'Frieren',
entryCount: 100,
fromCache: true,
updatedAt: 1000,
}),
buildMergedDictionary: async () => ({
zipPath: '/tmp/replacement.zip',
revision: 'new',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 100,
}),
getYomitanDictionaryInfo: async () => [
{ title: 'SubMiner Character Dictionary', revision: 'old' },
],
dictionaryImportReplacesExisting: () => true,
importYomitanDictionary: async () => {
imported = true;
return succeeds;
},
deleteYomitanDictionary: async () => {
assert.fail('must keep the old dictionary until replacement succeeds');
},
upsertYomitanDictionarySettings: async () => true,
now: () => 1000,
});
if (succeeds) await runtime.runSyncNow();
else await assert.rejects(runtime.runSyncNow(), /Failed to import dictionary ZIP/);
assert.equal(imported, true);
}
});
test('character dictionary manager snapshots, reorders, and removes MRU entries', () => { test('character dictionary manager snapshots, reorders, and removes MRU entries', () => {
const userDataPath = makeTempDir(); const userDataPath = makeTempDir();
const statePath = path.join(userDataPath, 'character-dictionaries', 'auto-sync-state.json'); const statePath = path.join(userDataPath, 'character-dictionaries', 'auto-sync-state.json');
@@ -80,6 +80,7 @@ export interface CharacterDictionaryAutoSyncRuntimeDeps {
waitForYomitanMutationReady?: () => Promise<void>; waitForYomitanMutationReady?: () => Promise<void>;
getYomitanDictionaryInfo: () => Promise<AutoSyncDictionaryInfo[]>; getYomitanDictionaryInfo: () => Promise<AutoSyncDictionaryInfo[]>;
importYomitanDictionary: (zipPath: string) => Promise<boolean>; importYomitanDictionary: (zipPath: string) => Promise<boolean>;
dictionaryImportReplacesExisting?: () => boolean;
deleteYomitanDictionary: (dictionaryTitle: string) => Promise<boolean>; deleteYomitanDictionary: (dictionaryTitle: string) => Promise<boolean>;
upsertYomitanDictionarySettings: ( upsertYomitanDictionarySettings: (
dictionaryTitle: string, dictionaryTitle: string,
@@ -669,7 +670,7 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
const importTimeoutMs = resolveImportTimeoutMs( const importTimeoutMs = resolveImportTimeoutMs(
merged?.zipPath ?? path.join(dictionariesDir, 'merged.zip'), merged?.zipPath ?? path.join(dictionariesDir, 'merged.zip'),
); );
if (existing !== null) { if (existing !== null && deps.dictionaryImportReplacesExisting?.() !== true) {
await withTimeout( await withTimeout(
`deleteYomitanDictionary(${dictionaryTitle})`, `deleteYomitanDictionary(${dictionaryTitle})`,
deps.deleteYomitanDictionary(dictionaryTitle), deps.deleteYomitanDictionary(dictionaryTitle),
@@ -815,6 +815,36 @@ test('switching to Hachidori requires its own dictionaries and persists backend
}); });
}); });
test('reopening setup for legacy plugin cleanup preserves both backend completions', async () => {
await withTempDir(async (configDir) => {
fs.writeFileSync(path.join(configDir, 'config.jsonc'), '{}');
const yomitan = createFirstRunSetupService({
configDir,
getYomitanDictionaryCount: async () => 1,
detectPluginInstalled: () => false,
detectLegacyMpvPluginCandidates: () => [
{ path: '/tmp/mpv/scripts/subminer.lua', kind: 'file' },
],
});
await yomitan.ensureSetupStateInitialized();
const reopened = await yomitan.markSetupInProgress();
assert.equal(reopened.state.status, 'in_progress');
assert.deepEqual(reopened.state.completedDictionaryBackends, ['yomitan']);
assert.equal(yomitan.isSetupCompleted(), false);
const hachidori = createFirstRunSetupService({
configDir,
getDictionaryBackend: () => 'hachidori',
getYomitanDictionaryCount: async () => 1,
detectPluginInstalled: () => false,
});
const switched = await hachidori.ensureSetupStateInitialized();
assert.deepEqual(switched.state.completedDictionaryBackends, ['yomitan', 'hachidori']);
const restored = await yomitan.getSetupStatus();
assert.equal(restored.state.status, 'completed');
});
});
test('a legacy completed Yomitan state file survives a first Hachidori run', async () => { test('a legacy completed Yomitan state file survives a first Hachidori run', async () => {
await withTempDir(async (configDir) => { await withTempDir(async (configDir) => {
fs.writeFileSync(path.join(configDir, 'config.jsonc'), '{}'); fs.writeFileSync(path.join(configDir, 'config.jsonc'), '{}');
+5 -2
View File
@@ -336,7 +336,6 @@ export function createFirstRunSetupService(deps: {
// incomplete until its own dictionaries are ready. // incomplete until its own dictionaries are ready.
const projectState = (stored: SetupState): SetupState => { const projectState = (stored: SetupState): SetupState => {
const backend = getDictionaryBackend(); const backend = getDictionaryBackend();
if (getSetupStateDictionaryBackend(stored) === backend) return stored;
const finishedBefore = hasCompletedSetupForBackend(stored, backend); const finishedBefore = hasCompletedSetupForBackend(stored, backend);
// Legacy files carry their completion only as the recorded status; keep it. // Legacy files carry their completion only as the recorded status; keep it.
const storedBackend = getSetupStateDictionaryBackend(stored); const storedBackend = getSetupStateDictionaryBackend(stored);
@@ -346,6 +345,7 @@ export function createFirstRunSetupService(deps: {
...(stored.status === 'completed' ? [storedBackend] : []), ...(stored.status === 'completed' ? [storedBackend] : []),
]), ]),
]; ];
if (storedBackend === backend) return { ...stored, completedDictionaryBackends };
return { return {
...stored, ...stored,
dictionaryBackend: backend, dictionaryBackend: backend,
@@ -364,7 +364,10 @@ export function createFirstRunSetupService(deps: {
state = { state = {
...state, ...state,
dictionaryBackend: backend, dictionaryBackend: backend,
completedDictionaryBackends: state.status === 'completed' ? [...others, backend] : others, completedDictionaryBackends:
state.status === 'completed'
? [...others, backend]
: (state.completedDictionaryBackends ?? []),
}; };
writeSetupState(setupStatePath, state); writeSetupState(setupStatePath, state);
completed = state.status === 'completed'; completed = state.status === 'completed';
@@ -1,5 +1,7 @@
import assert from 'node:assert/strict'; import assert from 'node:assert/strict';
import test, { after } from 'node:test'; import test, { after } from 'node:test';
import * as vm from 'node:vm';
import { createDeps } from '../../core/services/tokenizer/yomitan-scan-test-harness';
import { DEFAULT_CONFIG } from '../../config'; import { DEFAULT_CONFIG } from '../../config';
import { ImmersionTrackerService } from '../../core/services/immersion-tracker-service'; import { ImmersionTrackerService } from '../../core/services/immersion-tracker-service';
import { createAnilistRateLimiter } from '../../core/services/anilist/rate-limiter'; import { createAnilistRateLimiter } from '../../core/services/anilist/rate-limiter';
@@ -34,6 +36,7 @@ function createDeferred<T>() {
function createRuntimeHarness( function createRuntimeHarness(
startServer: NonNullable<StatsServerRuntimeDeps['startServer']>, startServer: NonNullable<StatsServerRuntimeDeps['startServer']>,
backgroundState: BackgroundStatsServerState | null = null, backgroundState: BackgroundStatsServerState | null = null,
overrides: Partial<StatsServerRuntimeDeps> = {},
) { ) {
const appStateValues: Array<StatsServer | null> = []; const appStateValues: Array<StatsServer | null> = [];
const tracker = new ImmersionTrackerService({ dbPath: ':memory:' }); const tracker = new ImmersionTrackerService({ dbPath: ':memory:' });
@@ -69,10 +72,54 @@ function createRuntimeHarness(
removeBackgroundStatsServerState: () => {}, removeBackgroundStatsServerState: () => {},
isBackgroundStatsServerProcessAlive: () => false, isBackgroundStatsServerProcessAlive: () => false,
startServer, startServer,
...overrides,
}); });
return { runtime, appStateValues }; return { runtime, appStateValues };
} }
test('dashboard word mining keeps the configured proxy as Hachidori Anki endpoint', async () => {
const settings: unknown[] = [];
const context = vm.createContext({
chrome: {},
__subminerSyncAnkiSettings: async (value: unknown) => {
settings.push(value);
return { updated: true, matched: true };
},
__subminerAddNote: async () => ({ noteId: 123, duplicateNoteIds: [] }),
});
vm.runInContext('window = globalThis', context);
const configs: Parameters<NonNullable<StatsServerRuntimeDeps['startServer']>>[0][] = [];
const { runtime } = createRuntimeHarness(
async (config) => {
configs.push(config);
return { close: async () => {} };
},
null,
{
...createDeps(async (script) => structuredClone(await vm.runInContext(script, context))),
getResolvedConfig: () => ({
...DEFAULT_CONFIG,
ankiConnect: {
...DEFAULT_CONFIG.ankiConnect,
enabled: true,
url: 'http://127.0.0.1:8765',
proxy: {
...DEFAULT_CONFIG.ankiConnect.proxy,
enabled: true,
host: '127.0.0.1',
port: 8766,
},
},
}),
},
);
await runtime.ensureStatsServerStarted();
assert.equal(await configs[0]?.addYomitanNote?.('入れる'), 123);
assert.ok(settings[0] && typeof settings[0] === 'object' && 'server' in settings[0]);
assert.equal(settings[0].server, 'http://127.0.0.1:8766');
await runtime.stopStatsServer();
});
test('detects self-owned background stats daemon state', () => { test('detects self-owned background stats daemon state', () => {
assert.equal( assert.equal(
isSelfOwnedBackgroundStatsDaemonState({ pid: process.pid, port: 6969, startedAtMs: 1 }), isSelfOwnedBackgroundStatsDaemonState({ pid: process.pid, port: 6969, startedAtMs: 1 }),
+5 -2
View File
@@ -18,7 +18,10 @@ import {
writeBackgroundStatsServerState, writeBackgroundStatsServerState,
} from './stats-daemon'; } from './stats-daemon';
import { createEnsureStatsServerUrlHandler } from './stats-server-routing'; import { createEnsureStatsServerUrlHandler } from './stats-server-routing';
import { shouldForceOverrideYomitanAnkiServer } from './yomitan-anki-server'; import {
getPreferredYomitanAnkiServerUrl,
shouldForceOverrideYomitanAnkiServer,
} from './yomitan-anki-server';
export function isSelfOwnedBackgroundStatsDaemonState(state: { export function isSelfOwnedBackgroundStatsDaemonState(state: {
pid: number; pid: number;
@@ -168,7 +171,7 @@ export function createStatsServerRuntime(deps: StatsServerRuntimeDeps): {
resolveSentenceSearchHeadwords: (term: string) => deps.resolveSentenceSearchHeadwords(term), resolveSentenceSearchHeadwords: (term: string) => deps.resolveSentenceSearchHeadwords(term),
addYomitanNote: async (word: string) => { addYomitanNote: async (word: string) => {
const ankiConnectConfig = deps.getResolvedConfig().ankiConnect; const ankiConnectConfig = deps.getResolvedConfig().ankiConnect;
const ankiUrl = ankiConnectConfig.url || 'http://127.0.0.1:8765'; const ankiUrl = getPreferredYomitanAnkiServerUrl(ankiConnectConfig);
await syncYomitanDefaultAnkiServerCore(ankiUrl, yomitanDeps, yomitanLogger, { await syncYomitanDefaultAnkiServerCore(ankiUrl, yomitanDeps, yomitanLogger, {
forceOverride: shouldForceOverrideYomitanAnkiServer(ankiConnectConfig), forceOverride: shouldForceOverrideYomitanAnkiServer(ankiConnectConfig),
deck: ankiConnectConfig.deck, deck: ankiConnectConfig.deck,
+2
View File
@@ -0,0 +1,2 @@
// Stats cards already own their sentence and media, including while polling Anki directly.
export const STATS_MINING_TAG = 'SubMiner::Stats';
+17
View File
@@ -9,6 +9,23 @@ export type HachidoriSharingRequest =
| { type: 'hd_sharing_client_link'; address: string } | { type: 'hd_sharing_client_link'; address: string }
| { type: 'hd_sharing_client_unlink' }; | { type: 'hd_sharing_client_unlink' };
export function parseHachidoriManagementUrl(value: unknown): string {
if (typeof value !== 'string') throw new Error('Expected an HTTP(S) origin or an empty string.');
if (!value.trim()) return '';
const url = new URL(value.trim());
if (
!['http:', 'https:'].includes(url.protocol) ||
url.username ||
url.password ||
url.pathname !== '/' ||
url.search ||
url.hash
) {
throw new Error('Expected an HTTP(S) origin without credentials, a path, query, or fragment.');
}
return url.origin;
}
function object(value: unknown): value is Record<string, unknown> { function object(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value); return typeof value === 'object' && value !== null && !Array.isArray(value);
} }
+2
View File
@@ -144,6 +144,7 @@ export type DictionaryBackend = 'yomitan' | 'hachidori';
export interface Config { export interface Config {
dictionaryBackend?: DictionaryBackend; dictionaryBackend?: DictionaryBackend;
hachidori?: { externalHostManagementUrl?: string };
subtitlePosition?: SubtitlePosition; subtitlePosition?: SubtitlePosition;
keybindings?: Keybinding[]; keybindings?: Keybinding[];
websocket?: WebSocketConfig; websocket?: WebSocketConfig;
@@ -187,6 +188,7 @@ export type RawConfig = Config;
export interface ResolvedConfig { export interface ResolvedConfig {
dictionaryBackend: DictionaryBackend; dictionaryBackend: DictionaryBackend;
hachidori: { externalHostManagementUrl: string };
subtitlePosition: SubtitlePosition; subtitlePosition: SubtitlePosition;
keybindings: Keybinding[]; keybindings: Keybinding[];
websocket: Required<WebSocketConfig>; websocket: Required<WebSocketConfig>;