feat(dictionary): add Hachidori backend support (#266)

scripts/build-hachidori.mjs

@@ -0,0 +1,69 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { fileURLToPath } from 'node:url';
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const source = path.join(repoRoot, 'vendor', 'hachidori');
+const output = path.join(repoRoot, 'build', 'hachidori');
+const provenance = JSON.parse(fs.readFileSync(path.join(source, 'SOURCE.json'), 'utf8'));
+
+// Upstream commits the engine binaries. Verify the pinned bytes before staging
+// so ordinary app builds need neither Emscripten nor network access.
+for (const [file, expected] of Object.entries(provenance.artifacts)) {
+  const actual = createHash('sha256')
+    .update(fs.readFileSync(path.join(source, file)))
+    .digest('hex');
+  if (actual !== expected) throw new Error(`Hachidori artifact checksum mismatch: ${file}`);
+}
+const extension = path.join(source, 'extension');
+const manifest = JSON.parse(fs.readFileSync(path.join(extension, 'manifest.json'), 'utf8'));
+for (const file of [
+  manifest.options_page,
+  manifest.background.service_worker,
+  'offscreen.html',
+  ...manifest.content_scripts.flatMap(({ js, css }) => [...js, ...css]),
+]) {
+  if (!fs.existsSync(path.join(extension, file)))
+    throw new Error(`Missing Hachidori asset: ${file}`);
+}
+fs.rmSync(output, { recursive: true, force: true });
+fs.mkdirSync(output, { recursive: true });
+fs.cpSync(extension, output, { recursive: true });
+// Host configuration belongs in the staged copy, leaving the fork usable in Chrome.
+const hostConfiguration = {
+  'overlay-mode.js': [
+    ['export const OVERLAY_MODE = false;', 'export const OVERLAY_MODE = true;'],
+    ['customJavaScript: true,', 'customJavaScript: false,'],
+  ],
+  // Overlay hosts seed the lookup highlight off because their Anki screenshot is
+  // the see-through viewport. SubMiner captures media from mpv, and without the
+  // highlight the sidebar shows nothing for the word being looked up.
+  'setup-state.js': [
+    [
+      'lookupMode: "hover",\n  sourceHighlightEnabled: false,',
+      'lookupMode: "hover",\n  sourceHighlightEnabled: true,',
+    ],
+  ],
+};
+for (const [file, replacements] of Object.entries(hostConfiguration)) {
+  const filePath = path.join(output, file);
+  let text = fs.readFileSync(filePath, 'utf8');
+  for (const [original, replacement] of replacements) {
+    if (!text.includes(original))
+      throw new Error(`Hachidori host configuration changed upstream: ${original}`);
+    text = text.replace(original, replacement);
+  }
+  fs.writeFileSync(filePath, text);
+}
+manifest.permissions = manifest.permissions.filter((permission) => permission !== 'userScripts');
+// A fixed key pins the extension ID, so Hachidori's storage origin (dictionaries,
+// settings, setup state) survives userData path changes. Electron otherwise
+// derives the ID from the unpacked directory path.
+manifest.key =
+  'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA40zTTbhCbrKnDOqj17tosDChP60VXuuMjR0tFFR/kAet21jrYn0JKPzqqNpXMGaJsx1chnuMOgaP0zuOT+gv9DNVV83wGLhQL7cC4LCoGN648WqnE2jYqBPj6hdD5A/N3uPeKFRAHkiC9HKoj2Et6sYZFTqag79QqQkkUz17X2VpjuTGmTP3lvhyEwhJWQKzOU0k6dTn6fKB2ZSqTebpw/JS2G8jf58vHlQOcfLhIiolKCG9DfMuo11aoG1s3AsFmQmw9VVyp4wgyQ80UI4lyBdqboanJxEiZ211+HKWPEgtG6VML1+Uo59qWS3uBU1apxE1+7E51UBEHaTsxajFIQIDAQAB';
+fs.writeFileSync(path.join(output, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
+for (const file of ['LICENSE', 'SOURCE.json', 'README.md']) {
+  fs.copyFileSync(path.join(source, file), path.join(output, file));
+}
+process.stdout.write(`Hachidori ${provenance.revision} staged in ${output}\n`);

scripts/build-hachidori.test.ts

@@ -0,0 +1,53 @@
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import test from 'node:test';
+
+test('Hachidori staging configures Electron without changing the fork source', async () => {
+  const files = ['overlay-mode.js', 'setup-state.js', 'manifest.json'];
+  const source = (file: string) =>
+    new URL(`../vendor/hachidori/extension/${file}`, import.meta.url);
+  const before = files.map((file) => readFileSync(source(file), 'utf8'));
+
+  execFileSync(process.execPath, [
+    fileURLToPath(new URL('./build-hachidori.mjs', import.meta.url)),
+  ]);
+
+  assert.deepEqual(
+    files.map((file) => readFileSync(source(file), 'utf8')),
+    before,
+  );
+  const staged = await import(new URL('../build/hachidori/overlay-mode.js', import.meta.url).href);
+  assert.equal(staged.OVERLAY_MODE, true);
+  assert.equal(staged.HOST_CAPABILITIES.customJavaScript, false);
+  const original = await import(source('overlay-mode.js').href);
+  assert.equal(original.OVERLAY_MODE, false);
+  assert.equal(original.HOST_CAPABILITIES.customJavaScript, true);
+  // SubMiner mines media from mpv, not the overlay viewport, so the lookup
+  // highlight stays on for a first install.
+  const stagedSetup = await import(
+    new URL('../build/hachidori/setup-state.js', import.meta.url).href
+  );
+  assert.equal(stagedSetup.OVERLAY_MODE_OPTIONS.sourceHighlightEnabled, true);
+  const originalSetup = await import(source('setup-state.js').href);
+  assert.equal(originalSetup.OVERLAY_MODE_OPTIONS.sourceHighlightEnabled, false);
+  const manifest = JSON.parse(
+    readFileSync(new URL('../build/hachidori/manifest.json', import.meta.url), 'utf8'),
+  );
+  const originalManifest = JSON.parse(readFileSync(source('manifest.json'), 'utf8'));
+  assert.deepEqual(manifest, {
+    ...originalManifest,
+    permissions: originalManifest.permissions.filter(
+      (permission: string) => permission !== 'userScripts',
+    ),
+    key: manifest.key,
+  });
+  // The ID keys Hachidori's stored dictionaries and settings; changing the key orphans them.
+  const digest = createHash('sha256').update(Buffer.from(manifest.key, 'base64')).digest('hex');
+  const extensionId = [...digest.slice(0, 32)]
+    .map((digit) => String.fromCharCode(97 + parseInt(digit, 16)))
+    .join('');
+  assert.equal(extensionId, 'jpfgmfknblendgepdejlfhocdcnjjelj');
+});

scripts/check-dictionary-backends.cjs

@@ -0,0 +1,139 @@
+// Exercises the real entry point with disposable Linux config and data directories.
+const { app, BrowserWindow, session, shell } = require('electron');
+const fs = require('node:fs');
+const path = require('node:path');
+const assert = require('node:assert/strict');
+if (process.platform !== 'linux')
+  throw new Error('This app-entry smoke requires Linux XDG isolation.');
+const root = process.cwd();
+const backend = process.argv.includes('--backend=yomitan') ? 'yomitan' : 'hachidori';
+const externalYomitanProfile =
+  backend === 'hachidori' && !process.argv.includes('--bundled-yomitan');
+const profile = process.env.SUBMINER_DICTIONARY_SMOKE_DATA;
+assert(
+  profile && fs.existsSync(profile),
+  'Use bun run test:dictionary:electron for profile isolation',
+);
+process.env.XDG_CONFIG_HOME = profile;
+process.env.XDG_DATA_HOME = path.join(profile, 'data');
+fs.mkdirSync(path.join(profile, 'SubMiner'));
+fs.writeFileSync(
+  path.join(profile, 'SubMiner', 'config.json'),
+  JSON.stringify({
+    dictionaryBackend: backend,
+    // External Yomitan profiles must not block Hachidori settings.
+    yomitan: {
+      externalProfilePath: externalYomitanProfile ? path.join(profile, 'external-yomitan') : '',
+    },
+    mpv: { socketPath: path.join(profile, 'missing-mpv.sock') },
+    ankiConnect: { enabled: false },
+    startupWarmups: { lowPowerMode: true },
+    discordPresence: { enabled: false },
+    updates: { enabled: false },
+  }),
+);
+const openedLinks = [];
+shell.openExternal = async (url) => {
+  openedLinks.push(url);
+};
+app.setAppPath(root);
+app.getVersion = () => require(path.join(root, 'package.json')).version;
+process.env.SUBMINER_APP_LOG = path.join(profile, 'app.log');
+app.commandLine.appendSwitch('ozone-platform', 'x11');
+app.commandLine.appendSwitch('disable-gpu');
+app.commandLine.appendSwitch('disable-dev-shm-usage');
+process.argv = [process.execPath, root, '--hachidori', '--log-level', 'debug'];
+require(path.join(root, 'dist/main-entry.js'));
+const deadline = setTimeout(() => {
+  console.error(
+    'FAIL timeout',
+    BrowserWindow.getAllWindows().map((w) => w.webContents.getURL()),
+  );
+  finish(1);
+}, 60000);
+function finish(exitCode) {
+  clearTimeout(deadline);
+  app.exit(exitCode);
+}
+(async () => {
+  await app.whenReady();
+  let window;
+  for (let i = 0; i < 300; i++) {
+    window = BrowserWindow.getAllWindows().find(
+      (w) => w.getTitle().includes('Hachidori') && w.isVisible(),
+    );
+    if (window) break;
+    await new Promise((r) => setTimeout(r, 100));
+  }
+  assert.ok(window, 'Hachidori settings window opens from actual app flag');
+  assert.equal(window.webContents.session, session.fromPartition('persist:hachidori'));
+  assert.match(window.webContents.getURL(), /settings.html/);
+  const status = await window.webContents.executeJavaScript(
+    `chrome.runtime.sendMessage({target:'hoshidicts-offscreen',type:'hd_status',requestId:'app-settings-smoke'})`,
+  );
+  assert.equal(status.ok, true);
+  assert.equal(status.ready, true);
+
+  console.log(
+    'PASS actual --hachidori startup, visible settings, isolated backend session, native engine ready',
+  );
+  app.emit('second-instance', {}, [process.execPath, root, '--yomitan'], root);
+  if (externalYomitanProfile) {
+    const blockedMessage =
+      'Yomitan settings unavailable while using read-only external-profile mode.';
+    for (let i = 0; i < 100; i++) {
+      if (fs.readFileSync(process.env.SUBMINER_APP_LOG, 'utf8').includes(blockedMessage)) break;
+      await new Promise((r) => setTimeout(r, 100));
+    }
+    assert.ok(
+      fs.readFileSync(process.env.SUBMINER_APP_LOG, 'utf8').includes(blockedMessage),
+      '--yomitan is rejected when an external Yomitan profile is configured',
+    );
+    assert.equal(session.defaultSession.extensions.getAllExtensions().length, 0);
+    assert.ok(!BrowserWindow.getAllWindows().some((w) => w.getTitle().includes('Yomitan')));
+    console.log('PASS external Yomitan settings blocked while Hachidori settings remain available');
+  } else {
+    let yomi;
+    for (let i = 0; i < 200; i++) {
+      yomi = BrowserWindow.getAllWindows().find(
+        (w) => w.getTitle().includes('Yomitan') && w.isVisible(),
+      );
+      if (yomi) break;
+      await new Promise((r) => setTimeout(r, 100));
+    }
+    assert.ok(yomi, 'inactive --yomitan settings opens');
+    assert.equal(yomi.webContents.session, session.defaultSession);
+  }
+  assert.equal(window.webContents.session, session.fromPartition('persist:hachidori'));
+  app.emit('second-instance', {}, [process.execPath, root, '--toggle-visible-overlay'], root);
+  let overlay;
+  for (let i = 0; i < 200; i++) {
+    overlay = BrowserWindow.getAllWindows().find((w) =>
+      w.webContents.getURL().includes('/renderer/index.html?'),
+    );
+    if (overlay && !overlay.webContents.isLoading()) break;
+    await new Promise((r) => setTimeout(r, 100));
+  }
+  assert.ok(overlay, 'actual overlay initialized');
+  assert.equal(
+    overlay.webContents.session,
+    backend === 'hachidori' ? session.fromPartition('persist:hachidori') : session.defaultSession,
+  );
+  const requestLink = (url) =>
+    overlay.webContents.executeJavaScript(`new Promise((resolve,reject)=>{
+  const timer=setTimeout(()=>reject(Error('No external link acknowledgment')),3000);
+  window.addEventListener('hachidori-open-external-result',e=>{clearTimeout(timer);resolve(e.detail);},{once:true});
+  window.dispatchEvent(new CustomEvent('hachidori-open-external',{detail:{requestId:'smoke-link',url:${JSON.stringify(url)}}}));
+ })`);
+  assert.equal((await requestLink('file:///tmp/private')).ok, false);
+  assert.equal((await requestLink('https://example.com/word')).ok, backend === 'hachidori');
+  assert.deepEqual(openedLinks, backend === 'hachidori' ? ['https://example.com/word'] : []);
+  console.log(
+    `PASS ${backend} overlay session, independent settings windows, real preload external link bridge`,
+  );
+
+  finish(0);
+})().catch((error) => {
+  console.error(error);
+  finish(1);
+});

scripts/check-hachidori-parser.cjs

@@ -0,0 +1,239 @@
+// Run after bun run build. This uses a temporary profile and small local ZIPs.
+const { app, BrowserWindow, protocol, session } = require('electron');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const http = require('node:http');
+const root = path.resolve(__dirname, '..');
+const parser = require(path.join(root, 'dist/core/services/tokenizer/yomitan-parser-runtime.js'));
+const { writeStoredZip } = require(path.join(root, 'dist/shared/stored-zip.js'));
+const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-hachidori-parser-'));
+
+protocol.registerSchemesAsPrivileged([
+  {
+    scheme: 'chrome-extension',
+    privileges: {
+      standard: true,
+      secure: true,
+      supportFetchAPI: true,
+      corsEnabled: true,
+      bypassCSP: true,
+    },
+  },
+]);
+app.setPath('userData', profile);
+app.setAppPath(root);
+app.disableHardwareAcceleration();
+app.on('window-all-closed', () => {});
+const deadline = setTimeout(() => {
+  console.error('Hachidori parser verification timed out', profile);
+  app.exit(1);
+}, 120_000);
+
+function fixture(name, title, bankName, entries, frequencyMode = 'rank-based') {
+  const zipPath = path.join(profile, name + '.zip');
+  writeStoredZip(zipPath, [
+    {
+      name: 'index.json',
+      data: Buffer.from(JSON.stringify({ title, revision: '1', format: 3, frequencyMode })),
+    },
+    { name: bankName, data: Buffer.from(JSON.stringify(entries)) },
+  ]);
+  return zipPath;
+}
+
+app
+  .whenReady()
+  .then(async () => {
+    const targetSession = session.fromPartition('persist:hachidori-check');
+    const extension = await targetSession.extensions.loadExtension(
+      path.join(root, 'build/hachidori'),
+      { allowFileAccess: true },
+    );
+    // Keep a host window alive while the dictionary importer opens/closes its
+    // temporary settings windows, matching the running app's window lifecycle.
+    const host = new BrowserWindow({
+      show: false,
+      webPreferences: { session: targetSession, contextIsolation: true, nodeIntegration: false },
+    });
+    await host.loadURL(`chrome-extension://${extension.id}/settings.html`);
+    let parserWindow = null;
+    let readyPromise = null;
+    let initPromise = null;
+    const deps = {
+      getYomitanExt: () => extension,
+      getYomitanSession: () => targetSession,
+      getYomitanParserWindow: () => parserWindow,
+      setYomitanParserWindow: (value) => {
+        parserWindow = value;
+      },
+      getYomitanParserReadyPromise: () => readyPromise,
+      setYomitanParserReadyPromise: (value) => {
+        readyPromise = value;
+      },
+      getYomitanParserInitPromise: () => initPromise,
+      setYomitanParserInitPromise: (value) => {
+        initPromise = value;
+      },
+    };
+    const errors = [];
+    const logger = {
+      error: (...args) => {
+        errors.push(args);
+        console.error(...args);
+      },
+    };
+    assert.deepEqual(await parser.getYomitanDictionaryInfo(deps, logger), []);
+    assert.equal(
+      await parser.syncYomitanDefaultAnkiServer('http://127.0.0.1:18766', deps, logger, {
+        forceOverride: true,
+        deck: 'Test Mining',
+      }),
+      true,
+    );
+    // Exercise discovery and revisioned options writes in the real extension,
+    // without creating a note or changing the user's Anki collection.
+    const fields = ['Term', 'Reading', 'Definition', 'Context', 'Pronunciation', 'Image'];
+    const metadataServer = http.createServer((request, response) => {
+      let body = '';
+      request.on('data', (chunk) => {
+        body += chunk;
+      });
+      request.on('end', () => {
+        const message = JSON.parse(body);
+        assert.equal(message.action, 'multi');
+        const result = message.params.actions.map(({ action }) => ({
+          result:
+            action === 'deckNames'
+              ? ['Test Mining']
+              : action === 'modelNames'
+                ? ['Custom Japanese']
+                : fields,
+          error: null,
+        }));
+        response.setHeader('Content-Type', 'application/json');
+        response.end(JSON.stringify({ result, error: null }));
+      });
+    });
+    await new Promise((resolve) => metadataServer.listen(0, '127.0.0.1', resolve));
+    try {
+      const url = `http://127.0.0.1:${metadataServer.address().port}`;
+      const ankiConfig = {
+        tags: ['SubMiner', 'Autofill'],
+        fields: {
+          word: 'Term',
+          sentence: 'Context',
+          wordAudio: 'Pronunciation',
+          image: 'Image',
+        },
+      };
+      assert.equal(
+        await parser.syncYomitanDefaultAnkiServer(url, deps, logger, {
+          forceOverride: true,
+          deck: 'Test Mining',
+          ankiConfig,
+        }),
+        true,
+      );
+      const anki = await parserWindow.webContents.executeJavaScript(
+        `(async () => (await chrome.storage.local.get('options')).options.anki)()`,
+      );
+      assert.equal(anki.model, 'Custom Japanese');
+      assert.equal(anki.deck, 'Test Mining');
+      assert.deepEqual(anki.tags, [...ankiConfig.tags, 'hachidori']);
+      assert.equal(anki.fieldTemplates.Term.value, '{expression}');
+      assert.equal(anki.fieldTemplates.Pronunciation.value, '{audio}');
+      assert.equal(anki.fieldTemplates.Context.value, '{sentence}');
+      assert.deepEqual(anki.templates[0].fieldTemplates, anki.fieldTemplates);
+      console.log('Hachidori Anki auto-population passed with an empty dictionary library');
+    } finally {
+      await new Promise((resolve) => metadataServer.close(resolve));
+    }
+    const archives = [
+      fixture('terms', 'SubMiner Test Terms', 'term_bank_1.json', [
+        ['食べる', 'たべる', '', 'v1', 0, ['to eat'], 1, ''],
+      ]),
+      fixture('names', 'SubMiner Character Dictionary (AniList 1)', 'term_bank_1.json', [
+        ['ミナト', 'みなと', '', 'n', 0, ['name'], 1, ''],
+      ]),
+      fixture('frequency', 'SubMiner Test Frequency', 'term_meta_bank_1.json', [
+        ['食べる', 'freq', { reading: 'たべる', frequency: 42 }],
+        ['頻度だけ', 'freq', { reading: 'ひんどだけ', frequency: 120 }],
+        ['頻度だけ', 'freq', { reading: 'べつのよみ', frequency: 250 }],
+        ['頻度だけ', 'freq', 17],
+      ]),
+      fixture(
+        'occurrences',
+        'SubMiner Test Occurrences',
+        'term_meta_bank_1.json',
+        [['頻度だけ', 'freq', 9000]],
+        'occurrence-based',
+      ),
+    ];
+    for (const archive of archives)
+      assert.equal(await parser.importYomitanDictionaryFromZip(archive, deps, logger), true);
+    const dictionaries = await parser.getYomitanDictionaryInfo(deps, logger);
+    assert.equal(dictionaries.length, 4);
+    parser.clearYomitanParserCachesForWindow(parserWindow);
+    const tokens = await parser.requestYomitanScanTokens('ミナト 食べた', deps, logger, {
+      includeNameMatchMetadata: true,
+      currentCharacterDictionaryMediaId: 1,
+    });
+    assert.equal(tokens[0].isNameMatch, true);
+    assert.equal(tokens[1].headword, '食べる');
+    assert.equal(tokens[1].reading, 'たべた');
+    assert.equal(tokens[1].startPos, 4);
+    assert.equal(tokens[1].endPos, 7);
+    assert.equal(tokens[1].frequencyRank, 42);
+    assert.deepEqual(tokens[1].wordClasses, ['v1']);
+    const exact = await parser.requestYomitanTermFrequencies(
+      [{ term: '食べる', reading: 'たべる' }],
+      deps,
+      logger,
+    );
+    assert.equal(exact.length, 1);
+    assert.equal(exact[0].frequency, 42);
+    assert.equal(exact[0].reading, 'たべる');
+    assert.equal(exact[0].hasReading, false);
+    const otherReading = await parser.requestYomitanTermFrequencies(
+      [{ term: '食べる', reading: 'べつのよみ' }],
+      deps,
+      logger,
+    );
+    // The shared frequency pipeline retries a missing reading as a term-only query.
+    assert.equal(otherReading[0]?.frequency, 42);
+    const unmatched = await parser.requestYomitanTermFrequencies(
+      [{ term: '頻度だけ', reading: null }],
+      deps,
+      logger,
+    );
+    assert.deepEqual(unmatched, []);
+    assert.equal(await parser.getYomitanCurrentAnkiDeckName(deps, logger), 'Test Mining');
+    assert.equal((await targetSession.extensions.getAllExtensions()).length, 1);
+    assert.equal(
+      await parser.syncYomitanDefaultAnkiServer('http://127.0.0.1:8765', deps, logger),
+      true,
+    );
+    const directSettings = await parser.getYomitanSettingsFull(deps, logger);
+    assert.equal(directSettings.profiles[0].options.anki.server, 'http://127.0.0.1:8765');
+    const proxyState = await host.webContents.executeJavaScript(
+      "chrome.storage.local.get('subminerAnkiProxyUrl')",
+    );
+    assert.equal(proxyState.subminerAnkiProxyUrl, null);
+    for (const entry of await parser.getYomitanDictionaryInfo(deps, logger)) {
+      assert.equal(await parser.deleteYomitanDictionaryByTitle(entry.title, deps, logger), true);
+    }
+    assert.deepEqual(await parser.getYomitanDictionaryInfo(deps, logger), []);
+    assert.equal(errors.length, 0);
+    console.log(
+      'PASS Hachidori native import, scanner, character names, term-entry API frequencies, settings and removal',
+    );
+    clearTimeout(deadline);
+    app.exit(0);
+  })
+  .catch((error) => {
+    console.error(error);
+    clearTimeout(deadline);
+    app.exit(1);
+  });

scripts/hachidori-anki-audio.test.ts

@@ -0,0 +1,137 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import '../vendor/hachidori/extension/reader-options.js';
+import { createAnkiGateway } from '../vendor/hachidori/extension/anki.js';
+import { createAnkiWorkerService } from '../vendor/hachidori/extension/anki-worker.js';
+
+async function mine({ proxy = true, audioFails = false } = {}) {
+  const events: string[] = [];
+  const filename = `hachidori_${'a'.repeat(64)}.mp3`;
+  let fields: Record<string, string> = {};
+  let initialAudio = '';
+  let stored = false;
+  const gateway = createAnkiGateway({
+    readSubminerProxyUrl: async () => (proxy ? 'http://127.0.0.1:8766' : null),
+    fetch: async (_url: string, options: RequestInit) => {
+      const { action, params } = JSON.parse(String(options.body));
+      events.push(action);
+      let result: unknown;
+      switch (action) {
+        case 'multi':
+          result = [
+            { result: ['Default'], error: null },
+            { result: ['Basic'], error: null },
+            { result: ['Expression', 'ExpressionAudio'], error: null },
+          ];
+          break;
+        case 'canAddNotes':
+          result = [true];
+          break;
+        case 'canAddNotesWithErrorDetail':
+          result = [{ canAdd: true, error: null }];
+          break;
+        case 'getMediaFilesNames':
+          result = stored ? [filename] : [];
+          break;
+        case 'storeMediaFile':
+          stored = true;
+          result = filename;
+          break;
+        case 'addNote':
+          fields = { ...params.note.fields };
+          initialAudio = fields.ExpressionAudio ?? '';
+          if (initialAudio) assert.ok(stored, 'audio must exist before the note references it');
+          result = 123;
+          break;
+        case 'notesInfo':
+          result = [
+            {
+              noteId: 123,
+              fields: Object.fromEntries(
+                Object.entries(fields).map(([key, value]) => [key, { value }]),
+              ),
+            },
+          ];
+          break;
+        case 'updateNoteFields':
+          Object.assign(fields, params.note.fields);
+          result = null;
+          break;
+        default:
+          throw new Error(`Unexpected action: ${action}`);
+      }
+      return Response.json({ result, error: null });
+    },
+  });
+  const service = createAnkiWorkerService({
+    gateway,
+    readOptions: async () => ({
+      anki: {
+        url: 'http://127.0.0.1:8766',
+        apiKey: '',
+        templates: [
+          {
+            id: 'default',
+            name: 'Default',
+            deck: 'Default',
+            model: 'Basic',
+            tags: [],
+            fields: {},
+            duplicateScope: 'model',
+            duplicateBehavior: 'prevent',
+            captureScreenshot: false,
+            fieldTemplates: {
+              Expression: { value: '{expression}', overwriteMode: 'overwrite' },
+              ExpressionAudio: { value: '{audio}', overwriteMode: 'overwrite' },
+            },
+          },
+        ],
+      },
+      audioSources: [
+        { id: 'test', enabled: true, type: 'custom', url: 'https://example.test/{term}' },
+      ],
+      experimental: {},
+    }),
+    readDictionaries: async () => [],
+    engine: async () => ({ ready: true, loading: false, generation: 1 }),
+    offscreen: async (message: { type: string; audio?: string }) => {
+      if (message.type === 'hd_anki_audio') {
+        events.push('pronunciation');
+        if (audioFails) throw new Error('No pronunciation available');
+        return { filename, data: 'YXVkaW8=' };
+      }
+      return { fields: { Expression: '猫', ExpressionAudio: message.audio ?? '' }, media: [] };
+    },
+    duplicateIndex: { source: async () => null, recordWrite: async () => {} },
+  });
+  const status = await service.status();
+  assert.equal(status.available, true, status.error);
+  const result = await service.submit({
+    configKey: status.configKey,
+    generation: 1,
+    term: { expression: '猫', reading: 'ねこ' },
+  });
+  return { result, initialAudio, fields, events, filename };
+}
+
+test('SubMiner receives pronunciation in the initial Hachidori note, before enrichment starts', async () => {
+  const value = await mine();
+  assert.equal(value.result.state, 'added');
+  assert.equal(value.initialAudio, `[sound:${value.filename}]`);
+  assert.equal(value.events.filter((event) => event === 'pronunciation').length, 1);
+});
+
+test('direct Hachidori keeps deferred pronunciation', async () => {
+  const value = await mine({ proxy: false });
+  assert.equal(value.initialAudio, '');
+  assert.equal(value.fields.ExpressionAudio, `[sound:${value.filename}]`);
+});
+
+test('unavailable pronunciation remains a warning without a late audio write', async () => {
+  const value = await mine({ audioFails: true });
+  assert.equal(value.result.state, 'added');
+  assert.match(value.result.warnings.join(' '), /No pronunciation available/);
+  assert.equal(value.initialAudio, '');
+  assert.equal(value.events.filter((event) => event === 'pronunciation').length, 1);
+  assert.equal(value.events.includes('updateNoteFields'), false);
+});

scripts/hachidori-host.test.ts

@@ -0,0 +1,132 @@
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import { runInNewContext } from 'node:vm';
+import test from 'node:test';
+import { CHARACTER_DICTIONARY_TITLE_PREFIX } from '../src/core/services/tokenizer/character-dictionary-title';
+import { createAnkiGateway } from '../vendor/hachidori/extension/anki.js';
+
+const bridge = readFileSync(
+  new URL('../vendor/hachidori/extension/subminer-host.js', import.meta.url),
+  'utf8',
+);
+
+function run(code: string) {
+  runInNewContext(`${bridge}\n${code}`, {
+    window: new EventTarget(),
+    EventTarget,
+    CustomEvent,
+    assert,
+    characterPrefix: CHARACTER_DICTIONARY_TITLE_PREFIX,
+    KeyboardEvent: class {
+      constructor(_type: string, options: KeyboardEventInit = {}) {
+        Object.assign(this, options);
+      }
+    },
+  });
+}
+
+test('Hachidori marks popup state independently from successful lookups', () => {
+  run(`
+    const events = [];
+    for (const name of ['yomitan-popup-shown', 'yomitan-popup-hidden', 'subminer-yomitan-lookup']) {
+      window.addEventListener(name, () => events.push(name));
+    }
+    const attributes = new Map();
+    const host = { setAttribute: (name, value) => attributes.set(name, value) };
+    SubMinerHachidori.markHost(host, true);
+    assert.equal(attributes.get('data-subminer-yomitan-popup-visible'), 'true');
+    assert.equal(events.join(','), '');
+    SubMinerHachidori.lookup();
+    SubMinerHachidori.markHost(host, false);
+    assert.equal(attributes.get('data-subminer-yomitan-popup-visible'), 'false');
+    assert.equal(events.join(','), 'subminer-yomitan-lookup');
+  `);
+});
+
+test('Hachidori promotes character glossaries and results without overriding linguistic rank', () => {
+  run(`
+    const result = (id, matched, dictionary, options = {}) => ({
+      matched, deinflected: matched, trace: [], preprocessorSteps: 0,
+      term: { expression: matched, reading: 'reading', glossaries: [{ dictionary }] }, ...options, id,
+    });
+    const character = characterPrefix + ' - Current show';
+    const longer = result('longer', '花子さん', 'General');
+    const normal = result('normal', '花子', 'General');
+    const person = result('person', '花子', character);
+    const shorter = result('shorter', '花', character);
+    const output = SubMinerHachidori.prioritizeCharacterResults([longer, normal, person, shorter]);
+    assert.equal(output.map(entry => entry.id).join(','), 'longer,person,normal,shorter');
+    const merged = result('merged', '花子', 'General');
+    merged.term.glossaries.push({ dictionary: character }, { dictionary: 'Second general' });
+    const [promoted] = SubMinerHachidori.prioritizeCharacterResults([merged]);
+    assert.equal(promoted.term.glossaries.map(g => g.dictionary).join(','), character + ',General,Second general');
+    assert.equal(merged.term.glossaries[0].dictionary, 'General');
+    for (const change of [{ preprocessorSteps: 1 }, { trace: [{}] }, { deinflected: '別の語' }]) {
+      const transformed = { ...person, ...change };
+      assert.equal(SubMinerHachidori.prioritizeCharacterResults([normal, transformed])[0].id, 'normal');
+    }
+    const preferredReading = { ...normal, term: { ...normal.term, reading: 'preferred' } };
+    assert.equal(SubMinerHachidori.prioritizeCharacterResults([person, preferredReading], {primaryReading: 'preferred'})[0].id, 'normal');
+    const aliased = result('aliased', '花子', 'Imported character data');
+    assert.equal(SubMinerHachidori.prioritizeCharacterResults([normal, aliased], {}, [{title: 'Imported character data', displayName: character}])[0].id, 'aliased');
+  `);
+});
+
+test('Hachidori sends private mining metadata only to the configured SubMiner proxy', async () => {
+  const requests: Array<{ url: string; body: string }> = [];
+  const proxy = 'http://127.0.0.1:8766';
+  const gateway = createAnkiGateway({
+    readSubminerProxyUrl: async () => proxy,
+    fetch: async (url: string, options: RequestInit) => {
+      assert.equal(typeof options.body, 'string');
+      requests.push({ url, body: String(options.body) });
+      return Response.json({ result: 123, error: null });
+    },
+  });
+  const params = {
+    note: { fields: { Expression: '花子' } },
+    subminerDuplicateNoteIds: [456],
+    subminerEnrich: true,
+  };
+  await gateway.invoke('addNote', params, '', 1000, proxy);
+  await gateway.invoke('addNote', params, '', 1000, 'http://127.0.0.1:8765');
+  assert.deepEqual(
+    requests.map((request) => JSON.parse(request.body).params),
+    [params, { note: params.note }],
+  );
+  assert.equal(params.subminerEnrich, true);
+  const direct = createAnkiGateway({
+    readSubminerProxyUrl: async () => null,
+    fetch: async (_url: string, options: RequestInit) => {
+      assert.deepEqual(JSON.parse(String(options.body)).params, { note: params.note });
+      return Response.json({ result: null, error: null });
+    },
+  });
+  await direct.invoke('updateNoteFields', params, '', 1000, proxy);
+});
+
+test('Hachidori routes host commands, validates keyboard input and disconnects', () => {
+  run(`
+    const calls = [];
+    const disconnect = SubMinerHachidori.connect({
+      hide: () => calls.push('hide'), clear: () => calls.push('clear'),
+      action: name => calls.push(name), cycleAudio: direction => calls.push(direction),
+      scroll: (x, y) => calls.push(x + ':' + y),
+      keydown: event => calls.push(event.key + ':' + event.ctrlKey + ':' + event.shiftKey),
+    });
+    const send = detail => window.dispatchEvent(new CustomEvent('subminer-yomitan-popup-command', {detail}));
+    send(null);
+    send({ type: 'forwardKeyDown', key: 12, modifiers: [] });
+    send({ type: 'mineSelected' });
+    send({ type: 'playCurrentAudio' });
+    send({ type: 'scanSelectedText' });
+    send({ type: 'cycleAudioSource', direction: -1 });
+    send({ type: 'scrollBy', deltaX: Infinity, deltaY: 40 });
+    send({ type: 'forwardKeyDown', key: 'j', code: 'KeyJ', modifiers: ['ctrl', 'shift'] });
+    send({ type: 'setVisible', visible: false });
+    send({ type: 'clearActiveTextSource' });
+    disconnect();
+    send({ type: 'mineSelected' });
+    assert.equal(calls.join(','), 'addNote,playAudio,scanSelectedText,-1,0:40,j:true:true,hide,clear');
+  `);
+});

scripts/package-audit.cjs

@@ -26,6 +26,14 @@ const REQUIRED_APP_FILES = [
   ]),
 ];
 const REQUIRED_RESOURCES = [
+  'hachidori/manifest.json',
+  'hachidori/settings.html',
+  'hachidori/subminer-host.js',
+  'hachidori/vendor/hoshidicts.wasm',
+  'hachidori/vendor/hoshidicts-threaded.wasm',
+  'hachidori/vendor/hoshidicts-threaded-idbfs.wasm',
+  'hachidori/LICENSE',
+  'hachidori/SOURCE.json',
   'yomitan/manifest.json',
   'yomitan/data/fonts/kanji-stroke-orders.ttf',
   'yomitan/fonts/NotoSansJP-Regular.ttf',

scripts/run-dictionary-settings-smoke.mjs

@@ -0,0 +1,26 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { createRequire } from 'node:module';
+import { fileURLToPath } from 'node:url';
+
+const require = createRequire(import.meta.url);
+const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-hachi-settings-'));
+const env = { ...process.env, SUBMINER_DICTIONARY_SMOKE_DATA: profile };
+delete env.ELECTRON_RUN_AS_NODE;
+try {
+  const result = spawnSync(
+    require('electron'),
+    [
+      '--no-sandbox',
+      fileURLToPath(new URL('./check-dictionary-backends.cjs', import.meta.url)),
+      ...process.argv.slice(2),
+    ],
+    { env, stdio: 'inherit', timeout: 75_000 },
+  );
+  if (result.error) throw result.error;
+  process.exitCode = result.status ?? 1;
+} finally {
+  fs.rmSync(profile, { recursive: true, force: true, maxRetries: 3 });
+}

scripts/smoke-package.cjs

@@ -22,39 +22,6 @@ const timeout = setTimeout(() => {
   app.exit(1);
 }, 60_000);
 
-const STATIC_TYPES = {
-  '.html': 'text/html',
-  '.js': 'text/javascript',
-  '.css': 'text/css',
-  '.png': 'image/png',
-  '.svg': 'image/svg+xml',
-  '.woff2': 'font/woff2',
-  '.ttf': 'font/ttf',
-  '.json': 'application/json',
-};
-
-// The stats dashboard is served by the stats HTTP server in the app, so load it
-// over loopback HTTP from the packaged stats/dist and treat missing static
-// assets as failures. API routes are not part of this smoke and may 404.
-function serveStatsDist(root, failedRequests) {
-  const server = http.createServer((req, res) => {
-    const pathname = new URL(req.url, 'http://127.0.0.1').pathname;
-    const relative = pathname === '/' ? 'index.html' : pathname.slice(1);
-    try {
-      const body = fs.readFileSync(path.join(root, relative));
-      res.writeHead(200, {
-        'Content-Type': STATIC_TYPES[path.extname(relative)] ?? 'application/octet-stream',
-      });
-      res.end(body);
-    } catch {
-      if (!pathname.startsWith('/api/')) failedRequests.push(`${req.url}: missing static asset`);
-      res.writeHead(404).end();
-    }
-  });
-  server.listen(0, '127.0.0.1');
-  return server;
-}
-
 async function smoke() {
   await app.whenReady();
   const packagedRequire = createRequire(path.join(archive, 'package.json'));
@@ -83,6 +50,10 @@ async function smoke() {
     { allowFileAccess: true },
   );
   assert(extension.id, 'Yomitan extension failed to load');
+  const hachidori = await session
+    .fromPartition('persist:hachidori')
+    .extensions.loadExtension(path.join(resources, 'hachidori'), { allowFileAccess: true });
+  assert(hachidori.id, 'Hachidori extension failed to load');
   const failedRequests = [];
   session.defaultSession.webRequest.onErrorOccurred(
     { urls: ['file://*/*', 'http://127.0.0.1/*'] },
@@ -93,9 +64,7 @@ async function smoke() {
         failedRequests.push(`${details.url}: ${details.error}`);
     },
   );
-  const statsServer = serveStatsDist(path.join(archive, 'stats', 'dist'), failedRequests);
-  await once(statsServer, 'listening');
-  for (const ui of ['renderer', 'settings', 'syncui', 'stats']) {
+  for (const ui of ['renderer', 'settings', 'syncui']) {
     const win = new BrowserWindow({
       show: false,
       webPreferences: {
@@ -104,25 +73,52 @@ async function smoke() {
       },
     });
     try {
-      if (ui === 'stats') {
-        await win.loadURL(`http://127.0.0.1:${statsServer.address().port}/`);
-        // Let in-flight font requests settle before the window goes away.
-        await win.webContents.executeJavaScript('document.fonts.ready.then(() => true)');
-      } else {
-        await win.loadFile(path.join(archive, `dist/${ui}/index.html`));
-        const loaded = await win.webContents.executeJavaScript(
-          `document.fonts.load('400 16px "M PLUS 1"', '日本語').then(fonts => fonts.length > 0 && fonts.every(font => font.status === 'loaded'))`,
-        );
-        assert(loaded, `${ui}: shared Japanese font failed to load`);
-      }
+      await win.loadFile(path.join(archive, `dist/${ui}/index.html`));
+      const loaded = await win.webContents.executeJavaScript(
+        `document.fonts.load('400 16px "M PLUS 1"', '日本語').then(fonts => fonts.length > 0 && fonts.every(font => font.status === 'loaded'))`,
+      );
+      assert(loaded, `${ui}: shared Japanese font failed to load`);
     } finally {
       win.destroy();
     }
   }
-  statsServer.close();
+  // The stats dashboard uses HTTP for both assets and API requests in the app.
+  const { ImmersionTrackerService } = packagedRequire(
+    './dist/core/services/immersion-tracker-service.js',
+  );
+  const { createStatsApp, startNodeHttpServer } = packagedRequire(
+    './dist/core/services/stats-server.js',
+  );
+  const tracker = new ImmersionTrackerService({ dbPath: path.join(isolatedData, 'stats.db') });
+  const statsConfig = { port: 0, staticDir: path.join(archive, 'stats/dist'), tracker };
+  let statsHttp;
+  const statsServer = await startNodeHttpServer(
+    createStatsApp(tracker, statsConfig),
+    statsConfig,
+    (listener) => (statsHttp = http.createServer(listener)),
+  );
+  const statsWindow = new BrowserWindow({ show: false });
+  try {
+    const url = `http://127.0.0.1:${statsHttp.address().port}`;
+    session.defaultSession.webRequest.onCompleted({ urls: [`${url}/*`] }, (details) => {
+      if (details.statusCode >= 400) failedRequests.push(`${details.url}: ${details.statusCode}`);
+    });
+    await statsWindow.loadURL(url);
+    // Let in-flight font requests settle before the window goes away.
+    await statsWindow.webContents.executeJavaScript('document.fonts.ready.then(() => true)');
+    for (const endpoint of ['overview', 'sessions']) {
+      const response = await fetch(`${url}/api/stats/${endpoint}`);
+      assert.equal(response.status, 200, `Stats ${endpoint} request failed`);
+      await response.json();
+    }
+  } finally {
+    statsWindow.destroy();
+    await statsServer.close();
+    tracker.destroy();
+  }
   assert.deepEqual(failedRequests, [], 'Packaged UI resources failed to load');
   console.log(
-    'Package smoke passed: SQLite, platform FFI, texthooker, Yomitan loading, UI pages, shared Japanese font.',
+    'Package smoke passed: SQLite, platform FFI, texthooker, both dictionary extensions, UI pages, stats HTTP, shared Japanese font.',
   );
 }