test(tokenizer): add golden-file regression corpus with 11 fixtures (#157)

.prettierignore

@@ -4,3 +4,4 @@ release
 coverage
 vendor
 *.log
+src/core/services/tokenizer/__fixtures__/golden/*.json

changes/golden-tokenizer-corpus.md

@@ -0,0 +1,7 @@
+type: internal
+area: tokenizer
+
+- Added a golden-file regression corpus for the tokenizer/annotation pipeline: recorded Yomitan backend responses and MeCab tokens replay through the real tokenizeSubtitle pipeline in bun tests without Electron or dictionaries.
+- Added `record-tokenizer-fixture:electron` script to capture new fixtures from a live Yomitan/MeCab session, with flags for known words, JLPT levels, and annotation toggles.
+- Seeded eleven fixtures covering the #147–#156 regression classes (grammar-helper suppression, lexical くれる, kanji non-independent nouns, N+1 targeting, reading collisions, unparsed runs, ordinal/honorific prefixes).
+- Added `compare-yomitan-api:electron` script that diffs SubMiner tokenization against a stock Yomitan instance via the yomitan-api bridge (segmentation, readings, headword forms).

package.json

@@ -13,6 +13,8 @@
     "get-frequency:electron": "bun run build:yomitan && bun build scripts/get_frequency.ts --format=cjs --target=node --outfile dist/scripts/get_frequency.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/get_frequency.js --pretty --color-top-x 10000 --yomitan-user-data ~/.config/SubMiner --colorized-line",
     "test-yomitan-parser": "bun run scripts/test-yomitan-parser.ts",
     "test-yomitan-parser:electron": "bun run build:yomitan && bun build scripts/test-yomitan-parser.ts --format=cjs --target=node --outfile dist/scripts/test-yomitan-parser.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/test-yomitan-parser.js",
+    "record-tokenizer-fixture:electron": "bun run build:yomitan && bun build scripts/record-tokenizer-fixture.ts --format=cjs --target=node --outfile dist/scripts/record-tokenizer-fixture.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/record-tokenizer-fixture.js",
+    "compare-yomitan-api:electron": "bun run build:yomitan && bun build scripts/compare-yomitan-api.ts --format=cjs --target=node --outfile dist/scripts/compare-yomitan-api.js --external electron && env -u ELECTRON_RUN_AS_NODE electron dist/scripts/compare-yomitan-api.js",
     "build:yomitan": "bun scripts/build-yomitan.mjs",
     "build:assets": "bun scripts/prepare-build-assets.mjs",
     "build:launcher": "bun build ./launcher/main.ts --target=bun --packages=bundle --banner='#!/usr/bin/env bun' --outfile=dist/launcher/subminer",

scripts/compare-yomitan-api.test.ts

@@ -0,0 +1,26 @@
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import test from 'node:test';
+
+const source = readFileSync('scripts/compare-yomitan-api.ts', 'utf8');
+
+test('creates the configured Yomitan user-data directory before Electron uses it', () => {
+  const setPathIndex = source.indexOf(
+    "electronModule.app.setPath('userData', options.yomitanUserDataPath)",
+  );
+  assert.notEqual(setPathIndex, -1);
+
+  const mkdirIndex = source.lastIndexOf(
+    'fs.mkdirSync(options.yomitanUserDataPath, { recursive: true })',
+    setPathIndex,
+  );
+  assert.ok(mkdirIndex > -1, 'must create the user-data directory');
+  assert.ok(mkdirIndex < setPathIndex, 'must create the directory before app.setPath');
+});
+
+test('lets output drain by using exitCode in the top-level completion handlers', () => {
+  const completionHandlers = source.slice(source.lastIndexOf('\nmain()'));
+  assert.match(completionHandlers, /process\.exitCode = process\.exitCode \?\? 0/);
+  assert.match(completionHandlers, /process\.exitCode = 1/);
+  assert.doesNotMatch(completionHandlers, /process\.exit\(/);
+});

scripts/compare-yomitan-api.ts

@@ -0,0 +1,589 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import process from 'node:process';
+
+import { createTokenizerDepsRuntime, tokenizeSubtitle } from '../src/core/services/tokenizer.js';
+import { selectYomitanParseTokens } from '../src/core/services/tokenizer/parser-selection-stage.js';
+import { enrichTokensWithMecabPos1 } from '../src/core/services/tokenizer/parser-enrichment-stage.js';
+import { resolveYomitanExtensionPath as resolveBuiltYomitanExtensionPath } from '../src/core/services/yomitan-extension-paths.js';
+import { MecabTokenizer } from '../src/mecab-tokenizer.js';
+import type { MergedToken } from '../src/types.js';
+
+import { fetchYomitanApi } from './yomitan-api-request.js';
+
+// Compares SubMiner's full tokenization pipeline against a stock Yomitan
+// instance reached through the yomitan-api native-messaging bridge
+// (https://github.com/yomidevs/yomitan-api, default http://127.0.0.1:19633).
+// The API response is raw parseText output; it is mapped through SubMiner's
+// own selectYomitanParseTokens so both sides share identical mapping
+// semantics and the diff isolates real segmentation/headword divergence.
+
+const DEFAULT_YOMITAN_USER_DATA_PATH = path.join(os.homedir(), '.config', 'SubMiner');
+const DEFAULT_API_URL = 'http://127.0.0.1:19633';
+const DEFAULT_SCAN_LENGTH = 40;
+const FIXTURE_DIR = path.join('src', 'core', 'services', 'tokenizer', '__fixtures__', 'golden');
+
+interface CliOptions {
+  sentences: string[];
+  fromFixtures: boolean;
+  apiUrl: string;
+  scanLength: number;
+  emitJson: boolean;
+  nameMatch: boolean;
+  yomitanExtensionPath?: string;
+  yomitanUserDataPath: string;
+  mecabCommand?: string;
+  mecabDictionaryPath?: string;
+}
+
+function printUsage(): void {
+  process.stdout.write(`Usage:
+  bun run compare-yomitan-api:electron -- [options] [sentence ...]
+
+  Compares SubMiner tokenization against a stock Yomitan instance via the
+  yomitan-api bridge. Without sentences, compares every golden-corpus fixture
+  text plus any --file lines.
+
+  --from-fixtures            Include golden-corpus fixture texts (default when
+                             no sentences are given).
+  --file <path>              Read additional sentences, one per line.
+  --api-url <url>            yomitan-api base URL (default: ${DEFAULT_API_URL}).
+  --scan-length <n>          Scan length for the API request (default: ${DEFAULT_SCAN_LENGTH}).
+  --name-match               Keep SubMiner's character-name pre-pass enabled.
+                             Off by default: stock Yomitan has no name regions,
+                             so it would only add noise to the diff.
+  --json                     Emit machine-readable JSON instead of a report.
+  --yomitan-extension <path> Path to built Yomitan extension directory.
+  --yomitan-user-data <path> Electron userData directory (default: ~/.config/SubMiner).
+  --mecab-command <path>     MeCab binary (default: mecab).
+  --mecab-dictionary <path>  MeCab dictionary directory.
+  -h, --help                 Show usage.
+
+  Exits 1 when any sentence diverges, 0 when everything matches.
+`);
+}
+
+function requireValue(args: string[], flag: string): string {
+  const next = args.shift();
+  if (!next) {
+    throw new Error(`Missing value for ${flag}`);
+  }
+  return next;
+}
+
+function parseCliArgs(argv: string[]): CliOptions {
+  const args = [...argv];
+  const sentences: string[] = [];
+  let fromFixtures = false;
+  let apiUrl = DEFAULT_API_URL;
+  let scanLength = DEFAULT_SCAN_LENGTH;
+  let emitJson = false;
+  let nameMatch = false;
+  let yomitanExtensionPath: string | undefined;
+  let yomitanUserDataPath = DEFAULT_YOMITAN_USER_DATA_PATH;
+  let mecabCommand: string | undefined;
+  let mecabDictionaryPath: string | undefined;
+
+  while (args.length > 0) {
+    const arg = args.shift();
+    if (!arg) break;
+
+    if (arg === '--help' || arg === '-h') {
+      printUsage();
+      process.exit(0);
+    }
+    if (arg === '--from-fixtures') {
+      fromFixtures = true;
+      continue;
+    }
+    if (arg === '--file') {
+      const filePath = requireValue(args, arg);
+      const lines = fs
+        .readFileSync(filePath, 'utf8')
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0);
+      sentences.push(...lines);
+      continue;
+    }
+    if (arg === '--api-url') {
+      apiUrl = requireValue(args, arg).replace(/\/$/, '');
+      continue;
+    }
+    if (arg === '--scan-length') {
+      const value = Number(requireValue(args, arg));
+      if (!Number.isInteger(value) || value <= 0) {
+        throw new Error('Invalid --scan-length value');
+      }
+      scanLength = value;
+      continue;
+    }
+    if (arg === '--json') {
+      emitJson = true;
+      continue;
+    }
+    if (arg === '--name-match') {
+      nameMatch = true;
+      continue;
+    }
+    if (arg === '--yomitan-extension') {
+      yomitanExtensionPath = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--yomitan-user-data') {
+      yomitanUserDataPath = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--mecab-command') {
+      mecabCommand = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--mecab-dictionary') {
+      mecabDictionaryPath = requireValue(args, arg);
+      continue;
+    }
+    if (arg.startsWith('-') && arg !== '-') {
+      throw new Error(`Unknown flag: ${arg}`);
+    }
+    sentences.push(arg);
+  }
+
+  if (sentences.length === 0) {
+    fromFixtures = true;
+  }
+
+  return {
+    sentences,
+    fromFixtures,
+    apiUrl,
+    scanLength,
+    emitJson,
+    nameMatch,
+    yomitanExtensionPath,
+    yomitanUserDataPath,
+    mecabCommand,
+    mecabDictionaryPath,
+  };
+}
+
+function loadFixtureSentences(): string[] {
+  if (!fs.existsSync(FIXTURE_DIR)) {
+    return [];
+  }
+  return fs
+    .readdirSync(FIXTURE_DIR)
+    .filter((entry) => entry.endsWith('.json'))
+    .sort()
+    .map((entry) => {
+      const fixture = JSON.parse(fs.readFileSync(path.join(FIXTURE_DIR, entry), 'utf8')) as {
+        input?: { text?: string };
+      };
+      return fixture.input?.text ?? '';
+    })
+    .filter((text) => text.length > 0);
+}
+
+function normalizeTokenizerText(text: string): string {
+  return text
+    .replace(/\r\n/g, '\n')
+    .replace(/\\N/g, '\n')
+    .replace(/\\n/g, '\n')
+    .trim()
+    .replace(/\n/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+interface FuriganaChunk {
+  startPos: number;
+  endPos: number;
+  reading: string;
+  headword: string;
+}
+
+// Walk the raw parseText content the same way the parse-result mapper does
+// (whole segments, char offsets accumulated across lines) and capture, per
+// chunk, the effective reading (kana chunks read as themselves) and the first
+// dictionary headword term. The mapper's token spans are authoritative for
+// grouping; these chunks supply the complete reading/lemma the legacy mapper
+// does not synthesize.
+function collectFuriganaChunks(parseResults: unknown): FuriganaChunk[] {
+  const chunks: FuriganaChunk[] = [];
+  if (!Array.isArray(parseResults)) {
+    return chunks;
+  }
+  const content = (parseResults[0] as { content?: unknown } | undefined)?.content;
+  if (!Array.isArray(content)) {
+    return chunks;
+  }
+  let charOffset = 0;
+  for (const line of content) {
+    if (!Array.isArray(line)) {
+      continue;
+    }
+    for (const segment of line as Array<Record<string, unknown>>) {
+      const text = typeof segment?.text === 'string' ? segment.text : '';
+      if (!text) {
+        continue;
+      }
+      const reading =
+        typeof segment.reading === 'string' && segment.reading.length > 0 ? segment.reading : text;
+      let headword = '';
+      const headwords = segment.headwords;
+      if (Array.isArray(headwords)) {
+        for (const row of headwords) {
+          const first = Array.isArray(row) ? (row[0] as { term?: unknown } | undefined) : undefined;
+          if (first && typeof first.term === 'string' && first.term.trim().length > 0) {
+            headword = first.term.trim();
+            break;
+          }
+        }
+      }
+      chunks.push({ startPos: charOffset, endPos: charOffset + text.length, reading, headword });
+      charOffset += text.length;
+    }
+  }
+  return chunks;
+}
+
+function overlayChunkMetadata(tokens: MergedToken[], chunks: FuriganaChunk[]): MergedToken[] {
+  return tokens.map((token) => {
+    const covered = chunks.filter(
+      (chunk) => chunk.startPos >= token.startPos && chunk.endPos <= token.endPos,
+    );
+    if (covered.length === 0) {
+      return token;
+    }
+    const reading = covered.map((chunk) => chunk.reading).join('');
+    const headword = covered.find((chunk) => chunk.headword)?.headword ?? token.surface;
+    return { ...token, reading, headword };
+  });
+}
+
+async function fetchApiTokens(
+  apiUrl: string,
+  text: string,
+  scanLength: number,
+): Promise<MergedToken[] | null> {
+  const response = await fetchYomitanApi(`${apiUrl}/tokenize`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ text, scanLength, parser: 'scanning-parser' }),
+  });
+  if (!response.ok) {
+    throw new Error(`yomitan-api /tokenize responded ${response.status}`);
+  }
+  const parseResults = (await response.json()) as unknown;
+  const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
+  if (!tokens) {
+    return tokens;
+  }
+  return overlayChunkMetadata(tokens, collectFuriganaChunks(parseResults));
+}
+
+interface ComparedToken {
+  surface: string;
+  reading: string;
+  headword: string;
+  startPos: number;
+  endPos: number;
+  frequencyRank?: number;
+}
+
+function toComparedTokens(tokens: MergedToken[] | null): ComparedToken[] {
+  return (tokens ?? []).map((token) => ({
+    surface: token.surface,
+    reading: token.reading,
+    headword: token.headword,
+    startPos: token.startPos,
+    endPos: token.endPos,
+    frequencyRank: token.frequencyRank,
+  }));
+}
+
+interface MismatchRegion {
+  subminer: ComparedToken[];
+  yomitan: ComparedToken[];
+}
+
+interface PairDiff {
+  surface: string;
+  field: 'headword' | 'reading';
+  subminer: string;
+  yomitan: string;
+}
+
+interface SentenceComparison {
+  text: string;
+  subminerTokens: ComparedToken[];
+  yomitanTokens: ComparedToken[];
+  segmentationMatches: boolean;
+  mismatchRegions: MismatchRegion[];
+  pairDiffs: PairDiff[];
+  error?: string;
+}
+
+// Walk both token lists in span order. Tokens whose [startPos, endPos) agree
+// are paired; disagreeing runs are grouped until the boundaries realign.
+function compareTokenLists(
+  subminer: ComparedToken[],
+  yomitan: ComparedToken[],
+): Pick<SentenceComparison, 'segmentationMatches' | 'mismatchRegions' | 'pairDiffs'> {
+  const mismatchRegions: MismatchRegion[] = [];
+  const pairDiffs: PairDiff[] = [];
+  let i = 0;
+  let j = 0;
+
+  while (i < subminer.length || j < yomitan.length) {
+    const a = subminer[i];
+    const b = yomitan[j];
+    if (a && b && a.startPos === b.startPos && a.endPos === b.endPos) {
+      if (a.headword !== b.headword) {
+        pairDiffs.push({
+          surface: a.surface,
+          field: 'headword',
+          subminer: a.headword,
+          yomitan: b.headword,
+        });
+      }
+      // An empty reading means "reads as its surface" (kana tokens, unparsed
+      // runs) — normalize before comparing.
+      if ((a.reading || a.surface) !== (b.reading || b.surface)) {
+        pairDiffs.push({
+          surface: a.surface,
+          field: 'reading',
+          subminer: a.reading,
+          yomitan: b.reading,
+        });
+      }
+      i += 1;
+      j += 1;
+      continue;
+    }
+
+    const region: MismatchRegion = { subminer: [], yomitan: [] };
+    let aEnd = a ? a.startPos : Number.MAX_SAFE_INTEGER;
+    let bEnd = b ? b.startPos : Number.MAX_SAFE_INTEGER;
+    do {
+      if (aEnd <= bEnd && i < subminer.length) {
+        const token = subminer[i];
+        if (token) {
+          region.subminer.push(token);
+          aEnd = token.endPos;
+        }
+        i += 1;
+      } else if (j < yomitan.length) {
+        const token = yomitan[j];
+        if (token) {
+          region.yomitan.push(token);
+          bEnd = token.endPos;
+        }
+        j += 1;
+      } else {
+        break;
+      }
+    } while (aEnd !== bEnd && (i < subminer.length || j < yomitan.length));
+    mismatchRegions.push(region);
+  }
+
+  return {
+    segmentationMatches: mismatchRegions.length === 0,
+    mismatchRegions,
+    pairDiffs,
+  };
+}
+
+function renderTokenRun(tokens: ComparedToken[]): string {
+  if (tokens.length === 0) {
+    return '(no tokens)';
+  }
+  return tokens.map((token) => token.surface).join('|');
+}
+
+function renderReport(comparisons: SentenceComparison[]): void {
+  let divergent = 0;
+  for (const comparison of comparisons) {
+    const clean =
+      !comparison.error && comparison.segmentationMatches && comparison.pairDiffs.length === 0;
+    if (!clean) {
+      divergent += 1;
+    }
+    process.stdout.write(`\n${clean ? 'MATCH' : 'DIFF '} ${comparison.text}\n`);
+    if (comparison.error) {
+      process.stdout.write(`  error: ${comparison.error}\n`);
+      continue;
+    }
+    if (clean) {
+      process.stdout.write(
+        `  ${comparison.subminerTokens.length} tokens: ${renderTokenRun(comparison.subminerTokens)}\n`,
+      );
+      continue;
+    }
+    if (!comparison.segmentationMatches) {
+      process.stdout.write('  segmentation:\n');
+      for (const region of comparison.mismatchRegions) {
+        process.stdout.write(`    subminer: ${renderTokenRun(region.subminer)}\n`);
+        process.stdout.write(`    yomitan:  ${renderTokenRun(region.yomitan)}\n`);
+      }
+    }
+    for (const diff of comparison.pairDiffs) {
+      process.stdout.write(
+        `  ${diff.surface}: ${diff.field} "${diff.subminer}" (subminer) vs "${diff.yomitan}" (yomitan)\n`,
+      );
+    }
+  }
+  process.stdout.write(
+    `\n${comparisons.length - divergent}/${comparisons.length} sentences match stock Yomitan\n`,
+  );
+}
+
+async function main(): Promise<void> {
+  const options = parseCliArgs(process.argv.slice(2));
+  const sentences = [...(options.fromFixtures ? loadFixtureSentences() : []), ...options.sentences];
+  if (sentences.length === 0) {
+    throw new Error('No sentences to compare (no fixtures found and none provided).');
+  }
+
+  // Fail fast with a clear message when the bridge is not running.
+  try {
+    const probe = await fetchYomitanApi(`${options.apiUrl}/serverVersion`, {
+      method: 'POST',
+      body: '{}',
+    });
+    if (!probe.ok) {
+      throw new Error(`status ${probe.status}`);
+    }
+  } catch (error) {
+    throw new Error(
+      `yomitan-api is not reachable at ${options.apiUrl} (${(error as Error).message}). ` +
+        'Ensure the browser is running and "Enable Yomitan API" is on in Yomitan settings.',
+    );
+  }
+
+  const electronModule = await import('electron').catch(() => null);
+  if (!electronModule?.app || !electronModule?.session) {
+    throw new Error('Electron runtime required; run via `bun run compare-yomitan-api:electron`.');
+  }
+
+  const mecabTokenizer = new MecabTokenizer({
+    mecabCommand: options.mecabCommand,
+    dictionaryPath: options.mecabDictionaryPath,
+  });
+  if (!(await mecabTokenizer.checkAvailability())) {
+    throw new Error('MeCab is not available on this system.');
+  }
+
+  if (typeof electronModule.app.setPath === 'function') {
+    fs.mkdirSync(options.yomitanUserDataPath, { recursive: true });
+    electronModule.app.setPath('userData', options.yomitanUserDataPath);
+  }
+  await electronModule.app.whenReady();
+
+  const extensionPath = resolveBuiltYomitanExtensionPath({
+    explicitPath: options.yomitanExtensionPath,
+    cwd: process.cwd(),
+  });
+  if (!extensionPath) {
+    throw new Error('No built Yomitan extension found; run `bun run build:yomitan` first.');
+  }
+  const extension = await electronModule.session.defaultSession.loadExtension(extensionPath, {
+    allowFileAccess: true,
+  });
+
+  let parserWindow: Electron.BrowserWindow | null = null;
+  let parserReadyPromise: Promise<void> | null = null;
+  let parserInitPromise: Promise<boolean> | null = null;
+
+  const runtimeDeps = createTokenizerDepsRuntime({
+    getYomitanExt: () => extension,
+    getYomitanParserWindow: () => parserWindow,
+    setYomitanParserWindow: (window) => {
+      parserWindow = window;
+    },
+    getYomitanParserReadyPromise: () => parserReadyPromise,
+    setYomitanParserReadyPromise: (promise) => {
+      parserReadyPromise = promise;
+    },
+    getYomitanParserInitPromise: () => parserInitPromise,
+    setYomitanParserInitPromise: (promise) => {
+      parserInitPromise = promise;
+    },
+    isKnownWord: () => false,
+    getKnownWordMatchMode: () => 'headword',
+    getJlptLevel: () => null,
+    getNameMatchEnabled: () => options.nameMatch,
+    getMecabTokenizer: () => ({
+      tokenize: (text: string) => mecabTokenizer.tokenize(text),
+    }),
+  });
+  const deps = {
+    ...runtimeDeps,
+    enrichTokensWithMecab: async (
+      tokens: Parameters<typeof enrichTokensWithMecabPos1>[0],
+      mecabTokens: Parameters<typeof enrichTokensWithMecabPos1>[1],
+    ) => enrichTokensWithMecabPos1(tokens, mecabTokens),
+  };
+
+  const comparisons: SentenceComparison[] = [];
+  try {
+    for (const sentence of sentences) {
+      const text = normalizeTokenizerText(sentence);
+      try {
+        const [subtitleData, apiTokens] = await Promise.all([
+          tokenizeSubtitle(text, deps),
+          fetchApiTokens(options.apiUrl, text, options.scanLength),
+        ]);
+        const subminerTokens = toComparedTokens(subtitleData.tokens);
+        const yomitanTokens = toComparedTokens(apiTokens);
+        comparisons.push({
+          text,
+          subminerTokens,
+          yomitanTokens,
+          ...compareTokenLists(subminerTokens, yomitanTokens),
+        });
+      } catch (error) {
+        comparisons.push({
+          text,
+          subminerTokens: [],
+          yomitanTokens: [],
+          segmentationMatches: false,
+          mismatchRegions: [],
+          pairDiffs: [],
+          error: (error as Error).message,
+        });
+      }
+    }
+  } finally {
+    if (parserWindow) {
+      const window = parserWindow as Electron.BrowserWindow;
+      if (!window.isDestroyed()) {
+        window.destroy();
+      }
+    }
+    electronModule.app.quit();
+  }
+
+  if (options.emitJson) {
+    process.stdout.write(`${JSON.stringify(comparisons, null, 2)}\n`);
+  } else {
+    renderReport(comparisons);
+  }
+
+  const anyDivergence = comparisons.some(
+    (comparison) =>
+      comparison.error !== undefined ||
+      !comparison.segmentationMatches ||
+      comparison.pairDiffs.length > 0,
+  );
+  process.exitCode = anyDivergence ? 1 : 0;
+}
+
+main()
+  .then(() => {
+    process.exitCode = process.exitCode ?? 0;
+  })
+  .catch((error) => {
+    console.error(`Error: ${(error as Error).message}`);
+    process.exitCode = 1;
+  });

scripts/record-tokenizer-fixture.ts

@@ -0,0 +1,496 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import process from 'node:process';
+
+import { createTokenizerDepsRuntime, tokenizeSubtitle } from '../src/core/services/tokenizer.js';
+import {
+  buildFixtureLookups,
+  classifyInjectedScript,
+  fixtureConfigGetters,
+  hashInjectedScript,
+  projectGoldenTokens,
+  pruneGoldenMessages,
+} from '../src/core/services/tokenizer/golden-corpus-harness.js';
+import type {
+  GoldenFixture,
+  GoldenFixtureConfig,
+  GoldenRecordedMessage,
+  GoldenRecordedScript,
+} from '../src/core/services/tokenizer/golden-corpus-harness.js';
+import { enrichTokensWithMecabPos1 } from '../src/core/services/tokenizer/parser-enrichment-stage.js';
+import { resolveYomitanExtensionPath as resolveBuiltYomitanExtensionPath } from '../src/core/services/yomitan-extension-paths.js';
+import { MecabTokenizer } from '../src/mecab-tokenizer.js';
+import type { JlptLevel, Token } from '../src/types.js';
+
+const DEFAULT_YOMITAN_USER_DATA_PATH = path.join(os.homedir(), '.config', 'SubMiner');
+const DEFAULT_FIXTURE_DIR = path.join(
+  'src',
+  'core',
+  'services',
+  'tokenizer',
+  '__fixtures__',
+  'golden',
+);
+const JLPT_LEVELS: ReadonlySet<string> = new Set(['N1', 'N2', 'N3', 'N4', 'N5']);
+
+interface CliOptions {
+  name: string;
+  text: string;
+  description?: string;
+  issueRefs: string[];
+  config: GoldenFixtureConfig;
+  outDir: string;
+  force: boolean;
+  yomitanExtensionPath?: string;
+  yomitanUserDataPath: string;
+  mecabCommand?: string;
+  mecabDictionaryPath?: string;
+}
+
+function printUsage(): void {
+  process.stdout.write(`Usage:
+  bun run record-tokenizer-fixture:electron -- --name <slug> [options] <subtitle text>
+
+  Records a golden tokenizer fixture: raw Yomitan backend responses, raw MeCab
+  tokens, and the annotated tokens the full pipeline produced for the text.
+  Review the "expected" block before committing — it becomes the assertion.
+
+  --name <slug>              Fixture name (file becomes <slug>.json). Required.
+  --description <text>       What behavior this fixture pins down.
+  --issue <ref>              Related issue/PR ref, repeatable (e.g. --issue "#156").
+  --known-word <w[:reading]> Mark a word known, repeatable.
+  --jlpt <term=level>        JLPT level for a term, repeatable (level N1..N5).
+  --local-frequency <t=rank> Local frequency-dictionary rank, repeatable.
+  --match-mode <mode>        Known-word match mode: headword|surface.
+  --min-sentence-words <n>   Minimum sentence words for N+1.
+  --no-known-words           Disable known-word annotation.
+  --no-nplusone              Disable N+1 targeting.
+  --no-jlpt                  Disable JLPT annotation.
+  --no-name-match            Disable character-name matching.
+  --no-frequency             Disable frequency annotation.
+  --out-dir <path>           Fixture directory (default: ${DEFAULT_FIXTURE_DIR}).
+  --force                    Overwrite an existing fixture file.
+  --yomitan-extension <path> Path to built Yomitan extension directory.
+  --yomitan-user-data <path> Electron userData directory (default: ~/.config/SubMiner).
+  --mecab-command <path>     MeCab binary (default: mecab).
+  --mecab-dictionary <path>  MeCab dictionary directory.
+  -h, --help                 Show usage.
+`);
+}
+
+function requireValue(args: string[], flag: string): string {
+  const next = args.shift();
+  if (!next) {
+    throw new Error(`Missing value for ${flag}`);
+  }
+  return next;
+}
+
+function parseCliArgs(argv: string[]): CliOptions {
+  const args = [...argv];
+  const inputParts: string[] = [];
+  const issueRefs: string[] = [];
+  const knownWords: GoldenFixtureConfig['knownWords'] = [];
+  const jlptLevels: Record<string, JlptLevel> = {};
+  const localFrequencyRanks: Record<string, number> = {};
+  const config: GoldenFixtureConfig = {};
+  let name: string | undefined;
+  let description: string | undefined;
+  let outDir = DEFAULT_FIXTURE_DIR;
+  let force = false;
+  let yomitanExtensionPath: string | undefined;
+  let yomitanUserDataPath = DEFAULT_YOMITAN_USER_DATA_PATH;
+  let mecabCommand: string | undefined;
+  let mecabDictionaryPath: string | undefined;
+
+  while (args.length > 0) {
+    const arg = args.shift();
+    if (!arg) break;
+
+    if (arg === '--help' || arg === '-h') {
+      printUsage();
+      process.exit(0);
+    }
+    if (arg === '--name') {
+      name = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--description') {
+      description = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--issue') {
+      issueRefs.push(requireValue(args, arg));
+      continue;
+    }
+    if (arg === '--known-word') {
+      const value = requireValue(args, arg);
+      const separator = value.indexOf(':');
+      knownWords.push(
+        separator === -1
+          ? value
+          : { text: value.slice(0, separator), reading: value.slice(separator + 1) },
+      );
+      continue;
+    }
+    if (arg === '--jlpt') {
+      const value = requireValue(args, arg);
+      const separator = value.indexOf('=');
+      const level = separator === -1 ? '' : value.slice(separator + 1);
+      if (separator === -1 || !JLPT_LEVELS.has(level)) {
+        throw new Error(`Invalid --jlpt value "${value}"; expected <term>=N1..N5`);
+      }
+      jlptLevels[value.slice(0, separator)] = level as JlptLevel;
+      continue;
+    }
+    if (arg === '--local-frequency') {
+      const value = requireValue(args, arg);
+      const separator = value.indexOf('=');
+      const rank = separator === -1 ? Number.NaN : Number(value.slice(separator + 1));
+      if (!Number.isInteger(rank) || rank <= 0) {
+        throw new Error(`Invalid --local-frequency value "${value}"; expected <term>=<rank>`);
+      }
+      localFrequencyRanks[value.slice(0, separator)] = rank;
+      continue;
+    }
+    if (arg === '--match-mode') {
+      const value = requireValue(args, arg);
+      if (value !== 'headword' && value !== 'surface') {
+        throw new Error(`Invalid --match-mode "${value}"; expected headword|surface`);
+      }
+      config.knownWordMatchMode = value;
+      config.frequencyMatchMode = value;
+      continue;
+    }
+    if (arg === '--min-sentence-words') {
+      const value = Number(requireValue(args, arg));
+      if (!Number.isInteger(value) || value < 0) {
+        throw new Error('Invalid --min-sentence-words value');
+      }
+      config.minSentenceWordsForNPlusOne = value;
+      continue;
+    }
+    if (arg === '--no-known-words') {
+      config.knownWordsEnabled = false;
+      continue;
+    }
+    if (arg === '--no-nplusone') {
+      config.nPlusOneEnabled = false;
+      continue;
+    }
+    if (arg === '--no-jlpt') {
+      config.jlptEnabled = false;
+      continue;
+    }
+    if (arg === '--no-name-match') {
+      config.nameMatchEnabled = false;
+      continue;
+    }
+    if (arg === '--no-frequency') {
+      config.frequencyEnabled = false;
+      continue;
+    }
+    if (arg === '--out-dir') {
+      outDir = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--force') {
+      force = true;
+      continue;
+    }
+    if (arg === '--yomitan-extension') {
+      yomitanExtensionPath = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--yomitan-user-data') {
+      yomitanUserDataPath = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--mecab-command') {
+      mecabCommand = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--mecab-dictionary') {
+      mecabDictionaryPath = requireValue(args, arg);
+      continue;
+    }
+    if (arg.startsWith('-') && arg !== '-') {
+      throw new Error(`Unknown flag: ${arg}`);
+    }
+    inputParts.push(arg);
+  }
+
+  if (!name || !/^[a-z0-9][a-z0-9-]*$/.test(name)) {
+    throw new Error('A kebab-case --name is required (e.g. --name kureru-lexical-keep)');
+  }
+  const text = inputParts.join(' ').trim();
+  if (!text) {
+    throw new Error('Provide the subtitle text to record as positional arguments.');
+  }
+
+  if (knownWords.length > 0) config.knownWords = knownWords;
+  if (Object.keys(jlptLevels).length > 0) config.jlptLevels = jlptLevels;
+  if (Object.keys(localFrequencyRanks).length > 0) {
+    config.localFrequencyRanks = localFrequencyRanks;
+  }
+
+  return {
+    name,
+    text,
+    description,
+    issueRefs,
+    config,
+    outDir,
+    force,
+    yomitanExtensionPath,
+    yomitanUserDataPath,
+    mecabCommand,
+    mecabDictionaryPath,
+  };
+}
+
+const RECORDER_SHIM_SCRIPT = `
+(() => {
+  const g = globalThis;
+  if (g.__subminerGoldenRecorderInstalled) { return true; }
+  g.__subminerGoldenRecorderInstalled = true;
+  g.__subminerGoldenMessages = [];
+  const runtime = chrome.runtime;
+  const original = runtime.sendMessage.bind(runtime);
+  runtime.sendMessage = (payload, callback) => {
+    return original(payload, (response) => {
+      try {
+        g.__subminerGoldenMessages.push(JSON.parse(JSON.stringify({
+          action: payload && payload.action ? payload.action : '',
+          params: payload && typeof payload.params !== 'undefined' ? payload.params : null,
+          response: typeof response !== 'undefined' ? response : null,
+        })));
+      } catch {}
+      if (callback) { callback(response); }
+    });
+  };
+  return true;
+})();
+`;
+
+const RECORDER_DRAIN_SCRIPT = `
+(() => {
+  const g = globalThis;
+  const log = g.__subminerGoldenMessages || [];
+  g.__subminerGoldenMessages = [];
+  return JSON.stringify(log);
+})();
+`;
+
+interface RecordingSink {
+  messages: GoldenRecordedMessage[];
+  scripts: GoldenRecordedScript[];
+  mecab: Record<string, Token[] | null>;
+}
+
+function patchParserWindowForRecording(
+  window: Electron.BrowserWindow | null,
+  patched: WeakSet<object>,
+  sink: RecordingSink,
+): void {
+  if (!window || window.isDestroyed()) {
+    return;
+  }
+  const webContents = window.webContents as unknown as {
+    executeJavaScript: (script: string, userGesture?: boolean) => Promise<unknown>;
+  };
+  if (patched.has(webContents)) {
+    return;
+  }
+  patched.add(webContents);
+
+  const original = webContents.executeJavaScript.bind(webContents);
+  let shimReady: Promise<unknown> | null = null;
+  const drain = async (): Promise<void> => {
+    try {
+      const raw = (await original(RECORDER_DRAIN_SCRIPT, true)) as string;
+      const drained = JSON.parse(raw) as GoldenRecordedMessage[];
+      sink.messages.push(...drained);
+    } catch (error) {
+      process.stderr.write(
+        `Warning: failed to drain recorded messages: ${(error as Error).message}\n`,
+      );
+    }
+  };
+
+  webContents.executeJavaScript = async (script: string, userGesture?: boolean) => {
+    if (!shimReady) {
+      shimReady = original(RECORDER_SHIM_SCRIPT, true);
+    }
+    try {
+      await shimReady;
+    } catch {
+      shimReady = null;
+    }
+    let result: unknown;
+    try {
+      result = await original(script, userGesture);
+    } finally {
+      await drain();
+    }
+    sink.scripts.push({
+      sha256: hashInjectedScript(script),
+      marker: classifyInjectedScript(script),
+      result: result === undefined ? null : result,
+    });
+    return result;
+  };
+}
+
+async function main(): Promise<void> {
+  const options = parseCliArgs(process.argv.slice(2));
+  const outPath = path.join(options.outDir, `${options.name}.json`);
+  if (fs.existsSync(outPath) && !options.force) {
+    throw new Error(`${outPath} already exists; pass --force to overwrite.`);
+  }
+
+  const electronModule = await import('electron').catch(() => null);
+  if (!electronModule?.app || !electronModule?.session) {
+    throw new Error(
+      'Electron runtime is required; run via `bun run record-tokenizer-fixture:electron`.',
+    );
+  }
+
+  const mecabTokenizer = new MecabTokenizer({
+    mecabCommand: options.mecabCommand,
+    dictionaryPath: options.mecabDictionaryPath,
+  });
+  if (!(await mecabTokenizer.checkAvailability())) {
+    throw new Error('MeCab is not available on this system.');
+  }
+
+  if (typeof electronModule.app.setPath === 'function') {
+    electronModule.app.setPath('userData', options.yomitanUserDataPath);
+  }
+  await electronModule.app.whenReady();
+
+  const extensionPath = resolveBuiltYomitanExtensionPath({
+    explicitPath: options.yomitanExtensionPath,
+    cwd: process.cwd(),
+  });
+  if (!extensionPath) {
+    throw new Error('No built Yomitan extension found; run `bun run build:yomitan` first.');
+  }
+  const extension = await electronModule.session.defaultSession.loadExtension(extensionPath, {
+    allowFileAccess: true,
+  });
+
+  const sink: RecordingSink = { messages: [], scripts: [], mecab: {} };
+  const patched = new WeakSet<object>();
+  let parserWindow: Electron.BrowserWindow | null = null;
+  let parserReadyPromise: Promise<void> | null = null;
+  let parserInitPromise: Promise<boolean> | null = null;
+
+  const lookups = buildFixtureLookups(options.config);
+  const runtimeDeps = createTokenizerDepsRuntime({
+    getYomitanExt: () => extension,
+    getYomitanParserWindow: () => {
+      patchParserWindowForRecording(parserWindow, patched, sink);
+      return parserWindow;
+    },
+    setYomitanParserWindow: (window) => {
+      parserWindow = window;
+      patchParserWindowForRecording(parserWindow, patched, sink);
+    },
+    getYomitanParserReadyPromise: () => parserReadyPromise,
+    setYomitanParserReadyPromise: (promise) => {
+      parserReadyPromise = promise;
+    },
+    getYomitanParserInitPromise: () => parserInitPromise,
+    setYomitanParserInitPromise: (promise) => {
+      parserInitPromise = promise;
+    },
+    isKnownWord: lookups.isKnownWord,
+    getJlptLevel: lookups.getJlptLevel,
+    getFrequencyRank: lookups.getFrequencyRank,
+    ...fixtureConfigGetters(options.config),
+    getMecabTokenizer: () => ({
+      tokenize: async (text: string) => {
+        const rawTokens = await mecabTokenizer.tokenize(text);
+        sink.mecab[text] = rawTokens;
+        return rawTokens;
+      },
+    }),
+  });
+  const deps = {
+    ...runtimeDeps,
+    // Same pinned enrichment implementation the replay harness uses.
+    enrichTokensWithMecab: async (
+      tokens: Parameters<typeof enrichTokensWithMecabPos1>[0],
+      mecabTokens: Parameters<typeof enrichTokensWithMecabPos1>[1],
+    ) => enrichTokensWithMecabPos1(tokens, mecabTokens),
+  };
+
+  try {
+    const subtitleData = await tokenizeSubtitle(options.text, deps);
+    const expectedTokens = projectGoldenTokens(subtitleData.tokens);
+
+    const prunedMessages = pruneGoldenMessages(sink.messages);
+    const fixture: GoldenFixture = {
+      name: options.name,
+      ...(options.description ? { description: options.description } : {}),
+      ...(options.issueRefs.length > 0 ? { issueRefs: options.issueRefs } : {}),
+      recordedAt: new Date().toISOString(),
+      input: { text: options.text },
+      config: options.config,
+      recording: {
+        messages: prunedMessages,
+        scripts: sink.scripts,
+        mecab: sink.mecab,
+      },
+      expected: { tokens: expectedTokens },
+    };
+
+    fs.mkdirSync(options.outDir, { recursive: true });
+    fs.writeFileSync(outPath, `${JSON.stringify(fixture, null, 2)}\n`);
+
+    process.stdout.write(`Recorded ${outPath}\n`);
+    process.stdout.write(
+      `  messages=${prunedMessages.length} scripts=${sink.scripts.length} mecabTexts=${Object.keys(sink.mecab).length}\n`,
+    );
+    process.stdout.write('\nExpected tokens (review before committing):\n');
+    if (!expectedTokens || expectedTokens.length === 0) {
+      process.stdout.write('  (none)\n');
+    } else {
+      for (const token of expectedTokens) {
+        const flags = [
+          token.isKnown ? 'known' : null,
+          token.isNPlusOneTarget ? 'n+1' : null,
+          token.isNameMatch ? 'name' : null,
+          token.isUnparsedRun ? 'unparsed' : null,
+          token.jlptLevel ?? null,
+          token.frequencyRank !== undefined ? `freq=${token.frequencyRank}` : null,
+        ]
+          .filter(Boolean)
+          .join(', ');
+        process.stdout.write(
+          `  ${token.surface} -> ${token.headword} (${token.reading})${flags ? ` [${flags}]` : ''}\n`,
+        );
+      }
+    }
+    process.stdout.write(
+      '\nVerify replay with: bun test src/core/services/tokenizer/golden-corpus.test.ts\n',
+    );
+  } finally {
+    if (parserWindow) {
+      const window = parserWindow as Electron.BrowserWindow;
+      if (!window.isDestroyed()) {
+        window.destroy();
+      }
+    }
+    electronModule.app.quit();
+  }
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((error) => {
+    console.error(`Error: ${(error as Error).message}`);
+    process.exit(1);
+  });

scripts/yomitan-api-request.test.ts

@@ -0,0 +1,69 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { fetchYomitanApi } from './yomitan-api-request.js';
+
+function createStalledServer(): ReturnType<typeof Bun.serve> {
+  return Bun.serve({
+    hostname: '127.0.0.1',
+    port: 0,
+    fetch: () => new Promise<Response>(() => {}),
+  });
+}
+
+test('fetchYomitanApi reports a clear timeout when the bridge stalls', async () => {
+  const server = createStalledServer();
+
+  try {
+    await assert.rejects(
+      () => fetchYomitanApi(`${server.url}tokenize`, { method: 'POST' }, 20),
+      /Yomitan API request timed out after 20ms/,
+    );
+  } finally {
+    server.stop(true);
+  }
+});
+
+test('fetchYomitanApi preserves an init cancellation signal', async () => {
+  const server = createStalledServer();
+  const controller = new AbortController();
+  const reason = new Error('cancelled by caller');
+
+  try {
+    const request = fetchYomitanApi(`${server.url}tokenize`, { signal: controller.signal }, 50);
+    controller.abort(reason);
+    await assert.rejects(request, (error) => error === reason);
+  } finally {
+    server.stop(true);
+  }
+});
+
+test('fetchYomitanApi preserves a Request input cancellation signal', async () => {
+  const server = createStalledServer();
+  const controller = new AbortController();
+  const reason = new Error('cancelled by request');
+  const input = new Request(`${server.url}tokenize`, { signal: controller.signal });
+
+  try {
+    const request = fetchYomitanApi(input, undefined, 50);
+    controller.abort(reason);
+    await assert.rejects(request, (error) => error === reason);
+  } finally {
+    server.stop(true);
+  }
+});
+
+test('fetchYomitanApi accepts a null init cancellation signal', async () => {
+  const server = Bun.serve({
+    hostname: '127.0.0.1',
+    port: 0,
+    fetch: () => new Response('ok'),
+  });
+
+  try {
+    const response = await fetchYomitanApi(`${server.url}tokenize`, { signal: null }, 50);
+    assert.equal(response.status, 200);
+  } finally {
+    server.stop(true);
+  }
+});

scripts/yomitan-api-request.ts

@@ -0,0 +1,26 @@
+const DEFAULT_YOMITAN_API_TIMEOUT_MS = 10_000;
+
+export async function fetchYomitanApi(
+  input: RequestInfo | URL,
+  init?: RequestInit,
+  timeoutMs = DEFAULT_YOMITAN_API_TIMEOUT_MS,
+): Promise<Response> {
+  const controller = new AbortController();
+  const requestSignal = input instanceof Request ? input.signal : undefined;
+  const signals = [controller.signal, init?.signal, requestSignal].filter(
+    (signal): signal is AbortSignal => signal != null,
+  );
+  const signal = signals.length === 1 ? controller.signal : AbortSignal.any(signals);
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+  try {
+    return await fetch(input, { ...init, signal });
+  } catch (error) {
+    if (controller.signal.aborted) {
+      throw new Error(`Yomitan API request timed out after ${timeoutMs}ms`, { cause: error });
+    }
+    throw error;
+  } finally {
+    clearTimeout(timeout);
+  }
+}

src/anki-integration/known-word-cache.test.ts

@@ -762,7 +762,10 @@ test('KnownWordCacheManager suppresses reading-only matches when disallowed', as
     // Reading-only match stays available for kana subtitle text…
     assert.equal(manager.isKnownWord('けいこく'), true);
     // …but a kanji token's reading (渓谷/けいこく) must not borrow 警告's.
-    assert.equal(manager.isKnownWord('けいこく', undefined, { allowReadingOnlyMatch: false }), false);
+    assert.equal(
+      manager.isKnownWord('けいこく', undefined, { allowReadingOnlyMatch: false }),
+      false,
+    );
     // Mined word texts still match regardless of the flag.
     assert.equal(manager.isKnownWord('警告', undefined, { allowReadingOnlyMatch: false }), true);
   } finally {

src/core/services/tokenizer/__fixtures__/golden/README.md

@@ -0,0 +1,72 @@
+# Tokenizer golden corpus
+
+End-to-end regression fixtures for the tokenizer/annotation pipeline. Each
+`.json` file captures one real subtitle line together with everything the
+pipeline consumed while tokenizing it live:
+
+- `recording.messages` — the raw Yomitan backend responses
+  (`chrome.runtime.sendMessage` level: `optionsGetFull`, `getDictionaryInfo`,
+  `parseText`, `termsFind`, `getTermFrequencies`), pruned to the fields the
+  injected scanning helpers actually read.
+- `recording.scripts` — sha256 → result pairs for each injected script, used
+  only as a replay fallback if a script cannot run in the vm.
+- `recording.mecab` — raw MeCab tokens keyed by the tokenized text.
+- `config` — annotation toggles plus fixture-local known words, JLPT levels,
+  and local frequency ranks (see `golden-corpus-harness.ts` for the simplified
+  known-word semantics shared by recorder and replay).
+- `expected.tokens` — the annotated tokens the full pipeline produced at
+  record time. **This is the assertion.** Review it before committing; it is a
+  characterization of current behavior, not a statement that the behavior is
+  ideal.
+
+`golden-corpus.test.ts` replays every fixture through the real
+`tokenizeSubtitle` (scan-token merge, MeCab enrichment, frequency ranks,
+annotation stage, noise suppression) by executing the real injected scripts in
+a `node:vm` sandbox against the recorded responses — no Electron, no
+dictionaries, no network.
+
+## Recording a fixture
+
+Requires a built Yomitan extension (`bun run build:yomitan`), your SubMiner
+Yomitan profile (`~/.config/SubMiner`), and MeCab:
+
+```sh
+bun run record-tokenizer-fixture:electron -- \
+  --name my-regression-case \
+  --issue "#123" \
+  --description "what behavior this pins down" \
+  --known-word 私 --known-word 要る:いる \
+  --jlpt 美しい=N4 \
+  そのまま字幕の一行
+```
+
+Run with `--help` for all options (annotation toggles, match mode, overwrite
+with `--force`, ...). The recorder prints the expected tokens for review and
+the fixture replays immediately via:
+
+```sh
+bun test src/core/services/tokenizer/golden-corpus.test.ts
+```
+
+## When a fixture fails
+
+A failure means the pipeline now produces different annotated tokens for that
+line. If the change is intentional, re-record the fixture with `--force`
+(same flags — they are stored in the fixture's `config`) and review the diff
+of `expected.tokens`; the diff _is_ the behavior change. If the change is not
+intentional, you found the regression before shipping it.
+
+Fixture dictionaries reflect whatever was installed in the recording profile,
+so re-recorded fixtures may differ in frequency ranks if dictionaries changed.
+
+## Comparing against stock Yomitan
+
+`bun run compare-yomitan-api:electron` diffs SubMiner's tokenization against a
+stock Yomitan instance reached through the
+[yomitan-api](https://github.com/yomidevs/yomitan-api) bridge
+(`http://127.0.0.1:19633`, enable "Yomitan API" in the browser extension's
+settings). Without arguments it compares every fixture text; pass sentences or
+`--file <path>` for ad-hoc checks. It reports segmentation, reading, and
+headword-form divergence and exits non-zero on any difference — useful for
+spot-checking that the pipeline still matches what Yomitan itself would
+produce, with your real browser profile and dictionaries as the reference.

src/core/services/tokenizer/golden-corpus-harness.ts

@@ -0,0 +1,475 @@
+import { createHash } from 'node:crypto';
+import vm from 'node:vm';
+import type {
+  FrequencyDictionaryMatchMode,
+  JlptLevel,
+  MergedToken,
+  NPlusOneMatchMode,
+  PartOfSpeech,
+  Token,
+} from '../../../types';
+import { createTokenizerDepsRuntime } from '../tokenizer';
+import type { TokenizerServiceDeps } from '../tokenizer';
+import { enrichTokensWithMecabPos1 } from './parser-enrichment-stage';
+
+// Golden-corpus fixtures capture a real subtitle line, the raw Yomitan
+// backend responses (chrome.runtime.sendMessage level) and raw MeCab tokens
+// observed while tokenizing it live, plus the annotated tokens the full
+// pipeline produced. Replay runs the real injected scanning scripts in a vm
+// against the recorded responses, so merge/enrichment/annotation/filtering
+// are exercised end-to-end without Electron or dictionaries.
+
+export interface GoldenFixtureKnownWord {
+  text: string;
+  reading?: string;
+}
+
+export interface GoldenFixtureConfig {
+  knownWords?: Array<string | GoldenFixtureKnownWord>;
+  knownWordMatchMode?: NPlusOneMatchMode;
+  jlptLevels?: Record<string, JlptLevel>;
+  localFrequencyRanks?: Record<string, number>;
+  knownWordsEnabled?: boolean;
+  nPlusOneEnabled?: boolean;
+  jlptEnabled?: boolean;
+  nameMatchEnabled?: boolean;
+  frequencyEnabled?: boolean;
+  frequencyMatchMode?: FrequencyDictionaryMatchMode;
+  minSentenceWordsForNPlusOne?: number;
+}
+
+export interface GoldenRecordedMessage {
+  action: string;
+  params: unknown;
+  /** Raw response object the page callback received: { result } or { error }. */
+  response: unknown;
+}
+
+export interface GoldenRecordedScript {
+  sha256: string;
+  /** Short classification of the script, for debugging only. */
+  marker: string;
+  result: unknown;
+}
+
+export interface GoldenFixtureRecording {
+  messages: GoldenRecordedMessage[];
+  scripts: GoldenRecordedScript[];
+  /** Raw MeCab tokens keyed by the exact text passed to tokenizeWithMecab. */
+  mecab: Record<string, Token[] | null>;
+}
+
+export interface GoldenExpectedToken {
+  surface: string;
+  reading: string;
+  headword: string;
+  headwordReading?: string;
+  startPos: number;
+  endPos: number;
+  partOfSpeech: PartOfSpeech;
+  pos1?: string;
+  pos2?: string;
+  pos3?: string;
+  isKnown: boolean;
+  isNPlusOneTarget: boolean;
+  isNameMatch?: true;
+  isUnparsedRun?: true;
+  jlptLevel?: JlptLevel;
+  frequencyRank?: number;
+}
+
+export interface GoldenFixture {
+  name: string;
+  description?: string;
+  issueRefs?: string[];
+  recordedAt?: string;
+  input: { text: string };
+  config: GoldenFixtureConfig;
+  recording: GoldenFixtureRecording;
+  expected: { tokens: GoldenExpectedToken[] | null };
+}
+
+// Only these backend actions are requested by the scanning/frequency scripts;
+// everything else in the recorded page traffic is Yomitan's own chatter.
+const GOLDEN_MESSAGE_ACTIONS: ReadonlySet<string> = new Set([
+  'optionsGetFull',
+  'getDictionaryInfo',
+  'parseText',
+  'termsFind',
+  'getTermFrequencies',
+]);
+
+// Field candidates appendDictionaryNames() in the injected helpers reads off
+// definitions/pronunciations — the only reason those arrays are consulted.
+const DICTIONARY_NAME_FIELDS = [
+  'dictionary',
+  'dictionaryName',
+  'name',
+  'title',
+  'dictionaryTitle',
+  'dictionaryAlias',
+] as const;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}
+
+function pruneToDictionaryNameFields(value: unknown): unknown {
+  if (!isRecord(value)) {
+    return {};
+  }
+  const pruned: Record<string, unknown> = {};
+  for (const field of DICTIONARY_NAME_FIELDS) {
+    if (typeof value[field] === 'string') {
+      pruned[field] = value[field];
+    }
+  }
+  return pruned;
+}
+
+// Fields getBestFrequencyRank() and the frequency grouping helpers read off a
+// termsFind frequency item, beyond the dictionary-name candidates.
+const FREQUENCY_ITEM_FIELDS = [
+  'headwordIndex',
+  'displayValue',
+  'displayValueParsed',
+  'frequency',
+  'dictionaryIndex',
+  'term',
+  'reading',
+] as const;
+
+function pruneFrequencyItem(item: unknown): unknown {
+  if (!isRecord(item)) {
+    return item;
+  }
+  const pruned = pruneToDictionaryNameFields(item) as Record<string, unknown>;
+  for (const field of FREQUENCY_ITEM_FIELDS) {
+    if (item[field] !== undefined) {
+      pruned[field] = item[field];
+    }
+  }
+  return pruned;
+}
+
+function pruneTermsFindEntry(entry: unknown): unknown {
+  if (!isRecord(entry)) {
+    return entry;
+  }
+  const pruned: Record<string, unknown> = { ...entry };
+  if (Array.isArray(entry.definitions)) {
+    pruned.definitions = entry.definitions.map(pruneToDictionaryNameFields);
+  }
+  if (Array.isArray(entry.pronunciations)) {
+    pruned.pronunciations = entry.pronunciations.map(pruneToDictionaryNameFields);
+  }
+  if (Array.isArray(entry.frequencies)) {
+    pruned.frequencies = entry.frequencies.map(pruneFrequencyItem);
+  }
+  return pruned;
+}
+
+function pruneMessageResult(action: string, result: unknown): unknown {
+  if (action === 'optionsGetFull' && isRecord(result)) {
+    const profiles = Array.isArray(result.profiles) ? result.profiles : [];
+    return {
+      profileCurrent: result.profileCurrent,
+      profiles: profiles.map((profile) => {
+        if (!isRecord(profile) || !isRecord(profile.options)) {
+          return {};
+        }
+        const options = profile.options;
+        return {
+          options: {
+            scanning: isRecord(options.scanning) ? { length: options.scanning.length } : {},
+            dictionaries: Array.isArray(options.dictionaries)
+              ? options.dictionaries.map((dictionary) =>
+                  isRecord(dictionary)
+                    ? {
+                        name: dictionary.name,
+                        enabled: dictionary.enabled,
+                        id: dictionary.id,
+                      }
+                    : {},
+                )
+              : [],
+          },
+        };
+      }),
+    };
+  }
+  if (action === 'getDictionaryInfo' && Array.isArray(result)) {
+    return result.map((entry) =>
+      isRecord(entry) ? { title: entry.title, frequencyMode: entry.frequencyMode } : {},
+    );
+  }
+  if (action === 'termsFind' && isRecord(result) && Array.isArray(result.dictionaryEntries)) {
+    return {
+      ...result,
+      dictionaryEntries: result.dictionaryEntries.map(pruneTermsFindEntry),
+    };
+  }
+  return result;
+}
+
+// Strip recorded page traffic down to what replay can ever request: drop
+// Yomitan's own background messages and the heavyweight response fields
+// (definition glossaries, full options blobs) the injected helpers never read.
+// Prune safety is verified by the replay tests themselves — a pruned fixture
+// must still reproduce its recorded expectation.
+export function pruneGoldenMessages(messages: GoldenRecordedMessage[]): GoldenRecordedMessage[] {
+  return messages
+    .filter((message) => GOLDEN_MESSAGE_ACTIONS.has(message.action))
+    .map((message) => {
+      if (isRecord(message.response) && 'result' in message.response) {
+        return {
+          ...message,
+          response: {
+            ...message.response,
+            result: pruneMessageResult(message.action, message.response.result),
+          },
+        };
+      }
+      return message;
+    });
+}
+
+export function hashInjectedScript(script: string): string {
+  return createHash('sha256').update(script).digest('hex');
+}
+
+export function classifyInjectedScript(script: string): string {
+  const markers = [
+    'optionsGetFull',
+    'parseText',
+    'termsFind',
+    'getTermFrequencies',
+    'setAllSettings',
+  ];
+  const found = markers.filter((marker) => script.includes(marker));
+  return found.length > 0 ? found.join('+') : 'unknown';
+}
+
+function stableStringify(value: unknown): string {
+  // Recorded params pass through JSON, which turns undefined into null.
+  return JSON.stringify(value ?? null, (_key, entry: unknown) => {
+    if (entry && typeof entry === 'object' && !Array.isArray(entry)) {
+      const record = entry as Record<string, unknown>;
+      const sorted: Record<string, unknown> = {};
+      for (const key of Object.keys(record).sort()) {
+        sorted[key] = record[key];
+      }
+      return sorted;
+    }
+    return entry;
+  });
+}
+
+export interface FixtureLookups {
+  isKnownWord: (
+    text: string,
+    reading?: string,
+    options?: { allowReadingOnlyMatch?: boolean },
+  ) => boolean;
+  getJlptLevel: (text: string) => JlptLevel | null;
+  getFrequencyRank: (term: string) => number | null;
+}
+
+// Deliberately simple known-word semantics (exact text, optional reading
+// constraint) shared by the recorder and the replay harness so both sides of
+// a fixture agree. The production reading-aware cache is exercised by its own
+// unit tests, not by the golden corpus.
+export function buildFixtureLookups(config: GoldenFixtureConfig): FixtureLookups {
+  const readingsByText = new Map<string, Set<string> | null>();
+  for (const entry of config.knownWords ?? []) {
+    const text = typeof entry === 'string' ? entry : entry.text;
+    const reading = typeof entry === 'string' ? undefined : entry.reading;
+    if (!reading) {
+      readingsByText.set(text, null);
+      continue;
+    }
+    const existing = readingsByText.get(text);
+    if (existing === null) {
+      continue;
+    }
+    const readings = existing ?? new Set<string>();
+    readings.add(reading);
+    readingsByText.set(text, readings);
+  }
+
+  return {
+    isKnownWord: (text, reading) => {
+      if (!readingsByText.has(text)) {
+        return false;
+      }
+      const readings = readingsByText.get(text);
+      if (readings === null || readings === undefined) {
+        return true;
+      }
+      return reading === undefined || readings.has(reading);
+    },
+    getJlptLevel: (text) => config.jlptLevels?.[text] ?? null,
+    getFrequencyRank: (term) => config.localFrequencyRanks?.[term] ?? null,
+  };
+}
+
+// Config-driven deps getters shared by the recorder and the replay harness so
+// a fixture is tokenized under identical toggles in both directions.
+export function fixtureConfigGetters(config: GoldenFixtureConfig) {
+  return {
+    getKnownWordMatchMode: () => config.knownWordMatchMode ?? ('headword' as NPlusOneMatchMode),
+    getKnownWordsEnabled: () => config.knownWordsEnabled !== false,
+    getNPlusOneEnabled: () => config.nPlusOneEnabled !== false,
+    getJlptEnabled: () => config.jlptEnabled !== false,
+    getNameMatchEnabled: () => config.nameMatchEnabled !== false,
+    getNameMatchImagesEnabled: () => false,
+    getFrequencyDictionaryEnabled: () => config.frequencyEnabled !== false,
+    getFrequencyDictionaryMatchMode: () =>
+      config.frequencyMatchMode ?? ('headword' as FrequencyDictionaryMatchMode),
+    getMinSentenceWordsForNPlusOne: () => config.minSentenceWordsForNPlusOne ?? 3,
+  };
+}
+
+interface ReplayMessageEntry extends GoldenRecordedMessage {
+  used: boolean;
+  paramsKey: string;
+}
+
+export interface ReplayMessageStore {
+  handle: (action: string, params: unknown) => unknown;
+  unusedCount: () => number;
+}
+
+export function createReplayMessageStore(messages: GoldenRecordedMessage[]): ReplayMessageStore {
+  const entries: ReplayMessageEntry[] = messages.map((message) => ({
+    ...message,
+    used: false,
+    paramsKey: stableStringify(message.params),
+  }));
+
+  return {
+    handle: (action, params) => {
+      const paramsKey = stableStringify(params);
+      const match =
+        entries.find((e) => !e.used && e.action === action && e.paramsKey === paramsKey) ??
+        entries.find((e) => !e.used && e.action === action) ??
+        entries.find((e) => e.action === action && e.paramsKey === paramsKey);
+      if (!match) {
+        throw new Error(
+          `golden fixture replay: no recorded response for action "${action}" params=${paramsKey}`,
+        );
+      }
+      match.used = true;
+      return match.response;
+    },
+    unusedCount: () => entries.filter((e) => !e.used).length,
+  };
+}
+
+async function runInjectedScriptInVm(script: string, store: ReplayMessageStore): Promise<unknown> {
+  return await vm.runInNewContext(script, {
+    chrome: {
+      runtime: {
+        lastError: null,
+        sendMessage: (
+          payload: { action?: string; params?: unknown },
+          callback: (response: unknown) => void,
+        ) => {
+          callback(store.handle(payload.action ?? '', payload.params));
+        },
+      },
+    },
+    Array,
+    Boolean,
+    Date,
+    Error,
+    JSON,
+    Map,
+    Math,
+    Number,
+    Object,
+    Promise,
+    RegExp,
+    Set,
+    String,
+  });
+}
+
+export function createReplayTokenizerDeps(fixture: GoldenFixture): TokenizerServiceDeps {
+  const store = createReplayMessageStore(fixture.recording.messages);
+  const scriptResults = new Map(
+    fixture.recording.scripts.map((entry) => [entry.sha256, entry] as const),
+  );
+
+  const parserWindow = {
+    isDestroyed: () => false,
+    webContents: {
+      executeJavaScript: async (script: string) => {
+        try {
+          return await runInjectedScriptInVm(script, store);
+        } catch (vmError) {
+          const recorded = scriptResults.get(hashInjectedScript(script));
+          if (recorded) {
+            return recorded.result;
+          }
+          throw new Error(
+            `golden fixture "${fixture.name}": injected script (${classifyInjectedScript(script)}) failed in vm replay and has no recorded script-level result: ${(vmError as Error).message}`,
+          );
+        }
+      },
+    },
+  };
+
+  const lookups = buildFixtureLookups(fixture.config);
+  const deps = createTokenizerDepsRuntime({
+    getYomitanExt: () => ({ id: 'golden-fixture-extension' }) as never,
+    getYomitanParserWindow: () => parserWindow as never,
+    setYomitanParserWindow: () => undefined,
+    getYomitanParserReadyPromise: () => null,
+    setYomitanParserReadyPromise: () => undefined,
+    getYomitanParserInitPromise: () => null,
+    setYomitanParserInitPromise: () => undefined,
+    isKnownWord: lookups.isKnownWord,
+    getJlptLevel: lookups.getJlptLevel,
+    getFrequencyRank: lookups.getFrequencyRank,
+    ...fixtureConfigGetters(fixture.config),
+    getMecabTokenizer: () => ({
+      tokenize: async (text: string) => fixture.recording.mecab[text] ?? null,
+    }),
+  });
+
+  return {
+    ...deps,
+    // Pin the synchronous enrichment stage so record and replay share one
+    // implementation and no worker spawns inside bun test.
+    enrichTokensWithMecab: async (tokens, mecabTokens) =>
+      enrichTokensWithMecabPos1(tokens, mecabTokens),
+  };
+}
+
+export function projectGoldenTokens(tokens: MergedToken[] | null): GoldenExpectedToken[] | null {
+  if (!tokens) {
+    return null;
+  }
+  return tokens.map((token) => {
+    const projected: GoldenExpectedToken = {
+      surface: token.surface,
+      reading: token.reading,
+      headword: token.headword,
+      startPos: token.startPos,
+      endPos: token.endPos,
+      partOfSpeech: token.partOfSpeech,
+      isKnown: token.isKnown,
+      isNPlusOneTarget: token.isNPlusOneTarget,
+    };
+    if (token.headwordReading !== undefined) projected.headwordReading = token.headwordReading;
+    if (token.pos1 !== undefined) projected.pos1 = token.pos1;
+    if (token.pos2 !== undefined) projected.pos2 = token.pos2;
+    if (token.pos3 !== undefined) projected.pos3 = token.pos3;
+    if (token.isNameMatch === true) projected.isNameMatch = true;
+    if (token.isUnparsedRun === true) projected.isUnparsedRun = true;
+    if (token.jlptLevel !== undefined) projected.jlptLevel = token.jlptLevel;
+    if (token.frequencyRank !== undefined) projected.frequencyRank = token.frequencyRank;
+    return projected;
+  });
+}

src/core/services/tokenizer/golden-corpus.test.ts

@@ -0,0 +1,69 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import test from 'node:test';
+import { tokenizeSubtitle } from '../tokenizer';
+import {
+  createReplayTokenizerDeps,
+  projectGoldenTokens,
+  type GoldenFixture,
+} from './golden-corpus-harness';
+
+// End-to-end regression corpus: each fixture replays recorded Yomitan backend
+// responses and raw MeCab tokens through the real tokenizeSubtitle pipeline
+// (scan-token merge, MeCab enrichment, frequency ranks, annotation stage,
+// noise suppression) and asserts the final annotated tokens.
+//
+// Record new fixtures with:
+//   bun run record-tokenizer-fixture:electron -- --name <slug> <text>
+// See __fixtures__/golden/README.md for the format and options.
+
+const FIXTURE_DIR = path.join(__dirname, '__fixtures__', 'golden');
+
+function loadFixtures(): GoldenFixture[] {
+  if (!fs.existsSync(FIXTURE_DIR)) {
+    return [];
+  }
+  return fs
+    .readdirSync(FIXTURE_DIR)
+    .filter((entry) => entry.endsWith('.json'))
+    .sort()
+    .map((entry) => {
+      const filePath = path.join(FIXTURE_DIR, entry);
+      const fixture = JSON.parse(fs.readFileSync(filePath, 'utf8')) as GoldenFixture;
+      assert.ok(fixture.name, `${entry}: fixture is missing "name"`);
+      assert.equal(
+        `${fixture.name}.json`,
+        entry,
+        `${entry}: fixture "name" must match its file name`,
+      );
+      return fixture;
+    });
+}
+
+const fixtures = loadFixtures();
+
+// Fixtures are not copied into dist by tsc; the corpus only runs from src.
+test('golden corpus has fixtures', { skip: !fs.existsSync(FIXTURE_DIR) }, () => {
+  assert.ok(
+    fixtures.length > 0,
+    `no golden fixtures found in ${FIXTURE_DIR}; record one with record-tokenizer-fixture:electron`,
+  );
+});
+
+for (const fixture of fixtures) {
+  const label = fixture.issueRefs?.length
+    ? `${fixture.name} (${fixture.issueRefs.join(', ')})`
+    : fixture.name;
+  test(`golden: ${label}`, async () => {
+    const deps = createReplayTokenizerDeps(fixture);
+    const subtitleData = await tokenizeSubtitle(fixture.input.text, deps);
+    assert.deepEqual(
+      projectGoldenTokens(subtitleData.tokens),
+      fixture.expected.tokens,
+      fixture.description
+        ? `${fixture.name}: ${fixture.description}`
+        : `${fixture.name}: annotated tokens diverged from recorded expectation`,
+    );
+  });
+}