test(tokenizer): add golden-file regression corpus with 11 fixtures (#157)

scripts/compare-yomitan-api.test.ts

@@ -0,0 +1,26 @@
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import test from 'node:test';
+
+const source = readFileSync('scripts/compare-yomitan-api.ts', 'utf8');
+
+test('creates the configured Yomitan user-data directory before Electron uses it', () => {
+  const setPathIndex = source.indexOf(
+    "electronModule.app.setPath('userData', options.yomitanUserDataPath)",
+  );
+  assert.notEqual(setPathIndex, -1);
+
+  const mkdirIndex = source.lastIndexOf(
+    'fs.mkdirSync(options.yomitanUserDataPath, { recursive: true })',
+    setPathIndex,
+  );
+  assert.ok(mkdirIndex > -1, 'must create the user-data directory');
+  assert.ok(mkdirIndex < setPathIndex, 'must create the directory before app.setPath');
+});
+
+test('lets output drain by using exitCode in the top-level completion handlers', () => {
+  const completionHandlers = source.slice(source.lastIndexOf('\nmain()'));
+  assert.match(completionHandlers, /process\.exitCode = process\.exitCode \?\? 0/);
+  assert.match(completionHandlers, /process\.exitCode = 1/);
+  assert.doesNotMatch(completionHandlers, /process\.exit\(/);
+});

scripts/compare-yomitan-api.ts

@@ -0,0 +1,589 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import process from 'node:process';
+
+import { createTokenizerDepsRuntime, tokenizeSubtitle } from '../src/core/services/tokenizer.js';
+import { selectYomitanParseTokens } from '../src/core/services/tokenizer/parser-selection-stage.js';
+import { enrichTokensWithMecabPos1 } from '../src/core/services/tokenizer/parser-enrichment-stage.js';
+import { resolveYomitanExtensionPath as resolveBuiltYomitanExtensionPath } from '../src/core/services/yomitan-extension-paths.js';
+import { MecabTokenizer } from '../src/mecab-tokenizer.js';
+import type { MergedToken } from '../src/types.js';
+
+import { fetchYomitanApi } from './yomitan-api-request.js';
+
+// Compares SubMiner's full tokenization pipeline against a stock Yomitan
+// instance reached through the yomitan-api native-messaging bridge
+// (https://github.com/yomidevs/yomitan-api, default http://127.0.0.1:19633).
+// The API response is raw parseText output; it is mapped through SubMiner's
+// own selectYomitanParseTokens so both sides share identical mapping
+// semantics and the diff isolates real segmentation/headword divergence.
+
+const DEFAULT_YOMITAN_USER_DATA_PATH = path.join(os.homedir(), '.config', 'SubMiner');
+const DEFAULT_API_URL = 'http://127.0.0.1:19633';
+const DEFAULT_SCAN_LENGTH = 40;
+const FIXTURE_DIR = path.join('src', 'core', 'services', 'tokenizer', '__fixtures__', 'golden');
+
+interface CliOptions {
+  sentences: string[];
+  fromFixtures: boolean;
+  apiUrl: string;
+  scanLength: number;
+  emitJson: boolean;
+  nameMatch: boolean;
+  yomitanExtensionPath?: string;
+  yomitanUserDataPath: string;
+  mecabCommand?: string;
+  mecabDictionaryPath?: string;
+}
+
+function printUsage(): void {
+  process.stdout.write(`Usage:
+  bun run compare-yomitan-api:electron -- [options] [sentence ...]
+
+  Compares SubMiner tokenization against a stock Yomitan instance via the
+  yomitan-api bridge. Without sentences, compares every golden-corpus fixture
+  text plus any --file lines.
+
+  --from-fixtures            Include golden-corpus fixture texts (default when
+                             no sentences are given).
+  --file <path>              Read additional sentences, one per line.
+  --api-url <url>            yomitan-api base URL (default: ${DEFAULT_API_URL}).
+  --scan-length <n>          Scan length for the API request (default: ${DEFAULT_SCAN_LENGTH}).
+  --name-match               Keep SubMiner's character-name pre-pass enabled.
+                             Off by default: stock Yomitan has no name regions,
+                             so it would only add noise to the diff.
+  --json                     Emit machine-readable JSON instead of a report.
+  --yomitan-extension <path> Path to built Yomitan extension directory.
+  --yomitan-user-data <path> Electron userData directory (default: ~/.config/SubMiner).
+  --mecab-command <path>     MeCab binary (default: mecab).
+  --mecab-dictionary <path>  MeCab dictionary directory.
+  -h, --help                 Show usage.
+
+  Exits 1 when any sentence diverges, 0 when everything matches.
+`);
+}
+
+function requireValue(args: string[], flag: string): string {
+  const next = args.shift();
+  if (!next) {
+    throw new Error(`Missing value for ${flag}`);
+  }
+  return next;
+}
+
+function parseCliArgs(argv: string[]): CliOptions {
+  const args = [...argv];
+  const sentences: string[] = [];
+  let fromFixtures = false;
+  let apiUrl = DEFAULT_API_URL;
+  let scanLength = DEFAULT_SCAN_LENGTH;
+  let emitJson = false;
+  let nameMatch = false;
+  let yomitanExtensionPath: string | undefined;
+  let yomitanUserDataPath = DEFAULT_YOMITAN_USER_DATA_PATH;
+  let mecabCommand: string | undefined;
+  let mecabDictionaryPath: string | undefined;
+
+  while (args.length > 0) {
+    const arg = args.shift();
+    if (!arg) break;
+
+    if (arg === '--help' || arg === '-h') {
+      printUsage();
+      process.exit(0);
+    }
+    if (arg === '--from-fixtures') {
+      fromFixtures = true;
+      continue;
+    }
+    if (arg === '--file') {
+      const filePath = requireValue(args, arg);
+      const lines = fs
+        .readFileSync(filePath, 'utf8')
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0);
+      sentences.push(...lines);
+      continue;
+    }
+    if (arg === '--api-url') {
+      apiUrl = requireValue(args, arg).replace(/\/$/, '');
+      continue;
+    }
+    if (arg === '--scan-length') {
+      const value = Number(requireValue(args, arg));
+      if (!Number.isInteger(value) || value <= 0) {
+        throw new Error('Invalid --scan-length value');
+      }
+      scanLength = value;
+      continue;
+    }
+    if (arg === '--json') {
+      emitJson = true;
+      continue;
+    }
+    if (arg === '--name-match') {
+      nameMatch = true;
+      continue;
+    }
+    if (arg === '--yomitan-extension') {
+      yomitanExtensionPath = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--yomitan-user-data') {
+      yomitanUserDataPath = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--mecab-command') {
+      mecabCommand = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--mecab-dictionary') {
+      mecabDictionaryPath = requireValue(args, arg);
+      continue;
+    }
+    if (arg.startsWith('-') && arg !== '-') {
+      throw new Error(`Unknown flag: ${arg}`);
+    }
+    sentences.push(arg);
+  }
+
+  if (sentences.length === 0) {
+    fromFixtures = true;
+  }
+
+  return {
+    sentences,
+    fromFixtures,
+    apiUrl,
+    scanLength,
+    emitJson,
+    nameMatch,
+    yomitanExtensionPath,
+    yomitanUserDataPath,
+    mecabCommand,
+    mecabDictionaryPath,
+  };
+}
+
+function loadFixtureSentences(): string[] {
+  if (!fs.existsSync(FIXTURE_DIR)) {
+    return [];
+  }
+  return fs
+    .readdirSync(FIXTURE_DIR)
+    .filter((entry) => entry.endsWith('.json'))
+    .sort()
+    .map((entry) => {
+      const fixture = JSON.parse(fs.readFileSync(path.join(FIXTURE_DIR, entry), 'utf8')) as {
+        input?: { text?: string };
+      };
+      return fixture.input?.text ?? '';
+    })
+    .filter((text) => text.length > 0);
+}
+
+function normalizeTokenizerText(text: string): string {
+  return text
+    .replace(/\r\n/g, '\n')
+    .replace(/\\N/g, '\n')
+    .replace(/\\n/g, '\n')
+    .trim()
+    .replace(/\n/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+interface FuriganaChunk {
+  startPos: number;
+  endPos: number;
+  reading: string;
+  headword: string;
+}
+
+// Walk the raw parseText content the same way the parse-result mapper does
+// (whole segments, char offsets accumulated across lines) and capture, per
+// chunk, the effective reading (kana chunks read as themselves) and the first
+// dictionary headword term. The mapper's token spans are authoritative for
+// grouping; these chunks supply the complete reading/lemma the legacy mapper
+// does not synthesize.
+function collectFuriganaChunks(parseResults: unknown): FuriganaChunk[] {
+  const chunks: FuriganaChunk[] = [];
+  if (!Array.isArray(parseResults)) {
+    return chunks;
+  }
+  const content = (parseResults[0] as { content?: unknown } | undefined)?.content;
+  if (!Array.isArray(content)) {
+    return chunks;
+  }
+  let charOffset = 0;
+  for (const line of content) {
+    if (!Array.isArray(line)) {
+      continue;
+    }
+    for (const segment of line as Array<Record<string, unknown>>) {
+      const text = typeof segment?.text === 'string' ? segment.text : '';
+      if (!text) {
+        continue;
+      }
+      const reading =
+        typeof segment.reading === 'string' && segment.reading.length > 0 ? segment.reading : text;
+      let headword = '';
+      const headwords = segment.headwords;
+      if (Array.isArray(headwords)) {
+        for (const row of headwords) {
+          const first = Array.isArray(row) ? (row[0] as { term?: unknown } | undefined) : undefined;
+          if (first && typeof first.term === 'string' && first.term.trim().length > 0) {
+            headword = first.term.trim();
+            break;
+          }
+        }
+      }
+      chunks.push({ startPos: charOffset, endPos: charOffset + text.length, reading, headword });
+      charOffset += text.length;
+    }
+  }
+  return chunks;
+}
+
+function overlayChunkMetadata(tokens: MergedToken[], chunks: FuriganaChunk[]): MergedToken[] {
+  return tokens.map((token) => {
+    const covered = chunks.filter(
+      (chunk) => chunk.startPos >= token.startPos && chunk.endPos <= token.endPos,
+    );
+    if (covered.length === 0) {
+      return token;
+    }
+    const reading = covered.map((chunk) => chunk.reading).join('');
+    const headword = covered.find((chunk) => chunk.headword)?.headword ?? token.surface;
+    return { ...token, reading, headword };
+  });
+}
+
+async function fetchApiTokens(
+  apiUrl: string,
+  text: string,
+  scanLength: number,
+): Promise<MergedToken[] | null> {
+  const response = await fetchYomitanApi(`${apiUrl}/tokenize`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ text, scanLength, parser: 'scanning-parser' }),
+  });
+  if (!response.ok) {
+    throw new Error(`yomitan-api /tokenize responded ${response.status}`);
+  }
+  const parseResults = (await response.json()) as unknown;
+  const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
+  if (!tokens) {
+    return tokens;
+  }
+  return overlayChunkMetadata(tokens, collectFuriganaChunks(parseResults));
+}
+
+interface ComparedToken {
+  surface: string;
+  reading: string;
+  headword: string;
+  startPos: number;
+  endPos: number;
+  frequencyRank?: number;
+}
+
+function toComparedTokens(tokens: MergedToken[] | null): ComparedToken[] {
+  return (tokens ?? []).map((token) => ({
+    surface: token.surface,
+    reading: token.reading,
+    headword: token.headword,
+    startPos: token.startPos,
+    endPos: token.endPos,
+    frequencyRank: token.frequencyRank,
+  }));
+}
+
+interface MismatchRegion {
+  subminer: ComparedToken[];
+  yomitan: ComparedToken[];
+}
+
+interface PairDiff {
+  surface: string;
+  field: 'headword' | 'reading';
+  subminer: string;
+  yomitan: string;
+}
+
+interface SentenceComparison {
+  text: string;
+  subminerTokens: ComparedToken[];
+  yomitanTokens: ComparedToken[];
+  segmentationMatches: boolean;
+  mismatchRegions: MismatchRegion[];
+  pairDiffs: PairDiff[];
+  error?: string;
+}
+
+// Walk both token lists in span order. Tokens whose [startPos, endPos) agree
+// are paired; disagreeing runs are grouped until the boundaries realign.
+function compareTokenLists(
+  subminer: ComparedToken[],
+  yomitan: ComparedToken[],
+): Pick<SentenceComparison, 'segmentationMatches' | 'mismatchRegions' | 'pairDiffs'> {
+  const mismatchRegions: MismatchRegion[] = [];
+  const pairDiffs: PairDiff[] = [];
+  let i = 0;
+  let j = 0;
+
+  while (i < subminer.length || j < yomitan.length) {
+    const a = subminer[i];
+    const b = yomitan[j];
+    if (a && b && a.startPos === b.startPos && a.endPos === b.endPos) {
+      if (a.headword !== b.headword) {
+        pairDiffs.push({
+          surface: a.surface,
+          field: 'headword',
+          subminer: a.headword,
+          yomitan: b.headword,
+        });
+      }
+      // An empty reading means "reads as its surface" (kana tokens, unparsed
+      // runs) — normalize before comparing.
+      if ((a.reading || a.surface) !== (b.reading || b.surface)) {
+        pairDiffs.push({
+          surface: a.surface,
+          field: 'reading',
+          subminer: a.reading,
+          yomitan: b.reading,
+        });
+      }
+      i += 1;
+      j += 1;
+      continue;
+    }
+
+    const region: MismatchRegion = { subminer: [], yomitan: [] };
+    let aEnd = a ? a.startPos : Number.MAX_SAFE_INTEGER;
+    let bEnd = b ? b.startPos : Number.MAX_SAFE_INTEGER;
+    do {
+      if (aEnd <= bEnd && i < subminer.length) {
+        const token = subminer[i];
+        if (token) {
+          region.subminer.push(token);
+          aEnd = token.endPos;
+        }
+        i += 1;
+      } else if (j < yomitan.length) {
+        const token = yomitan[j];
+        if (token) {
+          region.yomitan.push(token);
+          bEnd = token.endPos;
+        }
+        j += 1;
+      } else {
+        break;
+      }
+    } while (aEnd !== bEnd && (i < subminer.length || j < yomitan.length));
+    mismatchRegions.push(region);
+  }
+
+  return {
+    segmentationMatches: mismatchRegions.length === 0,
+    mismatchRegions,
+    pairDiffs,
+  };
+}
+
+function renderTokenRun(tokens: ComparedToken[]): string {
+  if (tokens.length === 0) {
+    return '(no tokens)';
+  }
+  return tokens.map((token) => token.surface).join('|');
+}
+
+function renderReport(comparisons: SentenceComparison[]): void {
+  let divergent = 0;
+  for (const comparison of comparisons) {
+    const clean =
+      !comparison.error && comparison.segmentationMatches && comparison.pairDiffs.length === 0;
+    if (!clean) {
+      divergent += 1;
+    }
+    process.stdout.write(`\n${clean ? 'MATCH' : 'DIFF '} ${comparison.text}\n`);
+    if (comparison.error) {
+      process.stdout.write(`  error: ${comparison.error}\n`);
+      continue;
+    }
+    if (clean) {
+      process.stdout.write(
+        `  ${comparison.subminerTokens.length} tokens: ${renderTokenRun(comparison.subminerTokens)}\n`,
+      );
+      continue;
+    }
+    if (!comparison.segmentationMatches) {
+      process.stdout.write('  segmentation:\n');
+      for (const region of comparison.mismatchRegions) {
+        process.stdout.write(`    subminer: ${renderTokenRun(region.subminer)}\n`);
+        process.stdout.write(`    yomitan:  ${renderTokenRun(region.yomitan)}\n`);
+      }
+    }
+    for (const diff of comparison.pairDiffs) {
+      process.stdout.write(
+        `  ${diff.surface}: ${diff.field} "${diff.subminer}" (subminer) vs "${diff.yomitan}" (yomitan)\n`,
+      );
+    }
+  }
+  process.stdout.write(
+    `\n${comparisons.length - divergent}/${comparisons.length} sentences match stock Yomitan\n`,
+  );
+}
+
+async function main(): Promise<void> {
+  const options = parseCliArgs(process.argv.slice(2));
+  const sentences = [...(options.fromFixtures ? loadFixtureSentences() : []), ...options.sentences];
+  if (sentences.length === 0) {
+    throw new Error('No sentences to compare (no fixtures found and none provided).');
+  }
+
+  // Fail fast with a clear message when the bridge is not running.
+  try {
+    const probe = await fetchYomitanApi(`${options.apiUrl}/serverVersion`, {
+      method: 'POST',
+      body: '{}',
+    });
+    if (!probe.ok) {
+      throw new Error(`status ${probe.status}`);
+    }
+  } catch (error) {
+    throw new Error(
+      `yomitan-api is not reachable at ${options.apiUrl} (${(error as Error).message}). ` +
+        'Ensure the browser is running and "Enable Yomitan API" is on in Yomitan settings.',
+    );
+  }
+
+  const electronModule = await import('electron').catch(() => null);
+  if (!electronModule?.app || !electronModule?.session) {
+    throw new Error('Electron runtime required; run via `bun run compare-yomitan-api:electron`.');
+  }
+
+  const mecabTokenizer = new MecabTokenizer({
+    mecabCommand: options.mecabCommand,
+    dictionaryPath: options.mecabDictionaryPath,
+  });
+  if (!(await mecabTokenizer.checkAvailability())) {
+    throw new Error('MeCab is not available on this system.');
+  }
+
+  if (typeof electronModule.app.setPath === 'function') {
+    fs.mkdirSync(options.yomitanUserDataPath, { recursive: true });
+    electronModule.app.setPath('userData', options.yomitanUserDataPath);
+  }
+  await electronModule.app.whenReady();
+
+  const extensionPath = resolveBuiltYomitanExtensionPath({
+    explicitPath: options.yomitanExtensionPath,
+    cwd: process.cwd(),
+  });
+  if (!extensionPath) {
+    throw new Error('No built Yomitan extension found; run `bun run build:yomitan` first.');
+  }
+  const extension = await electronModule.session.defaultSession.loadExtension(extensionPath, {
+    allowFileAccess: true,
+  });
+
+  let parserWindow: Electron.BrowserWindow | null = null;
+  let parserReadyPromise: Promise<void> | null = null;
+  let parserInitPromise: Promise<boolean> | null = null;
+
+  const runtimeDeps = createTokenizerDepsRuntime({
+    getYomitanExt: () => extension,
+    getYomitanParserWindow: () => parserWindow,
+    setYomitanParserWindow: (window) => {
+      parserWindow = window;
+    },
+    getYomitanParserReadyPromise: () => parserReadyPromise,
+    setYomitanParserReadyPromise: (promise) => {
+      parserReadyPromise = promise;
+    },
+    getYomitanParserInitPromise: () => parserInitPromise,
+    setYomitanParserInitPromise: (promise) => {
+      parserInitPromise = promise;
+    },
+    isKnownWord: () => false,
+    getKnownWordMatchMode: () => 'headword',
+    getJlptLevel: () => null,
+    getNameMatchEnabled: () => options.nameMatch,
+    getMecabTokenizer: () => ({
+      tokenize: (text: string) => mecabTokenizer.tokenize(text),
+    }),
+  });
+  const deps = {
+    ...runtimeDeps,
+    enrichTokensWithMecab: async (
+      tokens: Parameters<typeof enrichTokensWithMecabPos1>[0],
+      mecabTokens: Parameters<typeof enrichTokensWithMecabPos1>[1],
+    ) => enrichTokensWithMecabPos1(tokens, mecabTokens),
+  };
+
+  const comparisons: SentenceComparison[] = [];
+  try {
+    for (const sentence of sentences) {
+      const text = normalizeTokenizerText(sentence);
+      try {
+        const [subtitleData, apiTokens] = await Promise.all([
+          tokenizeSubtitle(text, deps),
+          fetchApiTokens(options.apiUrl, text, options.scanLength),
+        ]);
+        const subminerTokens = toComparedTokens(subtitleData.tokens);
+        const yomitanTokens = toComparedTokens(apiTokens);
+        comparisons.push({
+          text,
+          subminerTokens,
+          yomitanTokens,
+          ...compareTokenLists(subminerTokens, yomitanTokens),
+        });
+      } catch (error) {
+        comparisons.push({
+          text,
+          subminerTokens: [],
+          yomitanTokens: [],
+          segmentationMatches: false,
+          mismatchRegions: [],
+          pairDiffs: [],
+          error: (error as Error).message,
+        });
+      }
+    }
+  } finally {
+    if (parserWindow) {
+      const window = parserWindow as Electron.BrowserWindow;
+      if (!window.isDestroyed()) {
+        window.destroy();
+      }
+    }
+    electronModule.app.quit();
+  }
+
+  if (options.emitJson) {
+    process.stdout.write(`${JSON.stringify(comparisons, null, 2)}\n`);
+  } else {
+    renderReport(comparisons);
+  }
+
+  const anyDivergence = comparisons.some(
+    (comparison) =>
+      comparison.error !== undefined ||
+      !comparison.segmentationMatches ||
+      comparison.pairDiffs.length > 0,
+  );
+  process.exitCode = anyDivergence ? 1 : 0;
+}
+
+main()
+  .then(() => {
+    process.exitCode = process.exitCode ?? 0;
+  })
+  .catch((error) => {
+    console.error(`Error: ${(error as Error).message}`);
+    process.exitCode = 1;
+  });

scripts/record-tokenizer-fixture.ts

@@ -0,0 +1,496 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import process from 'node:process';
+
+import { createTokenizerDepsRuntime, tokenizeSubtitle } from '../src/core/services/tokenizer.js';
+import {
+  buildFixtureLookups,
+  classifyInjectedScript,
+  fixtureConfigGetters,
+  hashInjectedScript,
+  projectGoldenTokens,
+  pruneGoldenMessages,
+} from '../src/core/services/tokenizer/golden-corpus-harness.js';
+import type {
+  GoldenFixture,
+  GoldenFixtureConfig,
+  GoldenRecordedMessage,
+  GoldenRecordedScript,
+} from '../src/core/services/tokenizer/golden-corpus-harness.js';
+import { enrichTokensWithMecabPos1 } from '../src/core/services/tokenizer/parser-enrichment-stage.js';
+import { resolveYomitanExtensionPath as resolveBuiltYomitanExtensionPath } from '../src/core/services/yomitan-extension-paths.js';
+import { MecabTokenizer } from '../src/mecab-tokenizer.js';
+import type { JlptLevel, Token } from '../src/types.js';
+
+const DEFAULT_YOMITAN_USER_DATA_PATH = path.join(os.homedir(), '.config', 'SubMiner');
+const DEFAULT_FIXTURE_DIR = path.join(
+  'src',
+  'core',
+  'services',
+  'tokenizer',
+  '__fixtures__',
+  'golden',
+);
+const JLPT_LEVELS: ReadonlySet<string> = new Set(['N1', 'N2', 'N3', 'N4', 'N5']);
+
+interface CliOptions {
+  name: string;
+  text: string;
+  description?: string;
+  issueRefs: string[];
+  config: GoldenFixtureConfig;
+  outDir: string;
+  force: boolean;
+  yomitanExtensionPath?: string;
+  yomitanUserDataPath: string;
+  mecabCommand?: string;
+  mecabDictionaryPath?: string;
+}
+
+function printUsage(): void {
+  process.stdout.write(`Usage:
+  bun run record-tokenizer-fixture:electron -- --name <slug> [options] <subtitle text>
+
+  Records a golden tokenizer fixture: raw Yomitan backend responses, raw MeCab
+  tokens, and the annotated tokens the full pipeline produced for the text.
+  Review the "expected" block before committing — it becomes the assertion.
+
+  --name <slug>              Fixture name (file becomes <slug>.json). Required.
+  --description <text>       What behavior this fixture pins down.
+  --issue <ref>              Related issue/PR ref, repeatable (e.g. --issue "#156").
+  --known-word <w[:reading]> Mark a word known, repeatable.
+  --jlpt <term=level>        JLPT level for a term, repeatable (level N1..N5).
+  --local-frequency <t=rank> Local frequency-dictionary rank, repeatable.
+  --match-mode <mode>        Known-word match mode: headword|surface.
+  --min-sentence-words <n>   Minimum sentence words for N+1.
+  --no-known-words           Disable known-word annotation.
+  --no-nplusone              Disable N+1 targeting.
+  --no-jlpt                  Disable JLPT annotation.
+  --no-name-match            Disable character-name matching.
+  --no-frequency             Disable frequency annotation.
+  --out-dir <path>           Fixture directory (default: ${DEFAULT_FIXTURE_DIR}).
+  --force                    Overwrite an existing fixture file.
+  --yomitan-extension <path> Path to built Yomitan extension directory.
+  --yomitan-user-data <path> Electron userData directory (default: ~/.config/SubMiner).
+  --mecab-command <path>     MeCab binary (default: mecab).
+  --mecab-dictionary <path>  MeCab dictionary directory.
+  -h, --help                 Show usage.
+`);
+}
+
+function requireValue(args: string[], flag: string): string {
+  const next = args.shift();
+  if (!next) {
+    throw new Error(`Missing value for ${flag}`);
+  }
+  return next;
+}
+
+function parseCliArgs(argv: string[]): CliOptions {
+  const args = [...argv];
+  const inputParts: string[] = [];
+  const issueRefs: string[] = [];
+  const knownWords: GoldenFixtureConfig['knownWords'] = [];
+  const jlptLevels: Record<string, JlptLevel> = {};
+  const localFrequencyRanks: Record<string, number> = {};
+  const config: GoldenFixtureConfig = {};
+  let name: string | undefined;
+  let description: string | undefined;
+  let outDir = DEFAULT_FIXTURE_DIR;
+  let force = false;
+  let yomitanExtensionPath: string | undefined;
+  let yomitanUserDataPath = DEFAULT_YOMITAN_USER_DATA_PATH;
+  let mecabCommand: string | undefined;
+  let mecabDictionaryPath: string | undefined;
+
+  while (args.length > 0) {
+    const arg = args.shift();
+    if (!arg) break;
+
+    if (arg === '--help' || arg === '-h') {
+      printUsage();
+      process.exit(0);
+    }
+    if (arg === '--name') {
+      name = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--description') {
+      description = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--issue') {
+      issueRefs.push(requireValue(args, arg));
+      continue;
+    }
+    if (arg === '--known-word') {
+      const value = requireValue(args, arg);
+      const separator = value.indexOf(':');
+      knownWords.push(
+        separator === -1
+          ? value
+          : { text: value.slice(0, separator), reading: value.slice(separator + 1) },
+      );
+      continue;
+    }
+    if (arg === '--jlpt') {
+      const value = requireValue(args, arg);
+      const separator = value.indexOf('=');
+      const level = separator === -1 ? '' : value.slice(separator + 1);
+      if (separator === -1 || !JLPT_LEVELS.has(level)) {
+        throw new Error(`Invalid --jlpt value "${value}"; expected <term>=N1..N5`);
+      }
+      jlptLevels[value.slice(0, separator)] = level as JlptLevel;
+      continue;
+    }
+    if (arg === '--local-frequency') {
+      const value = requireValue(args, arg);
+      const separator = value.indexOf('=');
+      const rank = separator === -1 ? Number.NaN : Number(value.slice(separator + 1));
+      if (!Number.isInteger(rank) || rank <= 0) {
+        throw new Error(`Invalid --local-frequency value "${value}"; expected <term>=<rank>`);
+      }
+      localFrequencyRanks[value.slice(0, separator)] = rank;
+      continue;
+    }
+    if (arg === '--match-mode') {
+      const value = requireValue(args, arg);
+      if (value !== 'headword' && value !== 'surface') {
+        throw new Error(`Invalid --match-mode "${value}"; expected headword|surface`);
+      }
+      config.knownWordMatchMode = value;
+      config.frequencyMatchMode = value;
+      continue;
+    }
+    if (arg === '--min-sentence-words') {
+      const value = Number(requireValue(args, arg));
+      if (!Number.isInteger(value) || value < 0) {
+        throw new Error('Invalid --min-sentence-words value');
+      }
+      config.minSentenceWordsForNPlusOne = value;
+      continue;
+    }
+    if (arg === '--no-known-words') {
+      config.knownWordsEnabled = false;
+      continue;
+    }
+    if (arg === '--no-nplusone') {
+      config.nPlusOneEnabled = false;
+      continue;
+    }
+    if (arg === '--no-jlpt') {
+      config.jlptEnabled = false;
+      continue;
+    }
+    if (arg === '--no-name-match') {
+      config.nameMatchEnabled = false;
+      continue;
+    }
+    if (arg === '--no-frequency') {
+      config.frequencyEnabled = false;
+      continue;
+    }
+    if (arg === '--out-dir') {
+      outDir = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--force') {
+      force = true;
+      continue;
+    }
+    if (arg === '--yomitan-extension') {
+      yomitanExtensionPath = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--yomitan-user-data') {
+      yomitanUserDataPath = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--mecab-command') {
+      mecabCommand = requireValue(args, arg);
+      continue;
+    }
+    if (arg === '--mecab-dictionary') {
+      mecabDictionaryPath = requireValue(args, arg);
+      continue;
+    }
+    if (arg.startsWith('-') && arg !== '-') {
+      throw new Error(`Unknown flag: ${arg}`);
+    }
+    inputParts.push(arg);
+  }
+
+  if (!name || !/^[a-z0-9][a-z0-9-]*$/.test(name)) {
+    throw new Error('A kebab-case --name is required (e.g. --name kureru-lexical-keep)');
+  }
+  const text = inputParts.join(' ').trim();
+  if (!text) {
+    throw new Error('Provide the subtitle text to record as positional arguments.');
+  }
+
+  if (knownWords.length > 0) config.knownWords = knownWords;
+  if (Object.keys(jlptLevels).length > 0) config.jlptLevels = jlptLevels;
+  if (Object.keys(localFrequencyRanks).length > 0) {
+    config.localFrequencyRanks = localFrequencyRanks;
+  }
+
+  return {
+    name,
+    text,
+    description,
+    issueRefs,
+    config,
+    outDir,
+    force,
+    yomitanExtensionPath,
+    yomitanUserDataPath,
+    mecabCommand,
+    mecabDictionaryPath,
+  };
+}
+
+const RECORDER_SHIM_SCRIPT = `
+(() => {
+  const g = globalThis;
+  if (g.__subminerGoldenRecorderInstalled) { return true; }
+  g.__subminerGoldenRecorderInstalled = true;
+  g.__subminerGoldenMessages = [];
+  const runtime = chrome.runtime;
+  const original = runtime.sendMessage.bind(runtime);
+  runtime.sendMessage = (payload, callback) => {
+    return original(payload, (response) => {
+      try {
+        g.__subminerGoldenMessages.push(JSON.parse(JSON.stringify({
+          action: payload && payload.action ? payload.action : '',
+          params: payload && typeof payload.params !== 'undefined' ? payload.params : null,
+          response: typeof response !== 'undefined' ? response : null,
+        })));
+      } catch {}
+      if (callback) { callback(response); }
+    });
+  };
+  return true;
+})();
+`;
+
+const RECORDER_DRAIN_SCRIPT = `
+(() => {
+  const g = globalThis;
+  const log = g.__subminerGoldenMessages || [];
+  g.__subminerGoldenMessages = [];
+  return JSON.stringify(log);
+})();
+`;
+
+interface RecordingSink {
+  messages: GoldenRecordedMessage[];
+  scripts: GoldenRecordedScript[];
+  mecab: Record<string, Token[] | null>;
+}
+
+function patchParserWindowForRecording(
+  window: Electron.BrowserWindow | null,
+  patched: WeakSet<object>,
+  sink: RecordingSink,
+): void {
+  if (!window || window.isDestroyed()) {
+    return;
+  }
+  const webContents = window.webContents as unknown as {
+    executeJavaScript: (script: string, userGesture?: boolean) => Promise<unknown>;
+  };
+  if (patched.has(webContents)) {
+    return;
+  }
+  patched.add(webContents);
+
+  const original = webContents.executeJavaScript.bind(webContents);
+  let shimReady: Promise<unknown> | null = null;
+  const drain = async (): Promise<void> => {
+    try {
+      const raw = (await original(RECORDER_DRAIN_SCRIPT, true)) as string;
+      const drained = JSON.parse(raw) as GoldenRecordedMessage[];
+      sink.messages.push(...drained);
+    } catch (error) {
+      process.stderr.write(
+        `Warning: failed to drain recorded messages: ${(error as Error).message}\n`,
+      );
+    }
+  };
+
+  webContents.executeJavaScript = async (script: string, userGesture?: boolean) => {
+    if (!shimReady) {
+      shimReady = original(RECORDER_SHIM_SCRIPT, true);
+    }
+    try {
+      await shimReady;
+    } catch {
+      shimReady = null;
+    }
+    let result: unknown;
+    try {
+      result = await original(script, userGesture);
+    } finally {
+      await drain();
+    }
+    sink.scripts.push({
+      sha256: hashInjectedScript(script),
+      marker: classifyInjectedScript(script),
+      result: result === undefined ? null : result,
+    });
+    return result;
+  };
+}
+
+async function main(): Promise<void> {
+  const options = parseCliArgs(process.argv.slice(2));
+  const outPath = path.join(options.outDir, `${options.name}.json`);
+  if (fs.existsSync(outPath) && !options.force) {
+    throw new Error(`${outPath} already exists; pass --force to overwrite.`);
+  }
+
+  const electronModule = await import('electron').catch(() => null);
+  if (!electronModule?.app || !electronModule?.session) {
+    throw new Error(
+      'Electron runtime is required; run via `bun run record-tokenizer-fixture:electron`.',
+    );
+  }
+
+  const mecabTokenizer = new MecabTokenizer({
+    mecabCommand: options.mecabCommand,
+    dictionaryPath: options.mecabDictionaryPath,
+  });
+  if (!(await mecabTokenizer.checkAvailability())) {
+    throw new Error('MeCab is not available on this system.');
+  }
+
+  if (typeof electronModule.app.setPath === 'function') {
+    electronModule.app.setPath('userData', options.yomitanUserDataPath);
+  }
+  await electronModule.app.whenReady();
+
+  const extensionPath = resolveBuiltYomitanExtensionPath({
+    explicitPath: options.yomitanExtensionPath,
+    cwd: process.cwd(),
+  });
+  if (!extensionPath) {
+    throw new Error('No built Yomitan extension found; run `bun run build:yomitan` first.');
+  }
+  const extension = await electronModule.session.defaultSession.loadExtension(extensionPath, {
+    allowFileAccess: true,
+  });
+
+  const sink: RecordingSink = { messages: [], scripts: [], mecab: {} };
+  const patched = new WeakSet<object>();
+  let parserWindow: Electron.BrowserWindow | null = null;
+  let parserReadyPromise: Promise<void> | null = null;
+  let parserInitPromise: Promise<boolean> | null = null;
+
+  const lookups = buildFixtureLookups(options.config);
+  const runtimeDeps = createTokenizerDepsRuntime({
+    getYomitanExt: () => extension,
+    getYomitanParserWindow: () => {
+      patchParserWindowForRecording(parserWindow, patched, sink);
+      return parserWindow;
+    },
+    setYomitanParserWindow: (window) => {
+      parserWindow = window;
+      patchParserWindowForRecording(parserWindow, patched, sink);
+    },
+    getYomitanParserReadyPromise: () => parserReadyPromise,
+    setYomitanParserReadyPromise: (promise) => {
+      parserReadyPromise = promise;
+    },
+    getYomitanParserInitPromise: () => parserInitPromise,
+    setYomitanParserInitPromise: (promise) => {
+      parserInitPromise = promise;
+    },
+    isKnownWord: lookups.isKnownWord,
+    getJlptLevel: lookups.getJlptLevel,
+    getFrequencyRank: lookups.getFrequencyRank,
+    ...fixtureConfigGetters(options.config),
+    getMecabTokenizer: () => ({
+      tokenize: async (text: string) => {
+        const rawTokens = await mecabTokenizer.tokenize(text);
+        sink.mecab[text] = rawTokens;
+        return rawTokens;
+      },
+    }),
+  });
+  const deps = {
+    ...runtimeDeps,
+    // Same pinned enrichment implementation the replay harness uses.
+    enrichTokensWithMecab: async (
+      tokens: Parameters<typeof enrichTokensWithMecabPos1>[0],
+      mecabTokens: Parameters<typeof enrichTokensWithMecabPos1>[1],
+    ) => enrichTokensWithMecabPos1(tokens, mecabTokens),
+  };
+
+  try {
+    const subtitleData = await tokenizeSubtitle(options.text, deps);
+    const expectedTokens = projectGoldenTokens(subtitleData.tokens);
+
+    const prunedMessages = pruneGoldenMessages(sink.messages);
+    const fixture: GoldenFixture = {
+      name: options.name,
+      ...(options.description ? { description: options.description } : {}),
+      ...(options.issueRefs.length > 0 ? { issueRefs: options.issueRefs } : {}),
+      recordedAt: new Date().toISOString(),
+      input: { text: options.text },
+      config: options.config,
+      recording: {
+        messages: prunedMessages,
+        scripts: sink.scripts,
+        mecab: sink.mecab,
+      },
+      expected: { tokens: expectedTokens },
+    };
+
+    fs.mkdirSync(options.outDir, { recursive: true });
+    fs.writeFileSync(outPath, `${JSON.stringify(fixture, null, 2)}\n`);
+
+    process.stdout.write(`Recorded ${outPath}\n`);
+    process.stdout.write(
+      `  messages=${prunedMessages.length} scripts=${sink.scripts.length} mecabTexts=${Object.keys(sink.mecab).length}\n`,
+    );
+    process.stdout.write('\nExpected tokens (review before committing):\n');
+    if (!expectedTokens || expectedTokens.length === 0) {
+      process.stdout.write('  (none)\n');
+    } else {
+      for (const token of expectedTokens) {
+        const flags = [
+          token.isKnown ? 'known' : null,
+          token.isNPlusOneTarget ? 'n+1' : null,
+          token.isNameMatch ? 'name' : null,
+          token.isUnparsedRun ? 'unparsed' : null,
+          token.jlptLevel ?? null,
+          token.frequencyRank !== undefined ? `freq=${token.frequencyRank}` : null,
+        ]
+          .filter(Boolean)
+          .join(', ');
+        process.stdout.write(
+          `  ${token.surface} -> ${token.headword} (${token.reading})${flags ? ` [${flags}]` : ''}\n`,
+        );
+      }
+    }
+    process.stdout.write(
+      '\nVerify replay with: bun test src/core/services/tokenizer/golden-corpus.test.ts\n',
+    );
+  } finally {
+    if (parserWindow) {
+      const window = parserWindow as Electron.BrowserWindow;
+      if (!window.isDestroyed()) {
+        window.destroy();
+      }
+    }
+    electronModule.app.quit();
+  }
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((error) => {
+    console.error(`Error: ${(error as Error).message}`);
+    process.exit(1);
+  });

scripts/yomitan-api-request.test.ts

@@ -0,0 +1,69 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { fetchYomitanApi } from './yomitan-api-request.js';
+
+function createStalledServer(): ReturnType<typeof Bun.serve> {
+  return Bun.serve({
+    hostname: '127.0.0.1',
+    port: 0,
+    fetch: () => new Promise<Response>(() => {}),
+  });
+}
+
+test('fetchYomitanApi reports a clear timeout when the bridge stalls', async () => {
+  const server = createStalledServer();
+
+  try {
+    await assert.rejects(
+      () => fetchYomitanApi(`${server.url}tokenize`, { method: 'POST' }, 20),
+      /Yomitan API request timed out after 20ms/,
+    );
+  } finally {
+    server.stop(true);
+  }
+});
+
+test('fetchYomitanApi preserves an init cancellation signal', async () => {
+  const server = createStalledServer();
+  const controller = new AbortController();
+  const reason = new Error('cancelled by caller');
+
+  try {
+    const request = fetchYomitanApi(`${server.url}tokenize`, { signal: controller.signal }, 50);
+    controller.abort(reason);
+    await assert.rejects(request, (error) => error === reason);
+  } finally {
+    server.stop(true);
+  }
+});
+
+test('fetchYomitanApi preserves a Request input cancellation signal', async () => {
+  const server = createStalledServer();
+  const controller = new AbortController();
+  const reason = new Error('cancelled by request');
+  const input = new Request(`${server.url}tokenize`, { signal: controller.signal });
+
+  try {
+    const request = fetchYomitanApi(input, undefined, 50);
+    controller.abort(reason);
+    await assert.rejects(request, (error) => error === reason);
+  } finally {
+    server.stop(true);
+  }
+});
+
+test('fetchYomitanApi accepts a null init cancellation signal', async () => {
+  const server = Bun.serve({
+    hostname: '127.0.0.1',
+    port: 0,
+    fetch: () => new Response('ok'),
+  });
+
+  try {
+    const response = await fetchYomitanApi(`${server.url}tokenize`, { signal: null }, 50);
+    assert.equal(response.status, 200);
+  } finally {
+    server.stop(true);
+  }
+});

scripts/yomitan-api-request.ts

@@ -0,0 +1,26 @@
+const DEFAULT_YOMITAN_API_TIMEOUT_MS = 10_000;
+
+export async function fetchYomitanApi(
+  input: RequestInfo | URL,
+  init?: RequestInit,
+  timeoutMs = DEFAULT_YOMITAN_API_TIMEOUT_MS,
+): Promise<Response> {
+  const controller = new AbortController();
+  const requestSignal = input instanceof Request ? input.signal : undefined;
+  const signals = [controller.signal, init?.signal, requestSignal].filter(
+    (signal): signal is AbortSignal => signal != null,
+  );
+  const signal = signals.length === 1 ? controller.signal : AbortSignal.any(signals);
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+  try {
+    return await fetch(input, { ...init, signal });
+  } catch (error) {
+    if (controller.signal.aborted) {
+      throw new Error(`Yomitan API request timed out after ${timeoutMs}ms`, { cause: error });
+    }
+    throw error;
+  } finally {
+    clearTimeout(timeout);
+  }
+}