test(tokenizer): add golden-file regression corpus with 11 fixtures (#157)

src/anki-integration/known-word-cache.test.ts

@@ -762,7 +762,10 @@ test('KnownWordCacheManager suppresses reading-only matches when disallowed', as
     // Reading-only match stays available for kana subtitle text…
     assert.equal(manager.isKnownWord('けいこく'), true);
     // …but a kanji token's reading (渓谷/けいこく) must not borrow 警告's.
-    assert.equal(manager.isKnownWord('けいこく', undefined, { allowReadingOnlyMatch: false }), false);
+    assert.equal(
+      manager.isKnownWord('けいこく', undefined, { allowReadingOnlyMatch: false }),
+      false,
+    );
     // Mined word texts still match regardless of the flag.
     assert.equal(manager.isKnownWord('警告', undefined, { allowReadingOnlyMatch: false }), true);
   } finally {

src/core/services/tokenizer/__fixtures__/golden/README.md

@@ -0,0 +1,72 @@
+# Tokenizer golden corpus
+
+End-to-end regression fixtures for the tokenizer/annotation pipeline. Each
+`.json` file captures one real subtitle line together with everything the
+pipeline consumed while tokenizing it live:
+
+- `recording.messages` — the raw Yomitan backend responses
+  (`chrome.runtime.sendMessage` level: `optionsGetFull`, `getDictionaryInfo`,
+  `parseText`, `termsFind`, `getTermFrequencies`), pruned to the fields the
+  injected scanning helpers actually read.
+- `recording.scripts` — sha256 → result pairs for each injected script, used
+  only as a replay fallback if a script cannot run in the vm.
+- `recording.mecab` — raw MeCab tokens keyed by the tokenized text.
+- `config` — annotation toggles plus fixture-local known words, JLPT levels,
+  and local frequency ranks (see `golden-corpus-harness.ts` for the simplified
+  known-word semantics shared by recorder and replay).
+- `expected.tokens` — the annotated tokens the full pipeline produced at
+  record time. **This is the assertion.** Review it before committing; it is a
+  characterization of current behavior, not a statement that the behavior is
+  ideal.
+
+`golden-corpus.test.ts` replays every fixture through the real
+`tokenizeSubtitle` (scan-token merge, MeCab enrichment, frequency ranks,
+annotation stage, noise suppression) by executing the real injected scripts in
+a `node:vm` sandbox against the recorded responses — no Electron, no
+dictionaries, no network.
+
+## Recording a fixture
+
+Requires a built Yomitan extension (`bun run build:yomitan`), your SubMiner
+Yomitan profile (`~/.config/SubMiner`), and MeCab:
+
+```sh
+bun run record-tokenizer-fixture:electron -- \
+  --name my-regression-case \
+  --issue "#123" \
+  --description "what behavior this pins down" \
+  --known-word 私 --known-word 要る:いる \
+  --jlpt 美しい=N4 \
+  そのまま字幕の一行
+```
+
+Run with `--help` for all options (annotation toggles, match mode, overwrite
+with `--force`, ...). The recorder prints the expected tokens for review and
+the fixture replays immediately via:
+
+```sh
+bun test src/core/services/tokenizer/golden-corpus.test.ts
+```
+
+## When a fixture fails
+
+A failure means the pipeline now produces different annotated tokens for that
+line. If the change is intentional, re-record the fixture with `--force`
+(same flags — they are stored in the fixture's `config`) and review the diff
+of `expected.tokens`; the diff _is_ the behavior change. If the change is not
+intentional, you found the regression before shipping it.
+
+Fixture dictionaries reflect whatever was installed in the recording profile,
+so re-recorded fixtures may differ in frequency ranks if dictionaries changed.
+
+## Comparing against stock Yomitan
+
+`bun run compare-yomitan-api:electron` diffs SubMiner's tokenization against a
+stock Yomitan instance reached through the
+[yomitan-api](https://github.com/yomidevs/yomitan-api) bridge
+(`http://127.0.0.1:19633`, enable "Yomitan API" in the browser extension's
+settings). Without arguments it compares every fixture text; pass sentences or
+`--file <path>` for ad-hoc checks. It reports segmentation, reading, and
+headword-form divergence and exits non-zero on any difference — useful for
+spot-checking that the pipeline still matches what Yomitan itself would
+produce, with your real browser profile and dictionaries as the reference.

src/core/services/tokenizer/golden-corpus-harness.ts

@@ -0,0 +1,475 @@
+import { createHash } from 'node:crypto';
+import vm from 'node:vm';
+import type {
+  FrequencyDictionaryMatchMode,
+  JlptLevel,
+  MergedToken,
+  NPlusOneMatchMode,
+  PartOfSpeech,
+  Token,
+} from '../../../types';
+import { createTokenizerDepsRuntime } from '../tokenizer';
+import type { TokenizerServiceDeps } from '../tokenizer';
+import { enrichTokensWithMecabPos1 } from './parser-enrichment-stage';
+
+// Golden-corpus fixtures capture a real subtitle line, the raw Yomitan
+// backend responses (chrome.runtime.sendMessage level) and raw MeCab tokens
+// observed while tokenizing it live, plus the annotated tokens the full
+// pipeline produced. Replay runs the real injected scanning scripts in a vm
+// against the recorded responses, so merge/enrichment/annotation/filtering
+// are exercised end-to-end without Electron or dictionaries.
+
+export interface GoldenFixtureKnownWord {
+  text: string;
+  reading?: string;
+}
+
+export interface GoldenFixtureConfig {
+  knownWords?: Array<string | GoldenFixtureKnownWord>;
+  knownWordMatchMode?: NPlusOneMatchMode;
+  jlptLevels?: Record<string, JlptLevel>;
+  localFrequencyRanks?: Record<string, number>;
+  knownWordsEnabled?: boolean;
+  nPlusOneEnabled?: boolean;
+  jlptEnabled?: boolean;
+  nameMatchEnabled?: boolean;
+  frequencyEnabled?: boolean;
+  frequencyMatchMode?: FrequencyDictionaryMatchMode;
+  minSentenceWordsForNPlusOne?: number;
+}
+
+export interface GoldenRecordedMessage {
+  action: string;
+  params: unknown;
+  /** Raw response object the page callback received: { result } or { error }. */
+  response: unknown;
+}
+
+export interface GoldenRecordedScript {
+  sha256: string;
+  /** Short classification of the script, for debugging only. */
+  marker: string;
+  result: unknown;
+}
+
+export interface GoldenFixtureRecording {
+  messages: GoldenRecordedMessage[];
+  scripts: GoldenRecordedScript[];
+  /** Raw MeCab tokens keyed by the exact text passed to tokenizeWithMecab. */
+  mecab: Record<string, Token[] | null>;
+}
+
+export interface GoldenExpectedToken {
+  surface: string;
+  reading: string;
+  headword: string;
+  headwordReading?: string;
+  startPos: number;
+  endPos: number;
+  partOfSpeech: PartOfSpeech;
+  pos1?: string;
+  pos2?: string;
+  pos3?: string;
+  isKnown: boolean;
+  isNPlusOneTarget: boolean;
+  isNameMatch?: true;
+  isUnparsedRun?: true;
+  jlptLevel?: JlptLevel;
+  frequencyRank?: number;
+}
+
+export interface GoldenFixture {
+  name: string;
+  description?: string;
+  issueRefs?: string[];
+  recordedAt?: string;
+  input: { text: string };
+  config: GoldenFixtureConfig;
+  recording: GoldenFixtureRecording;
+  expected: { tokens: GoldenExpectedToken[] | null };
+}
+
+// Only these backend actions are requested by the scanning/frequency scripts;
+// everything else in the recorded page traffic is Yomitan's own chatter.
+const GOLDEN_MESSAGE_ACTIONS: ReadonlySet<string> = new Set([
+  'optionsGetFull',
+  'getDictionaryInfo',
+  'parseText',
+  'termsFind',
+  'getTermFrequencies',
+]);
+
+// Field candidates appendDictionaryNames() in the injected helpers reads off
+// definitions/pronunciations — the only reason those arrays are consulted.
+const DICTIONARY_NAME_FIELDS = [
+  'dictionary',
+  'dictionaryName',
+  'name',
+  'title',
+  'dictionaryTitle',
+  'dictionaryAlias',
+] as const;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}
+
+function pruneToDictionaryNameFields(value: unknown): unknown {
+  if (!isRecord(value)) {
+    return {};
+  }
+  const pruned: Record<string, unknown> = {};
+  for (const field of DICTIONARY_NAME_FIELDS) {
+    if (typeof value[field] === 'string') {
+      pruned[field] = value[field];
+    }
+  }
+  return pruned;
+}
+
+// Fields getBestFrequencyRank() and the frequency grouping helpers read off a
+// termsFind frequency item, beyond the dictionary-name candidates.
+const FREQUENCY_ITEM_FIELDS = [
+  'headwordIndex',
+  'displayValue',
+  'displayValueParsed',
+  'frequency',
+  'dictionaryIndex',
+  'term',
+  'reading',
+] as const;
+
+function pruneFrequencyItem(item: unknown): unknown {
+  if (!isRecord(item)) {
+    return item;
+  }
+  const pruned = pruneToDictionaryNameFields(item) as Record<string, unknown>;
+  for (const field of FREQUENCY_ITEM_FIELDS) {
+    if (item[field] !== undefined) {
+      pruned[field] = item[field];
+    }
+  }
+  return pruned;
+}
+
+function pruneTermsFindEntry(entry: unknown): unknown {
+  if (!isRecord(entry)) {
+    return entry;
+  }
+  const pruned: Record<string, unknown> = { ...entry };
+  if (Array.isArray(entry.definitions)) {
+    pruned.definitions = entry.definitions.map(pruneToDictionaryNameFields);
+  }
+  if (Array.isArray(entry.pronunciations)) {
+    pruned.pronunciations = entry.pronunciations.map(pruneToDictionaryNameFields);
+  }
+  if (Array.isArray(entry.frequencies)) {
+    pruned.frequencies = entry.frequencies.map(pruneFrequencyItem);
+  }
+  return pruned;
+}
+
+function pruneMessageResult(action: string, result: unknown): unknown {
+  if (action === 'optionsGetFull' && isRecord(result)) {
+    const profiles = Array.isArray(result.profiles) ? result.profiles : [];
+    return {
+      profileCurrent: result.profileCurrent,
+      profiles: profiles.map((profile) => {
+        if (!isRecord(profile) || !isRecord(profile.options)) {
+          return {};
+        }
+        const options = profile.options;
+        return {
+          options: {
+            scanning: isRecord(options.scanning) ? { length: options.scanning.length } : {},
+            dictionaries: Array.isArray(options.dictionaries)
+              ? options.dictionaries.map((dictionary) =>
+                  isRecord(dictionary)
+                    ? {
+                        name: dictionary.name,
+                        enabled: dictionary.enabled,
+                        id: dictionary.id,
+                      }
+                    : {},
+                )
+              : [],
+          },
+        };
+      }),
+    };
+  }
+  if (action === 'getDictionaryInfo' && Array.isArray(result)) {
+    return result.map((entry) =>
+      isRecord(entry) ? { title: entry.title, frequencyMode: entry.frequencyMode } : {},
+    );
+  }
+  if (action === 'termsFind' && isRecord(result) && Array.isArray(result.dictionaryEntries)) {
+    return {
+      ...result,
+      dictionaryEntries: result.dictionaryEntries.map(pruneTermsFindEntry),
+    };
+  }
+  return result;
+}
+
+// Strip recorded page traffic down to what replay can ever request: drop
+// Yomitan's own background messages and the heavyweight response fields
+// (definition glossaries, full options blobs) the injected helpers never read.
+// Prune safety is verified by the replay tests themselves — a pruned fixture
+// must still reproduce its recorded expectation.
+export function pruneGoldenMessages(messages: GoldenRecordedMessage[]): GoldenRecordedMessage[] {
+  return messages
+    .filter((message) => GOLDEN_MESSAGE_ACTIONS.has(message.action))
+    .map((message) => {
+      if (isRecord(message.response) && 'result' in message.response) {
+        return {
+          ...message,
+          response: {
+            ...message.response,
+            result: pruneMessageResult(message.action, message.response.result),
+          },
+        };
+      }
+      return message;
+    });
+}
+
+export function hashInjectedScript(script: string): string {
+  return createHash('sha256').update(script).digest('hex');
+}
+
+export function classifyInjectedScript(script: string): string {
+  const markers = [
+    'optionsGetFull',
+    'parseText',
+    'termsFind',
+    'getTermFrequencies',
+    'setAllSettings',
+  ];
+  const found = markers.filter((marker) => script.includes(marker));
+  return found.length > 0 ? found.join('+') : 'unknown';
+}
+
+function stableStringify(value: unknown): string {
+  // Recorded params pass through JSON, which turns undefined into null.
+  return JSON.stringify(value ?? null, (_key, entry: unknown) => {
+    if (entry && typeof entry === 'object' && !Array.isArray(entry)) {
+      const record = entry as Record<string, unknown>;
+      const sorted: Record<string, unknown> = {};
+      for (const key of Object.keys(record).sort()) {
+        sorted[key] = record[key];
+      }
+      return sorted;
+    }
+    return entry;
+  });
+}
+
+export interface FixtureLookups {
+  isKnownWord: (
+    text: string,
+    reading?: string,
+    options?: { allowReadingOnlyMatch?: boolean },
+  ) => boolean;
+  getJlptLevel: (text: string) => JlptLevel | null;
+  getFrequencyRank: (term: string) => number | null;
+}
+
+// Deliberately simple known-word semantics (exact text, optional reading
+// constraint) shared by the recorder and the replay harness so both sides of
+// a fixture agree. The production reading-aware cache is exercised by its own
+// unit tests, not by the golden corpus.
+export function buildFixtureLookups(config: GoldenFixtureConfig): FixtureLookups {
+  const readingsByText = new Map<string, Set<string> | null>();
+  for (const entry of config.knownWords ?? []) {
+    const text = typeof entry === 'string' ? entry : entry.text;
+    const reading = typeof entry === 'string' ? undefined : entry.reading;
+    if (!reading) {
+      readingsByText.set(text, null);
+      continue;
+    }
+    const existing = readingsByText.get(text);
+    if (existing === null) {
+      continue;
+    }
+    const readings = existing ?? new Set<string>();
+    readings.add(reading);
+    readingsByText.set(text, readings);
+  }
+
+  return {
+    isKnownWord: (text, reading) => {
+      if (!readingsByText.has(text)) {
+        return false;
+      }
+      const readings = readingsByText.get(text);
+      if (readings === null || readings === undefined) {
+        return true;
+      }
+      return reading === undefined || readings.has(reading);
+    },
+    getJlptLevel: (text) => config.jlptLevels?.[text] ?? null,
+    getFrequencyRank: (term) => config.localFrequencyRanks?.[term] ?? null,
+  };
+}
+
+// Config-driven deps getters shared by the recorder and the replay harness so
+// a fixture is tokenized under identical toggles in both directions.
+export function fixtureConfigGetters(config: GoldenFixtureConfig) {
+  return {
+    getKnownWordMatchMode: () => config.knownWordMatchMode ?? ('headword' as NPlusOneMatchMode),
+    getKnownWordsEnabled: () => config.knownWordsEnabled !== false,
+    getNPlusOneEnabled: () => config.nPlusOneEnabled !== false,
+    getJlptEnabled: () => config.jlptEnabled !== false,
+    getNameMatchEnabled: () => config.nameMatchEnabled !== false,
+    getNameMatchImagesEnabled: () => false,
+    getFrequencyDictionaryEnabled: () => config.frequencyEnabled !== false,
+    getFrequencyDictionaryMatchMode: () =>
+      config.frequencyMatchMode ?? ('headword' as FrequencyDictionaryMatchMode),
+    getMinSentenceWordsForNPlusOne: () => config.minSentenceWordsForNPlusOne ?? 3,
+  };
+}
+
+interface ReplayMessageEntry extends GoldenRecordedMessage {
+  used: boolean;
+  paramsKey: string;
+}
+
+export interface ReplayMessageStore {
+  handle: (action: string, params: unknown) => unknown;
+  unusedCount: () => number;
+}
+
+export function createReplayMessageStore(messages: GoldenRecordedMessage[]): ReplayMessageStore {
+  const entries: ReplayMessageEntry[] = messages.map((message) => ({
+    ...message,
+    used: false,
+    paramsKey: stableStringify(message.params),
+  }));
+
+  return {
+    handle: (action, params) => {
+      const paramsKey = stableStringify(params);
+      const match =
+        entries.find((e) => !e.used && e.action === action && e.paramsKey === paramsKey) ??
+        entries.find((e) => !e.used && e.action === action) ??
+        entries.find((e) => e.action === action && e.paramsKey === paramsKey);
+      if (!match) {
+        throw new Error(
+          `golden fixture replay: no recorded response for action "${action}" params=${paramsKey}`,
+        );
+      }
+      match.used = true;
+      return match.response;
+    },
+    unusedCount: () => entries.filter((e) => !e.used).length,
+  };
+}
+
+async function runInjectedScriptInVm(script: string, store: ReplayMessageStore): Promise<unknown> {
+  return await vm.runInNewContext(script, {
+    chrome: {
+      runtime: {
+        lastError: null,
+        sendMessage: (
+          payload: { action?: string; params?: unknown },
+          callback: (response: unknown) => void,
+        ) => {
+          callback(store.handle(payload.action ?? '', payload.params));
+        },
+      },
+    },
+    Array,
+    Boolean,
+    Date,
+    Error,
+    JSON,
+    Map,
+    Math,
+    Number,
+    Object,
+    Promise,
+    RegExp,
+    Set,
+    String,
+  });
+}
+
+export function createReplayTokenizerDeps(fixture: GoldenFixture): TokenizerServiceDeps {
+  const store = createReplayMessageStore(fixture.recording.messages);
+  const scriptResults = new Map(
+    fixture.recording.scripts.map((entry) => [entry.sha256, entry] as const),
+  );
+
+  const parserWindow = {
+    isDestroyed: () => false,
+    webContents: {
+      executeJavaScript: async (script: string) => {
+        try {
+          return await runInjectedScriptInVm(script, store);
+        } catch (vmError) {
+          const recorded = scriptResults.get(hashInjectedScript(script));
+          if (recorded) {
+            return recorded.result;
+          }
+          throw new Error(
+            `golden fixture "${fixture.name}": injected script (${classifyInjectedScript(script)}) failed in vm replay and has no recorded script-level result: ${(vmError as Error).message}`,
+          );
+        }
+      },
+    },
+  };
+
+  const lookups = buildFixtureLookups(fixture.config);
+  const deps = createTokenizerDepsRuntime({
+    getYomitanExt: () => ({ id: 'golden-fixture-extension' }) as never,
+    getYomitanParserWindow: () => parserWindow as never,
+    setYomitanParserWindow: () => undefined,
+    getYomitanParserReadyPromise: () => null,
+    setYomitanParserReadyPromise: () => undefined,
+    getYomitanParserInitPromise: () => null,
+    setYomitanParserInitPromise: () => undefined,
+    isKnownWord: lookups.isKnownWord,
+    getJlptLevel: lookups.getJlptLevel,
+    getFrequencyRank: lookups.getFrequencyRank,
+    ...fixtureConfigGetters(fixture.config),
+    getMecabTokenizer: () => ({
+      tokenize: async (text: string) => fixture.recording.mecab[text] ?? null,
+    }),
+  });
+
+  return {
+    ...deps,
+    // Pin the synchronous enrichment stage so record and replay share one
+    // implementation and no worker spawns inside bun test.
+    enrichTokensWithMecab: async (tokens, mecabTokens) =>
+      enrichTokensWithMecabPos1(tokens, mecabTokens),
+  };
+}
+
+export function projectGoldenTokens(tokens: MergedToken[] | null): GoldenExpectedToken[] | null {
+  if (!tokens) {
+    return null;
+  }
+  return tokens.map((token) => {
+    const projected: GoldenExpectedToken = {
+      surface: token.surface,
+      reading: token.reading,
+      headword: token.headword,
+      startPos: token.startPos,
+      endPos: token.endPos,
+      partOfSpeech: token.partOfSpeech,
+      isKnown: token.isKnown,
+      isNPlusOneTarget: token.isNPlusOneTarget,
+    };
+    if (token.headwordReading !== undefined) projected.headwordReading = token.headwordReading;
+    if (token.pos1 !== undefined) projected.pos1 = token.pos1;
+    if (token.pos2 !== undefined) projected.pos2 = token.pos2;
+    if (token.pos3 !== undefined) projected.pos3 = token.pos3;
+    if (token.isNameMatch === true) projected.isNameMatch = true;
+    if (token.isUnparsedRun === true) projected.isUnparsedRun = true;
+    if (token.jlptLevel !== undefined) projected.jlptLevel = token.jlptLevel;
+    if (token.frequencyRank !== undefined) projected.frequencyRank = token.frequencyRank;
+    return projected;
+  });
+}

src/core/services/tokenizer/golden-corpus.test.ts

@@ -0,0 +1,69 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import test from 'node:test';
+import { tokenizeSubtitle } from '../tokenizer';
+import {
+  createReplayTokenizerDeps,
+  projectGoldenTokens,
+  type GoldenFixture,
+} from './golden-corpus-harness';
+
+// End-to-end regression corpus: each fixture replays recorded Yomitan backend
+// responses and raw MeCab tokens through the real tokenizeSubtitle pipeline
+// (scan-token merge, MeCab enrichment, frequency ranks, annotation stage,
+// noise suppression) and asserts the final annotated tokens.
+//
+// Record new fixtures with:
+//   bun run record-tokenizer-fixture:electron -- --name <slug> <text>
+// See __fixtures__/golden/README.md for the format and options.
+
+const FIXTURE_DIR = path.join(__dirname, '__fixtures__', 'golden');
+
+function loadFixtures(): GoldenFixture[] {
+  if (!fs.existsSync(FIXTURE_DIR)) {
+    return [];
+  }
+  return fs
+    .readdirSync(FIXTURE_DIR)
+    .filter((entry) => entry.endsWith('.json'))
+    .sort()
+    .map((entry) => {
+      const filePath = path.join(FIXTURE_DIR, entry);
+      const fixture = JSON.parse(fs.readFileSync(filePath, 'utf8')) as GoldenFixture;
+      assert.ok(fixture.name, `${entry}: fixture is missing "name"`);
+      assert.equal(
+        `${fixture.name}.json`,
+        entry,
+        `${entry}: fixture "name" must match its file name`,
+      );
+      return fixture;
+    });
+}
+
+const fixtures = loadFixtures();
+
+// Fixtures are not copied into dist by tsc; the corpus only runs from src.
+test('golden corpus has fixtures', { skip: !fs.existsSync(FIXTURE_DIR) }, () => {
+  assert.ok(
+    fixtures.length > 0,
+    `no golden fixtures found in ${FIXTURE_DIR}; record one with record-tokenizer-fixture:electron`,
+  );
+});
+
+for (const fixture of fixtures) {
+  const label = fixture.issueRefs?.length
+    ? `${fixture.name} (${fixture.issueRefs.join(', ')})`
+    : fixture.name;
+  test(`golden: ${label}`, async () => {
+    const deps = createReplayTokenizerDeps(fixture);
+    const subtitleData = await tokenizeSubtitle(fixture.input.text, deps);
+    assert.deepEqual(
+      projectGoldenTokens(subtitleData.tokens),
+      fixture.expected.tokens,
+      fixture.description
+        ? `${fixture.name}: ${fixture.description}`
+        : `${fixture.name}: annotated tokens diverged from recorded expectation`,
+    );
+  });
+}