test: cut low-value tests, rewrite the rest around behavior, parallelize lanes (#285)

src/core/services/tokenizer/parser-enrichment-stage.test.ts

@@ -99,77 +99,3 @@ test('enrichTokensWithMecabPos1 passes through unchanged when mecab tokens are n
   const emptyResult = enrichTokensWithMecabPos1(tokens, []);
   assert.strictEqual(emptyResult, tokens);
 });
-
-test('enrichTokensWithMecabPos1 avoids repeated full scans over distant mecab surfaces', () => {
-  const tokens = Array.from({ length: 12 }, (_, index) =>
-    makeToken({ surface: `w${index}`, startPos: index, endPos: index + 1, pos1: '' }),
-  );
-  const mecabTokens = tokens.map((token) =>
-    makeToken({
-      surface: token.surface,
-      startPos: token.startPos,
-      endPos: token.endPos,
-      pos1: '名詞',
-    }),
-  );
-
-  let distantSurfaceReads = 0;
-  const distantToken = makeToken({ surface: '遠', startPos: 500, endPos: 501, pos1: '記号' });
-  Object.defineProperty(distantToken, 'surface', {
-    configurable: true,
-    get() {
-      distantSurfaceReads += 1;
-      if (distantSurfaceReads > 3) {
-        throw new Error('repeated full scan detected');
-      }
-      return '遠';
-    },
-  });
-  mecabTokens.push(distantToken);
-
-  const enriched = enrichTokensWithMecabPos1(tokens, mecabTokens);
-  assert.equal(enriched.length, tokens.length);
-  for (const token of enriched) {
-    assert.equal(token.pos1, '名詞');
-  }
-});
-
-test('enrichTokensWithMecabPos1 avoids repeated active-candidate filter scans', () => {
-  const tokens = Array.from({ length: 8 }, (_, index) =>
-    makeToken({ surface: `u${index}`, startPos: index, endPos: index + 1, pos1: '' }),
-  );
-  const mecabTokens = [
-    makeToken({ surface: 'SENTINEL', startPos: 0, endPos: 100, pos1: '記号' }),
-    ...tokens.map((token, index) =>
-      makeToken({
-        surface: `m${index}`,
-        startPos: token.startPos,
-        endPos: token.endPos,
-        pos1: '名詞',
-      }),
-    ),
-  ];
-
-  let sentinelFilterCalls = 0;
-  const originalFilter = Array.prototype.filter;
-  Array.prototype.filter = function filterWithSentinelCheck(
-    this: unknown[],
-    ...args: any[]
-  ): any[] {
-    const target = this as Array<{ surface?: string }>;
-    if (target.some((candidate) => candidate?.surface === 'SENTINEL')) {
-      sentinelFilterCalls += 1;
-      if (sentinelFilterCalls > 2) {
-        throw new Error('repeated active candidate filter scan detected');
-      }
-    }
-    return (originalFilter as (...params: any[]) => any[]).apply(this, args);
-  } as typeof Array.prototype.filter;
-
-  try {
-    const enriched = enrichTokensWithMecabPos1(tokens, mecabTokens);
-    assert.equal(enriched.length, tokens.length);
-  } finally {
-    Array.prototype.filter = originalFilter;
-  }
-});

src/core/services/tokenizer/parser-selection-stage.test.ts

@@ -175,18 +175,6 @@ test('still drops punctuation-only and whitespace-only unparsed runs', () => {
   assert.equal(tokens?.map((token) => token.surface).join(','), '猫,犬');
 });
 
-test('candidate with only unparsed tokens still yields no dictionary match', () => {
-  const parseResults = [
-    makeParseItem('scanning-parser', [
-      [{ text: '戻', reading: '' }],
-      [{ text: '轟', reading: '' }],
-    ]),
-  ];
-
-  const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword');
-  assert.equal(tokens, null);
-});
-
 test('prefers the longest dictionary headword across merged segments', () => {
   const parseResults = [
     makeParseItem('scanning-parser', [
@@ -379,50 +367,30 @@ test('merges trailing katakana continuation without headword into previous token
 // is merged with trailing function particles (e.g. かかってこいよ → headword かかってくる).
 // When a competing multi-token candidate splits content and function separately, the
 // multi-token candidate should win so the content token remains frequency-highlightable.
-test('multi-token candidate beats single merged content+function token candidate (frequency regression)', () => {
-  // Candidate A: single merged token — content verb fused with trailing sentence-final particle
-  // This is the "bad" candidate: downstream annotation would exclude frequency for the whole
-  // token because the merged pos1 would contain a function-word component.
+test('multi-token candidate beats single merged content+function token candidate in either input order (frequency regression)', () => {
+  // Merged candidate: content verb fused with the trailing sentence-final particle. Downstream
+  // annotation would exclude frequency for the whole token because its merged pos1 contains a
+  // function-word component.
   const mergedCandidate = makeParseItem('scanning-parser', [
     [{ text: 'かかってこいよ', reading: 'かかってこいよ', headword: 'かかってくる' }],
   ]);
-
-  // Candidate B: two tokens — content verb surface + particle separately.
-  // The content token is frequency-eligible on its own.
+  // Split candidate: the content token stays frequency-eligible on its own.
   const splitCandidate = makeParseItem('scanning-parser', [
     [{ text: 'かかってこい', reading: 'かかってこい', headword: 'かかってくる' }],
     [{ text: 'よ', reading: 'よ', headword: 'よ' }],
   ]);
 
-  // When merged candidate comes first in the array, multi-token split still wins.
-  const tokens = selectYomitanParseTokens(
+  for (const candidates of [
     [mergedCandidate, splitCandidate],
-    () => false,
-    'headword',
-  );
-  assert.equal(tokens?.length, 2);
-  assert.equal(tokens?.[0]?.surface, 'かかってこい');
-  assert.equal(tokens?.[0]?.headword, 'かかってくる');
-  assert.equal(tokens?.[1]?.surface, 'よ');
-});
-
-test('multi-token candidate beats single merged content+function token regardless of input order', () => {
-  const mergedCandidate = makeParseItem('scanning-parser', [
-    [{ text: 'かかってこいよ', reading: 'かかってこいよ', headword: 'かかってくる' }],
-  ]);
-
-  const splitCandidate = makeParseItem('scanning-parser', [
-    [{ text: 'かかってこい', reading: 'かかってこい', headword: 'かかってくる' }],
-    [{ text: 'よ', reading: 'よ', headword: 'よ' }],
-  ]);
-
-  // Split candidate comes first — should still win over merged.
-  const tokens = selectYomitanParseTokens(
     [splitCandidate, mergedCandidate],
-    () => false,
-    'headword',
-  );
-  assert.equal(tokens?.length, 2);
-  assert.equal(tokens?.[0]?.surface, 'かかってこい');
-  assert.equal(tokens?.[1]?.surface, 'よ');
+  ]) {
+    const tokens = selectYomitanParseTokens(candidates, () => false, 'headword');
+    assert.deepEqual(
+      tokens?.map((token) => ({ surface: token.surface, headword: token.headword })),
+      [
+        { surface: 'かかってこい', headword: 'かかってくる' },
+        { surface: 'よ', headword: 'よ' },
+      ],
+    );
+  }
 });

src/core/services/tokenizer/subtitle-annotation-filter.test.ts

@@ -4,10 +4,8 @@ import { MergedToken, PartOfSpeech } from '../../../types';
 import {
   createSubtitleAnnotationRuleContext,
   shouldExcludeTokenFromSubtitleAnnotations,
-  SUBTITLE_ANNOTATION_EXCLUDED_TERMS,
   SUBTITLE_ANNOTATION_RULES,
 } from './subtitle-annotation-filter';
-import { isKanaChar } from './token-classification';
 
 function makeToken(overrides: Partial<MergedToken> = {}): MergedToken {
   return {
@@ -24,96 +22,6 @@ function makeToken(overrides: Partial<MergedToken> = {}): MergedToken {
   };
 }
 
-test('subtitle annotation rules expose stable ordered provenance', () => {
-  assert.deepEqual(
-    SUBTITLE_ANNOTATION_RULES.map(({ id, issueRef }) => ({ id, issueRef })),
-    [
-      { id: 'unparsed-run', issueRef: '#153' },
-      { id: 'configured-pos1-exclusion', issueRef: '#19' },
-      { id: 'configured-pos2-exclusion', issueRef: '#150' },
-      { id: 'coarse-grammar-pos-fallback', issueRef: '#19' },
-      { id: 'auxiliary-stem-grammar-tail', issueRef: '#19' },
-      { id: 'kana-non-independent-noun-helper', issueRef: '#56' },
-      { id: 'standalone-auxiliary-inflection', issueRef: '#57' },
-      { id: 'auxiliary-only-helper-span', issueRef: '#57' },
-      { id: 'standalone-suru-te-helper', issueRef: '#57' },
-      { id: 'standalone-grammar-particle', issueRef: '#57' },
-      { id: 'single-kana-fragment', issueRef: '#57' },
-      { id: 'merged-trailing-quote-particle', issueRef: '#19' },
-      { id: 'lexical-kureru-keep', issueRef: '#57' },
-      { id: 'excluded-term-or-pattern', issueRef: '#19, #33, #57' },
-    ],
-  );
-  assert.equal(new Set(SUBTITLE_ANNOTATION_RULES.map(({ id }) => id)).size, 14);
-  assert.ok(SUBTITLE_ANNOTATION_RULES.every(({ description }) => description.length > 0));
-  assert.ok(
-    SUBTITLE_ANNOTATION_RULES.every(
-      ({ data }) => Object.isFrozen(data) && Object.values(data).every(Object.isFrozen),
-    ),
-  );
-});
-
-test('lexical kureru keep rule precedes and overrides the excluded-term rule', () => {
-  const token = makeToken({
-    surface: 'くれ',
-    headword: 'くれる',
-    reading: 'クレ',
-    partOfSpeech: PartOfSpeech.verb,
-    pos1: '動詞',
-    pos2: '自立',
-  });
-  const context = createSubtitleAnnotationRuleContext(token);
-  const keepIndex = SUBTITLE_ANNOTATION_RULES.findIndex(({ id }) => id === 'lexical-kureru-keep');
-  const termIndex = SUBTITLE_ANNOTATION_RULES.findIndex(
-    ({ id }) => id === 'excluded-term-or-pattern',
-  );
-
-  assert.ok(keepIndex >= 0 && keepIndex < termIndex);
-  assert.ok(SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has('くれ'));
-  assert.equal(SUBTITLE_ANNOTATION_RULES[keepIndex]?.test(context), 'keep');
-  assert.equal(SUBTITLE_ANNOTATION_RULES[termIndex]?.test(context), 'exclude');
-  assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
-});
-
-test('adding an exported excluded term changes annotation matching', () => {
-  const term = '超語彙';
-  const token = makeToken({
-    surface: term,
-    headword: term,
-    reading: term,
-    pos1: '名詞',
-    pos2: '一般',
-  });
-
-  SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term);
-  try {
-    assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
-    SUBTITLE_ANNOTATION_EXCLUDED_TERMS.add(term);
-    assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
-  } finally {
-    SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term);
-  }
-});
-
-test('deleting an exported excluded term changes annotation matching', () => {
-  const term = 'あなた';
-  const token = makeToken({
-    surface: term,
-    headword: term,
-    reading: term,
-    pos1: '名詞',
-    pos2: '一般',
-  });
-
-  assert.ok(SUBTITLE_ANNOTATION_EXCLUDED_TERMS.has(term));
-  try {
-    SUBTITLE_ANNOTATION_EXCLUDED_TERMS.delete(term);
-    assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false);
-  } finally {
-    SUBTITLE_ANNOTATION_EXCLUDED_TERMS.add(term);
-  }
-});
-
 test('configured POS rules consume the context exclusion sets in table order', () => {
   const token = makeToken({ pos1: '名詞', pos2: '一般' });
   const context = createSubtitleAnnotationRuleContext(token, {
@@ -175,9 +83,3 @@ test('configured POS2 rule preserves supplementary-plane kanji nouns', () => {
     false,
   );
 });
-
-test('kana detection excludes katakana punctuation boundaries', () => {
-  assert.equal(isKanaChar('゠'), false);
-  assert.equal(isKanaChar('・'), false);
-  assert.equal(isKanaChar('ァ'), true);
-});

src/core/services/tokenizer/yomitan-scan-runtime.test.ts

@@ -258,20 +258,6 @@ test('halfwidth voiced kana compose into the reading instead of leaving a stray
   ]);
 });
 
-test('requestYomitanScanTokens falls back to the exhaustive name scan without candidates', async () => {
-  const withoutLookups: string[] = [];
-  const withoutCandidates = await requestYomitanScanTokens(
-    NAME_SCAN_LINE,
-    createNameScanDeps(withoutLookups),
-    { error: () => undefined },
-    { includeNameMatchMetadata: true, currentCharacterDictionaryMediaId: 1, nameCandidates: null },
-  );
-
-  assert.equal(withoutCandidates?.[0]?.isNameMatch, true);
-  // No candidate list means every Japanese position is probed, as before.
-  assert.ok(countTermsFindLookups(withoutLookups, '校に') > 0);
-});
-
 test('requestYomitanScanTokens reinstalls name candidates when the media changes', async () => {
   const lookups: string[] = [];
   const deps = createNameScanDeps(lookups);
@@ -302,3 +288,35 @@ test('requestYomitanScanTokens reinstalls name candidates when the media changes
   assert.equal(correctMedia?.[0]?.surface, 'ミナト');
   assert.equal(correctMedia?.[0]?.isNameMatch, true);
 });
+
+test('requestYomitanScanTokens skips the greedy name pre-pass when name matching is off', async () => {
+  const cases = [
+    {
+      name: 'name-match metadata disabled',
+      dictionaries: undefined,
+      includeNameMatchMetadata: false,
+      expectedNameMatch: false,
+    },
+    {
+      name: 'no enabled character dictionary',
+      dictionaries: ['JMdict'],
+      includeNameMatchMetadata: true,
+      // The main walk still classifies the match; only the pre-pass is gated.
+      expectedNameMatch: true,
+    },
+  ];
+  for (const c of cases) {
+    const lookups: string[] = [];
+    const result = await requestYomitanScanTokens(
+      NAME_SCAN_LINE,
+      createNameScanDeps(lookups, NAME_SCAN_WORDS, c.dictionaries),
+      { error: () => undefined },
+      { includeNameMatchMetadata: c.includeNameMatchMetadata },
+    );
+
+    assert.equal(result?.[0]?.surface, 'ミナト', c.name);
+    assert.equal(result?.[0]?.isNameMatch, c.expectedNameMatch, c.name);
+    // Mid-token positions are only probed by the pre-pass.
+    assert.equal(countTermsFindLookups(lookups, '校に'), 0, c.name);
+  }
+});

src/core/services/tokenizer/yomitan-scan-test-harness.ts

@@ -39,7 +39,11 @@ function createYomitanScriptSandbox(handler: (action: string, params: unknown) =
           callback: (response: { result?: unknown; error?: { message?: string } }) => void,
         ) => {
           try {
-            callback({ result: handler(payload.action ?? '', payload.params) });
+            // Messages cross a process boundary in Electron; clone params into
+            // the host realm so handlers can compare them structurally.
+            const params =
+              payload.params === undefined ? undefined : structuredClone(payload.params);
+            callback({ result: handler(payload.action ?? '', params) });
           } catch (error) {
             callback({ error: { message: (error as Error).message } });
           }
@@ -98,8 +102,128 @@ export function countTermsFindLookups(lookups: string[], prefix: string): number
   return lookups.filter((lookupText) => lookupText.startsWith(prefix)).length;
 }
 
+export interface TermsFindResult {
+  originalTextLength: number;
+  dictionaryEntries: unknown[];
+}
+
+// One termsFind dictionary entry whose headword exactly matches `originalText`
+// (the scanned surface, defaulting to the term). `dictionary` attaches a
+// definition from that dictionary, e.g. a SubMiner character dictionary.
+export function termEntry(
+  term: string,
+  reading: string,
+  options: { originalText?: string; isPrimary?: boolean; dictionary?: string } = {},
+) {
+  return {
+    headwords: [
+      {
+        term,
+        reading,
+        sources: [
+          {
+            originalText: options.originalText ?? term,
+            isPrimary: options.isPrimary ?? true,
+            matchType: 'exact',
+          },
+        ],
+      },
+    ],
+    ...(options.dictionary ? { definitions: [{ dictionary: options.dictionary }] } : {}),
+  };
+}
+
+export function termsFound(
+  originalTextLength: number,
+  ...dictionaryEntries: unknown[]
+): TermsFindResult {
+  return { originalTextLength, dictionaryEntries };
+}
+
+// Scan deps backed by a fake Yomitan backend. The active profile enables
+// `dictionaries` (in priority order; omitted means no dictionary list),
+// getDictionaryInfo returns `dictionaryInfo`, and termsFind answers through
+// `termsFind` (null/undefined means no match) while recording each text in
+// `lookups`. `actions` handles any other backend action and overrides the
+// defaults above; anything else throws. Every action name lands in `actionLog`.
+export function createBackendDeps(
+  options: {
+    dictionaries?: string[];
+    dictionaryInfo?: unknown[];
+    termsFind?: (text: string) => TermsFindResult | null | undefined;
+    lookups?: string[];
+    actions?: Record<string, (params: unknown) => unknown>;
+    actionLog?: string[];
+    onScript?: (script: string) => void;
+  } = {},
+) {
+  return createScanDeps(
+    (action, params) => {
+      options.actionLog?.push(action);
+      const override = options.actions?.[action];
+      if (override) {
+        return override(params);
+      }
+      if (action === 'optionsGetFull') {
+        return {
+          profileCurrent: 0,
+          profiles: [
+            {
+              options: {
+                scanning: { length: 40 },
+                ...(options.dictionaries
+                  ? {
+                      dictionaries: options.dictionaries.map((name, id) => ({
+                        name,
+                        enabled: true,
+                        id,
+                      })),
+                    }
+                  : {}),
+              },
+            },
+          ],
+        };
+      }
+      if (action === 'getDictionaryInfo') {
+        return options.dictionaryInfo ?? [];
+      }
+      if (action === 'termsFind' && options.termsFind) {
+        const text = (params as { text?: string } | undefined)?.text ?? '';
+        options.lookups?.push(text);
+        return options.termsFind(text) ?? termsFound(0);
+      }
+      throw new Error(`unexpected action: ${action}`);
+    },
+    { onScript: options.onScript },
+  );
+}
+
+// Deps whose hidden settings.html window runs injected scripts against a fake
+// `__subminerYomitanSettingsAutomation` bridge, like the real settings page.
+export function createSettingsAutomationDeps(automation: Record<string, unknown>) {
+  const context = vm.createContext({
+    __subminerYomitanSettingsAutomation: { ready: true, ...automation },
+    setTimeout,
+  });
+  const settingsWindow = {
+    isDestroyed: () => false,
+    destroy: () => undefined,
+    webContents: {
+      executeJavaScript: async (script: string) => await vm.runInContext(script, context),
+    },
+  };
+  return createDeps(async () => true, {
+    createYomitanExtensionWindow: async (pageName) =>
+      pageName === 'settings.html' ? settingsWindow : null,
+  });
+}
+
+export const SUBMINER_TEST_CHARACTER_DICTIONARY = 'SubMiner Character Dictionary (AniList 1)';
+
 // Backend stub for the greedy name pre-pass: one character name (ミナト) in a
-// line of ordinary words, with the SubMiner character dictionary enabled.
+// line of ordinary words, with the SubMiner character dictionary enabled
+// unless `dictionaries` says otherwise.
 export const NAME_SCAN_WORDS: Array<[string, string, string, boolean]> = [
   ['ミナト', 'ミナト', 'みなと', true],
   ['は', 'は', 'は', false],
@@ -112,57 +236,24 @@ export const NAME_SCAN_WORDS: Array<[string, string, string, boolean]> = [
 export function createNameScanDeps(
   lookups: string[],
   words: Array<[string, string, string, boolean]> = NAME_SCAN_WORDS,
+  dictionaries: string[] = ['JMdict', SUBMINER_TEST_CHARACTER_DICTIONARY],
 ) {
-  return createScanDeps((action, params) => {
-    if (action === 'optionsGetFull') {
-      return {
-        profileCurrent: 0,
-        profiles: [
-          {
-            options: {
-              scanning: { length: 40 },
-              dictionaries: [
-                { name: 'JMdict', enabled: true, id: 0 },
-                {
-                  name: 'SubMiner Character Dictionary (AniList 1)',
-                  enabled: true,
-                  id: 1,
-                },
-              ],
-            },
-          },
-        ],
-      };
-    }
-    if (action === 'getDictionaryInfo') {
-      return [];
-    }
-    if (action !== 'termsFind') {
-      throw new Error(`unexpected action: ${action}`);
-    }
-    const text = (params as { text?: string } | undefined)?.text ?? '';
-    lookups.push(text);
-    for (const [surface, term, reading, isName] of words) {
-      if (text.startsWith(surface)) {
-        return {
-          originalTextLength: surface.length,
-          dictionaryEntries: [
-            {
-              headwords: [
-                {
-                  term,
-                  reading,
-                  sources: [{ originalText: surface, isPrimary: true, matchType: 'exact' }],
-                },
-              ],
-              definitions: [
-                { dictionary: isName ? 'SubMiner Character Dictionary (AniList 1)' : 'JMdict' },
-              ],
-            },
-          ],
-        };
+  return createBackendDeps({
+    dictionaries,
+    lookups,
+    termsFind: (text) => {
+      const match = words.find(([surface]) => text.startsWith(surface));
+      if (!match) {
+        return null;
       }
-    }
-    return { originalTextLength: 0, dictionaryEntries: [] };
+      const [surface, term, reading, isName] = match;
+      return termsFound(
+        surface.length,
+        termEntry(term, reading, {
+          originalText: surface,
+          dictionary: isName ? SUBMINER_TEST_CHARACTER_DICTIONARY : 'JMdict',
+        }),
+      );
+    },
   });
 }