perf(tokenizer): single-pass Yomitan scan with cross-line caching and prefetch fixes (#185)

src/main/character-dictionary-runtime/constants.ts

@@ -1,7 +1,7 @@
 export const ANILIST_GRAPHQL_URL = 'https://graphql.anilist.co';
 export const ANILIST_REQUEST_DELAY_MS = 2000;
 export const CHARACTER_IMAGE_DOWNLOAD_DELAY_MS = 250;
-export const CHARACTER_DICTIONARY_FORMAT_VERSION = 19;
+export const CHARACTER_DICTIONARY_FORMAT_VERSION = 20;
 export const CHARACTER_DICTIONARY_MERGED_TITLE = 'SubMiner Character Dictionary';
 
 export const HONORIFIC_SUFFIXES = [

src/main/character-dictionary-runtime/name-candidates.test.ts

@@ -0,0 +1,163 @@
+import assert from 'node:assert/strict';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import test from 'node:test';
+import { CHARACTER_DICTIONARY_FORMAT_VERSION } from './constants';
+import { createCharacterNameCandidateLookup } from './name-candidates';
+
+function writeSnapshot(outputDir: string, mediaId: number, entries: Array<[string, string]>): void {
+  const snapshotsDir = path.join(outputDir, 'snapshots');
+  fs.mkdirSync(snapshotsDir, { recursive: true });
+  fs.writeFileSync(
+    path.join(snapshotsDir, `anilist-${mediaId}.json`),
+    JSON.stringify({
+      formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
+      mediaId,
+      mediaTitle: `title-${mediaId}`,
+      entryCount: entries.length,
+      updatedAt: 1,
+      termEntries: entries.map(([term, reading]) => [
+        term,
+        reading,
+        'name main',
+        '',
+        100,
+        [],
+        0,
+        '',
+      ]),
+      images: [],
+    }),
+  );
+}
+
+function withTempDir<T>(run: (dir: string) => T): T {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-name-candidates-'));
+  try {
+    return run(dir);
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+test('collects terms and readings for the current media', () => {
+  withTempDir((dir) => {
+    writeSnapshot(dir, 1, [
+      ['ミナト', 'みなと'],
+      ['湊', 'みなと'],
+    ]);
+    writeSnapshot(dir, 2, [['カズマ', 'かずま']]);
+
+    const lookup = createCharacterNameCandidateLookup({
+      outputDir: dir,
+      getCurrentMediaId: () => 1,
+    });
+    const candidates = lookup.get();
+
+    assert.ok(candidates);
+    assert.deepEqual([...candidates.forms].sort(), ['みなと', 'ミナト', '湊'].sort());
+    // Deduplicated: both entries share the みなと reading.
+    assert.equal(candidates.forms.length, 3);
+  });
+});
+
+test('returns null without a media scope so the scanner stays exhaustive', () => {
+  withTempDir((dir) => {
+    writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
+
+    const lookup = createCharacterNameCandidateLookup({
+      outputDir: dir,
+      getCurrentMediaId: () => null,
+    });
+
+    assert.equal(lookup.get(), null);
+  });
+});
+
+test('returns null for a media with no cached snapshot', () => {
+  withTempDir((dir) => {
+    writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
+
+    const lookup = createCharacterNameCandidateLookup({
+      outputDir: dir,
+      getCurrentMediaId: () => 999,
+    });
+
+    assert.equal(lookup.get(), null);
+  });
+});
+
+test('key changes when the snapshot content changes', () => {
+  withTempDir((dir) => {
+    writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
+    const lookup = createCharacterNameCandidateLookup({
+      outputDir: dir,
+      getCurrentMediaId: () => 1,
+    });
+    const first = lookup.get();
+
+    writeSnapshot(dir, 1, [
+      ['ミナト', 'みなと'],
+      ['アクア', 'あくあ'],
+    ]);
+    lookup.invalidate();
+    const second = lookup.get();
+
+    assert.ok(first && second);
+    assert.notEqual(first.key, second.key);
+    assert.equal(second.forms.length, 4);
+  });
+});
+
+// The lookup runs once per subtitle line, so it must not stat the snapshot
+// directory every call. Asserted behaviorally: an unannounced on-disk change is
+// invisible until the recheck interval elapses, which can only be true if the
+// filesystem is not consulted per lookup.
+test('does not re-read the snapshot directory on every lookup', () => {
+  withTempDir((dir) => {
+    writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
+    let nowMs = 1_000_000;
+    const lookup = createCharacterNameCandidateLookup({
+      outputDir: dir,
+      getCurrentMediaId: () => 1,
+      now: () => nowMs,
+    });
+
+    assert.equal(lookup.get()?.forms.length, 2);
+
+    writeSnapshot(dir, 1, [
+      ['ミナト', 'みなと'],
+      ['アクア', 'あくあ'],
+    ]);
+
+    nowMs += 1000;
+    assert.equal(lookup.get()?.forms.length, 2, 'expected the cached list within the interval');
+
+    nowMs += 10_000;
+    assert.equal(lookup.get()?.forms.length, 4, 'expected a refresh past the interval');
+  });
+});
+
+test('invalidate picks up a snapshot change immediately', () => {
+  withTempDir((dir) => {
+    writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
+    let nowMs = 1_000_000;
+    const lookup = createCharacterNameCandidateLookup({
+      outputDir: dir,
+      getCurrentMediaId: () => 1,
+      now: () => nowMs,
+    });
+
+    assert.equal(lookup.get()?.forms.length, 2);
+
+    writeSnapshot(dir, 1, [
+      ['ミナト', 'みなと'],
+      ['アクア', 'あくあ'],
+    ]);
+    nowMs += 1;
+    lookup.invalidate();
+
+    assert.equal(lookup.get()?.forms.length, 4);
+  });
+});

src/main/character-dictionary-runtime/name-candidates.ts

@@ -0,0 +1,159 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { readCachedSnapshots } from './cache';
+import type { CharacterDictionarySnapshot } from './types';
+
+// Candidate name forms for the greedy name pre-pass in the Yomitan scan
+// runtime. The scanner otherwise has to ask the backend at every Japanese
+// position, because a character name can start mid-token; knowing which forms
+// exist lets it look up only where a name can actually begin.
+//
+// A form is any string Yomitan could match a character entry by: the term and
+// its reading. Both come from the dictionary SubMiner generated, so the pair is
+// the complete matchable set for an entry. Callers treat a missing list as
+// "scan every position", so a stale or absent snapshot costs speed, never a
+// missed name.
+
+function getSnapshotsDir(outputDir: string): string {
+  return path.join(outputDir, 'snapshots');
+}
+
+function collectSnapshotNameForms(snapshot: CharacterDictionarySnapshot): string[] {
+  const forms = new Set<string>();
+  for (const entry of snapshot.termEntries) {
+    const term = typeof entry[0] === 'string' ? entry[0].trim() : '';
+    if (term) {
+      forms.add(term);
+    }
+    const reading = typeof entry[1] === 'string' ? entry[1].trim() : '';
+    if (reading) {
+      forms.add(reading);
+    }
+  }
+  return [...forms];
+}
+
+// The signature grows with the size of the dictionary library, and it rides
+// along in every per-line scan call, so it is folded into a fixed-width digest
+// first. Collisions only matter against the immediately previous signature (the
+// runtime compares keys for equality), and FNV-1a over the file list is far
+// beyond what that needs.
+function digestSnapshotDirectorySignature(signature: string): string {
+  let hash = 0x811c9dc5;
+  for (let index = 0; index < signature.length; index += 1) {
+    hash ^= signature.charCodeAt(index);
+    hash = Math.imul(hash, 0x01000193);
+  }
+  return (hash >>> 0).toString(36);
+}
+
+function getSnapshotDirectorySignature(outputDir: string): string {
+  let entries: fs.Dirent[] = [];
+  try {
+    entries = fs.readdirSync(getSnapshotsDir(outputDir), { withFileTypes: true });
+  } catch {
+    return '';
+  }
+
+  const parts: string[] = [];
+  for (const entry of entries) {
+    if (!entry.isFile() || !/^anilist-\d+\.json$/.test(entry.name)) {
+      continue;
+    }
+    try {
+      const stat = fs.statSync(path.join(getSnapshotsDir(outputDir), entry.name));
+      parts.push(`${entry.name}:${stat.mtimeMs}:${stat.size}`);
+    } catch {
+      // Ignore files that disappear during a refresh; the next lookup rebuilds.
+    }
+  }
+  return parts.sort().join('|');
+}
+
+export interface CharacterNameCandidateSet {
+  /** Identifies this exact form list, so the scan runtime can cache it. */
+  key: string;
+  forms: string[];
+}
+
+// This lookup is consulted once per subtitle line, so it must not stat the
+// snapshot directory every time. Dictionary writes are rare and always call
+// invalidate(), which forces the next lookup to re-read; the interval only
+// bounds staleness from changes made behind our back.
+const SNAPSHOT_SIGNATURE_RECHECK_INTERVAL_MS = 5000;
+
+export function createCharacterNameCandidateLookup(deps: {
+  userDataPath?: string;
+  outputDir?: string;
+  getCurrentMediaId?: () => number | null | undefined;
+  now?: () => number;
+}): {
+  get: (mediaId?: number | null) => CharacterNameCandidateSet | null;
+  invalidate: () => void;
+} {
+  const outputDir =
+    deps.outputDir ??
+    (deps.userDataPath ? path.join(deps.userDataPath, 'character-dictionaries') : '');
+  const now = deps.now ?? (() => Date.now());
+  let signature: string | null = null;
+  let lastSignatureCheckAtMs = 0;
+  let formsByMediaId = new Map<number, string[]>();
+
+  function refreshIfNeeded(): void {
+    if (!outputDir) {
+      formsByMediaId = new Map<number, string[]>();
+      signature = '';
+      return;
+    }
+    const nowMs = now();
+    if (
+      signature !== null &&
+      nowMs - lastSignatureCheckAtMs < SNAPSHOT_SIGNATURE_RECHECK_INTERVAL_MS
+    ) {
+      return;
+    }
+    lastSignatureCheckAtMs = nowMs;
+    const nextSignature = getSnapshotDirectorySignature(outputDir);
+    if (nextSignature === signature) {
+      return;
+    }
+    signature = nextSignature;
+    formsByMediaId = new Map<number, string[]>();
+    for (const snapshot of readCachedSnapshots(outputDir)) {
+      const forms = collectSnapshotNameForms(snapshot);
+      if (forms.length > 0) {
+        formsByMediaId.set(snapshot.mediaId, forms);
+      }
+    }
+  }
+
+  return {
+    get(mediaId?: number | null): CharacterNameCandidateSet | null {
+      refreshIfNeeded();
+      const rawMediaId = mediaId ?? deps.getCurrentMediaId?.() ?? null;
+      const normalizedMediaId =
+        typeof rawMediaId === 'number' && Number.isFinite(rawMediaId) && rawMediaId > 0
+          ? Math.floor(rawMediaId)
+          : null;
+
+      // Without a media scope the pre-pass would need every character of every
+      // cached title, which is both slow to match and pointless: report no
+      // candidates so the scanner keeps its exhaustive behavior.
+      if (normalizedMediaId === null) {
+        return null;
+      }
+      const forms = formsByMediaId.get(normalizedMediaId);
+      if (!forms || forms.length === 0) {
+        return null;
+      }
+      return {
+        key: `${digestSnapshotDirectorySignature(signature ?? '')}:${normalizedMediaId}`,
+        forms,
+      };
+    },
+    invalidate(): void {
+      signature = null;
+      lastSignatureCheckAtMs = 0;
+    },
+  };
+}

src/main/character-dictionary-runtime/name-reading.ts

@@ -1,3 +1,4 @@
+import { isHanCodePoint } from '../../core/text/han-code-points';
 import { HONORIFIC_SUFFIXES } from './constants';
 import type { JapaneseNameParts, NameReadings, ResolvedNameSplits } from './types';
 
@@ -26,10 +27,12 @@ export function buildReading(term: string): string {
   return katakanaToHiragana(compact);
 }
 
+// Code points, not code units: a supplementary-plane kanji (𠮷, U+20BB7) is a
+// surrogate pair, and reading only the high surrogate would classify a real
+// single-character name as non-kanji and drop it.
 export function containsKanji(value: string): boolean {
   for (const char of value) {
-    const code = char.charCodeAt(0);
-    if ((code >= 0x4e00 && code <= 0x9fff) || (code >= 0x3400 && code <= 0x4dbf)) {
+    if (isHanCodePoint(char.codePointAt(0) ?? 0)) {
       return true;
     }
   }

src/main/character-dictionary-runtime/term-building.test.ts

@@ -36,3 +36,134 @@ test('buildNameTerms adds surname honorifics from Japanese localized aliases', (
   assert.ok(terms.includes('馬渕さん'));
   assert.ok(!terms.includes('송치'));
 });
+
+test('buildNameTerms drops the disambiguator letter of a mob character name', () => {
+  const terms = buildNameTerms(
+    characterRecord({
+      firstNameHint: '',
+      lastNameHint: '',
+      fullName: 'Joshi A',
+      nativeName: '女子A',
+    }),
+  );
+
+  // ア would match every あ〜 in the subtitles; the letter is a disambiguator
+  // (Girl A / Girl B), not a name.
+  assert.ok(!terms.includes('ア'));
+  assert.ok(!terms.includes('アさん'));
+  assert.ok(terms.includes('女子A'));
+  assert.ok(terms.includes('ジョシア'));
+});
+
+test('buildNameTerms keeps a character whose whole name is one kana', () => {
+  const terms = buildNameTerms(
+    characterRecord({
+      firstNameHint: '',
+      lastNameHint: '',
+      fullName: 'A',
+      nativeName: 'あ',
+    }),
+  );
+
+  // The mob-label rule only judges parts a name was split into; a name the
+  // source gives us whole is the character's actual name.
+  assert.ok(terms.includes('あ'));
+  assert.ok(terms.includes('あさん'));
+  // Romanized forms are never lookup targets (the subtitles are Japanese), and
+  // the single-kana alias "A" transliterates to is dropped as a collision.
+  assert.ok(!terms.includes('A'));
+  assert.ok(!terms.includes('ア'));
+});
+
+test('buildNameTerms keeps a one-character name written in another script', () => {
+  const terms = buildNameTerms(
+    characterRecord({
+      firstNameHint: '',
+      lastNameHint: '',
+      fullName: 'Byeol',
+      nativeName: '별',
+      alternativeNames: ['Я'],
+    }),
+  );
+
+  assert.ok(terms.includes('별'));
+  assert.ok(terms.includes('별さん'));
+  assert.ok(terms.includes('Я'));
+});
+
+test('buildNameTerms yields nothing for a character whose only name is a bare letter', () => {
+  // Documented policy rather than an oversight: a romanized name is never a
+  // term on its own (the subtitles are Japanese), and the single kana a bare
+  // letter transliterates to would match every あ〜 in the line.
+  assert.deepEqual(
+    buildNameTerms(
+      characterRecord({
+        firstNameHint: '',
+        lastNameHint: '',
+        fullName: 'A',
+        nativeName: '',
+      }),
+    ),
+    [],
+  );
+});
+
+test('buildNameTerms keeps one-character split parts that are not mob labels', () => {
+  const hangul = buildNameTerms(
+    characterRecord({
+      firstNameHint: '',
+      lastNameHint: '',
+      fullName: 'Byeol Kim',
+      nativeName: '별 김',
+    }),
+  );
+
+  assert.ok(hangul.includes('별'));
+  assert.ok(hangul.includes('김'));
+
+  const middleDot = buildNameTerms(
+    characterRecord({
+      firstNameHint: '',
+      lastNameHint: '',
+      fullName: 'A Be',
+      nativeName: 'ア・ベ',
+    }),
+  );
+
+  assert.ok(middleDot.includes('ア'));
+  assert.ok(middleDot.includes('ベ'));
+});
+
+test('buildNameTerms keeps a single-kanji name part', () => {
+  // The name is an alias, not the native name, so the parts come from the
+  // space split rather than from the native-name split.
+  const terms = buildNameTerms(
+    characterRecord({
+      firstNameHint: 'Sora',
+      lastNameHint: 'Yamada',
+      fullName: 'Sora Yamada',
+      nativeName: '',
+      alternativeNames: ['山田 空'],
+    }),
+  );
+
+  assert.ok(terms.includes('山田'));
+  assert.ok(terms.includes('空'));
+});
+
+test('buildNameTerms keeps a single supplementary-plane kanji name part', () => {
+  // 𠮷 (U+20BB7) is a surrogate pair: a code-unit kanji check reads only the
+  // high surrogate and drops the part as if it were a mob disambiguator.
+  const terms = buildNameTerms(
+    characterRecord({
+      firstNameHint: 'Tsukasa',
+      lastNameHint: 'Yoshi',
+      fullName: 'Tsukasa Yoshi',
+      nativeName: '',
+      alternativeNames: ['𠮷 司'],
+    }),
+  );
+
+  assert.ok(terms.includes('𠮷'));
+  assert.ok(terms.includes('司'));
+});

src/main/character-dictionary-runtime/term-building.ts

@@ -1,3 +1,4 @@
+import { HAN_REGEXP_CLASS_BODY } from '../../core/text/han-code-points';
 import { HONORIFIC_SUFFIXES } from './constants';
 import {
   addRomanizedKanaAliases,
@@ -42,11 +43,29 @@ export function expandRawNameVariants(rawName: string): string[] {
   return [...variants];
 }
 
+// The label AniList appends to unnamed mob characters: one letter or digit,
+// halfwidth or fullwidth (女子A / "Joshi A" / 女子１). Nothing else qualifies —
+// a one-character part in any script is a real name part (별 김, ア・ベ, 山田 空).
+const SINGLE_LABEL_CHARACTER = /^[0-9A-Za-z\uff10-\uff19\uff21-\uff3a\uff41-\uff5a]$/u;
+
+// Judged on split parts only: a name the source gives us whole in a script the
+// subtitles can contain is kept whatever it looks like, because a character
+// really can be called あ or 별. (A romanized name is a separate matter: it is
+// never a term on its own, only a source of kana aliases. See below.)
+function isUsableNameSplitPart(part: string): boolean {
+  return !SINGLE_LABEL_CHARACTER.test(part);
+}
+
+// Kana, Han (shared ranges), and the marks that only ever appear inside a
+// Japanese name: iteration marks and the small ka/ke used in place names.
+const JAPANESE_NAME_CHARACTERS = new RegExp(
+  `^[\\u3040-\\u30ff${HAN_REGEXP_CLASS_BODY}\u3005\u3006\u30f5\u30f6\u30fc]+$`,
+  'u',
+);
+
 export function isJapaneseNameSplitCandidate(name: string): boolean {
   const compact = name.replace(/[\s\u3000・･·•]/g, '');
-  return (
-    containsKanji(compact) && /^[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff々〆ヵヶー]+$/.test(compact)
-  );
+  return containsKanji(compact) && JAPANESE_NAME_CHARACTERS.test(compact);
 }
 
 function addJapaneseNameParts(
@@ -97,8 +116,11 @@ export function buildNameTerms(
 
       const split = name.split(/[\s\u3000]+/).filter((part) => part.trim().length > 0);
       if (split.length === 2) {
-        target.add(split[0]!);
-        target.add(split[1]!);
+        for (const part of split) {
+          if (isUsableNameSplitPart(part)) {
+            target.add(part);
+          }
+        }
       }
 
       const splitByMiddleDot = name
@@ -107,7 +129,9 @@ export function buildNameTerms(
         .filter((part) => part.length > 0);
       if (splitByMiddleDot.length >= 2) {
         for (const part of splitByMiddleDot) {
-          target.add(part);
+          if (isUsableNameSplitPart(part)) {
+            target.add(part);
+          }
         }
       }
 
@@ -117,7 +141,15 @@ export function buildNameTerms(
     }
   }
 
+  // Romanized names never become terms themselves — the subtitles are Japanese,
+  // so "Joshi A" would never appear in one — they only contribute the kana a
+  // Japanese writer would spell them with.
   for (const alias of addRomanizedKanaAliases(romanizedBase)) {
+    // Except when the whole name is one letter: it transliterates to a single
+    // kana (A → ア) that matches every あ〜 in the subtitles. A character whose
+    // only recorded name is a bare letter therefore yields no terms at all,
+    // which is the intended outcome: those are unnamed mob characters.
+    if ([...alias].length === 1) continue;
     base.add(alias);
   }