perf(tokenizer): skip character-name lookups where no name can start

The greedy name pre-pass asked the Yomitan backend at every Japanese
position, because a character name can begin mid-token. With the character
dictionary enabled that roughly doubled the round trips per line (measured
10 -> 21 on a 23-char line).

SubMiner generates the character dictionary, so the cached snapshots already
list every form a character entry can be matched by (term and reading). Those
forms are installed into the scan runtime once per media and the pre-pass now
probes only positions where one of them starts, compared after kana
normalization so a katakana name still matches a hiragana reading form. The
overhead drops to zero (21 -> 10, the same as with the dictionary disabled).

Fail-safe: with no candidate list (no media id, no cached snapshot, failed
install) the pre-pass keeps its exhaustive behavior, so stale character data
costs speed rather than a missing name. Halfwidth katakana positions bypass
the filter since kana normalization does not fold them.

The candidate lookup is consulted per subtitle line, so it caches its snapshot
directory signature for 5s; dictionary writes still call invalidate().
This commit is contained in:
2026-08-03 23:25:45 -07:00
parent 030c94934e
commit b0a2ce6e8a
9 changed files with 639 additions and 4 deletions
@@ -9,3 +9,4 @@ area: subtitles
- Subtitle changes no longer restart the prefetch run per line (which discarded in-flight tokenization work); prefetch now only pauses for the live line and restarts on real seeks, cache invalidation, or option changes. Prefetch also stays paused across a provisional raw-subtitle emit and resumes only after the tokenized payload lands, so it never competes with the on-screen line for the parser window.
- Added per-stage debug timings (`scanMs`, `mecabMs`, `frequencyMs`, `annotateMs`) to the subtitle tokenization pipeline log.
- Fixed a reading that stopped covering its surface when an unmatched kana run extended the preceding token (for example a trailing る on 待ち合わせ), which silently disabled the known-word reading fallback for those tokens.
- Character name annotations no longer cost a dictionary lookup at every position in a line. The scanner now knows which name forms the current title's character dictionary actually contains and only checks where one can start, which removes the whole overhead of having the character dictionary enabled (measured: 21 lookups per line down to 10, the same as with it disabled). Titles with no cached character data keep the previous exhaustive scan, so a missing snapshot costs speed rather than a missing name.
+4
View File
@@ -70,6 +70,7 @@ export interface TokenizerServiceDeps {
getNameMatchImagesEnabled?: () => boolean;
getCharacterNameImage?: (term: string) => CharacterNameImage | null;
getCurrentCharacterDictionaryMediaId?: () => number | null;
getCharacterNameCandidates?: () => { key: string; forms: string[] } | null;
getFrequencyDictionaryEnabled?: () => boolean;
getFrequencyDictionaryMatchMode?: () => FrequencyDictionaryMatchMode;
getFrequencyRank?: FrequencyDictionaryLookup;
@@ -106,6 +107,7 @@ export interface TokenizerDepsRuntimeOptions {
getNameMatchImagesEnabled?: () => boolean;
getCharacterNameImage?: (term: string) => CharacterNameImage | null;
getCurrentCharacterDictionaryMediaId?: () => number | null;
getCharacterNameCandidates?: () => { key: string; forms: string[] } | null;
getFrequencyDictionaryEnabled?: () => boolean;
getFrequencyDictionaryMatchMode?: () => FrequencyDictionaryMatchMode;
getFrequencyRank?: FrequencyDictionaryLookup;
@@ -266,6 +268,7 @@ export function createTokenizerDepsRuntime(
getNameMatchImagesEnabled: options.getNameMatchImagesEnabled,
getCharacterNameImage: options.getCharacterNameImage,
getCurrentCharacterDictionaryMediaId: options.getCurrentCharacterDictionaryMediaId,
getCharacterNameCandidates: options.getCharacterNameCandidates,
getFrequencyDictionaryEnabled: options.getFrequencyDictionaryEnabled,
getFrequencyDictionaryMatchMode: options.getFrequencyDictionaryMatchMode ?? (() => 'headword'),
getFrequencyRank: options.getFrequencyRank,
@@ -735,6 +738,7 @@ async function parseWithYomitanInternalParser(
const selectedTokens = await requestYomitanScanTokens(text, deps, logger, {
includeNameMatchMetadata: options.nameMatchEnabled,
currentCharacterDictionaryMediaId: deps.getCurrentCharacterDictionaryMediaId?.() ?? null,
nameCandidates: deps.getCharacterNameCandidates?.() ?? null,
});
if (stageTimings) {
stageTimings.scanMs = Date.now() - scanStartedAtMs;
@@ -110,6 +110,175 @@ function countTermsFindLookups(lookups: string[], prefix: string): number {
return lookups.filter((lookupText) => lookupText.startsWith(prefix)).length;
}
// Backend stub for the greedy name pre-pass: one character name (ミナト) in a
// line of ordinary words, with the SubMiner character dictionary enabled.
const NAME_SCAN_WORDS: Array<[string, string, string, boolean]> = [
['ミナト', 'ミナト', 'みなと', true],
['は', 'は', 'は', false],
['まだ', 'まだ', 'まだ', false],
['学校', '学校', 'がっこう', false],
['に', 'に', 'に', false],
['いない', 'いる', 'いる', false],
];
function createNameScanDeps(lookups: string[]) {
return createScanDeps((action, params) => {
if (action === 'optionsGetFull') {
return {
profileCurrent: 0,
profiles: [
{
options: {
scanning: { length: 40 },
dictionaries: [
{ name: 'JMdict', enabled: true, id: 0 },
{
name: 'SubMiner Character Dictionary (AniList 1)',
enabled: true,
id: 1,
},
],
},
},
],
};
}
if (action === 'getDictionaryInfo') {
return [];
}
if (action !== 'termsFind') {
throw new Error(`unexpected action: ${action}`);
}
const text = (params as { text?: string } | undefined)?.text ?? '';
lookups.push(text);
for (const [surface, term, reading, isName] of NAME_SCAN_WORDS) {
if (text.startsWith(surface)) {
return {
originalTextLength: surface.length,
dictionaryEntries: [
{
headwords: [
{
term,
reading,
sources: [{ originalText: surface, isPrimary: true, matchType: 'exact' }],
},
],
definitions: [
{ dictionary: isName ? 'SubMiner Character Dictionary (AniList 1)' : 'JMdict' },
],
},
],
};
}
}
return { originalTextLength: 0, dictionaryEntries: [] };
});
}
const NAME_SCAN_LINE = 'ミナトはまだ学校にいない';
test('requestYomitanScanTokens skips name pre-pass lookups where no candidate name can start', async () => {
const exhaustiveLookups: string[] = [];
const exhaustive = await requestYomitanScanTokens(
NAME_SCAN_LINE,
createNameScanDeps(exhaustiveLookups),
{ error: () => undefined },
{ includeNameMatchMetadata: true },
);
const prefilteredLookups: string[] = [];
const prefiltered = await requestYomitanScanTokens(
NAME_SCAN_LINE,
createNameScanDeps(prefilteredLookups),
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
// Terms and readings the generated dictionary exposes for this media.
nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] },
},
);
// Same tokenization, including the name match, with fewer round trips.
assert.deepEqual(prefiltered, exhaustive);
assert.equal(prefiltered?.[0]?.surface, 'ミナト');
assert.equal(prefiltered?.[0]?.isNameMatch, true);
assert.ok(
prefilteredLookups.length < exhaustiveLookups.length,
`expected fewer lookups with candidates (${prefilteredLookups.length} vs ${exhaustiveLookups.length})`,
);
// Mid-token positions are exactly what the pre-pass used to probe (a name can
// start mid-token); with candidates they cost nothing, while the main walk's
// own token-start lookups are unaffected.
assert.ok(countTermsFindLookups(exhaustiveLookups, '校に') > 0);
assert.equal(countTermsFindLookups(prefilteredLookups, '校に'), 0);
});
test('requestYomitanScanTokens matches a katakana name from its kana-normalized candidate form', async () => {
const lookups: string[] = [];
const result = await requestYomitanScanTokens(
NAME_SCAN_LINE,
createNameScanDeps(lookups),
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
// Only the hiragana reading is listed; the katakana surface in the line
// must still be found through kana normalization.
nameCandidates: { key: 'media-1', forms: ['みなと'] },
},
);
assert.equal(result?.[0]?.surface, 'ミナト');
assert.equal(result?.[0]?.isNameMatch, true);
});
test('requestYomitanScanTokens falls back to the exhaustive name scan without candidates', async () => {
const withoutLookups: string[] = [];
const withoutCandidates = await requestYomitanScanTokens(
NAME_SCAN_LINE,
createNameScanDeps(withoutLookups),
{ error: () => undefined },
{ includeNameMatchMetadata: true, currentCharacterDictionaryMediaId: 1, nameCandidates: null },
);
assert.equal(withoutCandidates?.[0]?.isNameMatch, true);
// No candidate list means every Japanese position is probed, as before.
assert.ok(countTermsFindLookups(withoutLookups, '校に') > 0);
});
test('requestYomitanScanTokens reinstalls name candidates when the media changes', async () => {
const lookups: string[] = [];
const deps = createNameScanDeps(lookups);
// First media's candidates cannot match this line's name.
const otherMedia = await requestYomitanScanTokens(
NAME_SCAN_LINE,
deps,
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 2,
nameCandidates: { key: 'media-2', forms: ['カズマ'] },
},
);
assert.equal(otherMedia?.[0]?.isNameMatch, undefined);
const correctMedia = await requestYomitanScanTokens(
NAME_SCAN_LINE,
deps,
{ error: () => undefined },
{
includeNameMatchMetadata: true,
currentCharacterDictionaryMediaId: 1,
nameCandidates: { key: 'media-1', forms: ['ミナト'] },
},
);
assert.equal(correctMedia?.[0]?.surface, 'ミナト');
assert.equal(correctMedia?.[0]?.isNameMatch, true);
});
test('syncYomitanDefaultAnkiServer updates default profile server when script reports update', async () => {
let scriptValue = '';
const deps = createDeps(async (script) => {
@@ -5,6 +5,7 @@ import * as path from 'path';
import { selectYomitanParseTokens } from './parser-selection-stage';
import {
buildYomitanScanCallScript,
buildYomitanScanNameCandidatesScript,
CHARACTER_DICTIONARY_TITLE_PREFIX,
YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT,
YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL,
@@ -832,6 +833,40 @@ async function serveDictionaryZipOnce<T>(
async function installYomitanScanRuntime(parserWindow: BrowserWindow): Promise<void> {
await parserWindow.webContents.executeJavaScript(YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT, true);
// A fresh runtime has no candidate list; force the next scan to reinstall it.
yomitanScanNameCandidateKeyByWindow.delete(parserWindow);
}
// Key of the character-name candidate list currently installed in each parser
// window, so an unchanged list costs nothing per line.
const yomitanScanNameCandidateKeyByWindow = new WeakMap<BrowserWindow, string>();
async function ensureYomitanScanNameCandidates(
parserWindow: BrowserWindow,
nameCandidates: { key: string; forms: string[] } | null,
logger: LoggerLike,
): Promise<void> {
const installedKey = yomitanScanNameCandidateKeyByWindow.get(parserWindow);
const nextKey = nameCandidates?.key ?? '';
if (installedKey === nextKey) {
return;
}
try {
await parserWindow.webContents.executeJavaScript(
buildYomitanScanNameCandidatesScript(nameCandidates),
true,
);
yomitanScanNameCandidateKeyByWindow.set(parserWindow, nextKey);
} catch (err) {
// The scan falls back to checking every position when the list is absent,
// so a failed install costs speed, never a missed name.
logger.warn?.(
'Failed to install Yomitan character-name scan candidates:',
(err as Error).message,
);
yomitanScanNameCandidateKeyByWindow.delete(parserWindow);
}
}
export async function requestYomitanParseResults(
@@ -949,6 +984,7 @@ export async function requestYomitanScanTokens(
options?: {
includeNameMatchMetadata?: boolean;
currentCharacterDictionaryMediaId?: number | null;
nameCandidates?: { key: string; forms: string[] } | null;
},
): Promise<YomitanScanToken[] | null> {
const yomitanExt = deps.getYomitanExt();
@@ -972,6 +1008,12 @@ export async function requestYomitanScanTokens(
name.startsWith(CHARACTER_DICTIONARY_TITLE_PREFIX),
);
// Candidate name forms let the in-page pre-pass skip positions where no
// character name can start. Installed only when it changes (per media), so
// the per-line call stays a single tiny script.
const nameCandidates = greedyNameScanEnabled ? (options?.nameCandidates ?? null) : null;
await ensureYomitanScanNameCandidates(parserWindow, nameCandidates, logger);
const callScript = buildYomitanScanCallScript({
text,
profileIndex,
@@ -987,14 +1029,17 @@ export async function requestYomitanScanTokens(
dictionaryPriorityByName: metadata?.dictionaryPriorityByName ?? {},
dictionaryFrequencyModeByName: metadata?.dictionaryFrequencyModeByName ?? {},
cacheEpoch: getYomitanScanCacheEpoch(parserWindow),
nameCandidateKey: nameCandidates?.key ?? null,
});
try {
let rawResult = await parserWindow.webContents.executeJavaScript(callScript, true);
if (rawResult === YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL) {
// First request for this window, or the page reloaded and dropped the
// installed runtime: install and retry once.
// installed runtime: install and retry once. The candidate list lives in
// the same page state, so it has to be reinstalled alongside it.
await installYomitanScanRuntime(parserWindow);
await ensureYomitanScanNameCandidates(parserWindow, nameCandidates, logger);
rawResult = await parserWindow.webContents.executeJavaScript(callScript, true);
}
if (isScanTokenArray(rawResult)) {
@@ -495,7 +495,7 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
// Bump whenever the install script below changes so already-loaded parser
// windows re-install the new scan runtime instead of running the stale one.
export const YOMITAN_SCAN_RUNTIME_VERSION = 2;
export const YOMITAN_SCAN_RUNTIME_VERSION = 3;
export const YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL = '__subminer-yomitan-scan-runtime-missing__';
export interface YomitanScanRequestParams {
@@ -508,6 +508,11 @@ export interface YomitanScanRequestParams {
dictionaryPriorityByName: Record<string, number>;
dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>;
cacheEpoch: number;
/**
* Key of the character-name candidate list installed for the current media,
* or null to scan every Japanese position (see the pre-pass prefilter).
*/
nameCandidateKey: string | null;
}
// Installed once per parser window (and re-installed after in-page reloads):
@@ -544,6 +549,22 @@ export const YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT = String.raw`
const TERMS_FIND_CACHE_LIMIT = 2000;
let termsFindCacheEpoch = -1;
const MAX_SHRINKING_WINDOW_RETRY_LOOKUPS = 4;
// Character-name candidate forms for the current media, installed
// separately from the per-line scan call so the per-line script stays tiny.
// Stored raw here; the normalized lookup index is built inside the scan,
// where the kana-normalization helper is in scope, and reused by key.
let rawNameCandidates = null;
let nameCandidateIndex = null;
globalThis.__subminerYomitanScanSetNameCandidates = (key, forms) => {
if (!key || !Array.isArray(forms) || forms.length === 0) {
rawNameCandidates = null;
nameCandidateIndex = null;
return false;
}
rawNameCandidates = { key, forms };
nameCandidateIndex = null;
return true;
};
globalThis.__subminerYomitanScanVersion = ${YOMITAN_SCAN_RUNTIME_VERSION};
globalThis.__subminerYomitanScan = async (scanParams) => {
const {
@@ -555,7 +576,8 @@ export const YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT = String.raw`
currentCharacterDictionaryMediaId,
dictionaryPriorityByName,
dictionaryFrequencyModeByName,
cacheEpoch
cacheEpoch,
nameCandidateKey
} = scanParams;
if (cacheEpoch !== termsFindCacheEpoch) {
termsFindCache.clear();
@@ -681,6 +703,44 @@ ${YOMITAN_SCANNING_HELPERS}
}
return { token: buildScanToken(position, source, preferredHeadword), matchedLength: originalTextLength };
}
// Halfwidth katakana survives kana normalization unchanged, so a name
// written that way would not prefix-match a candidate form. Those
// positions bypass the prefilter rather than risk a missed name.
function isHalfwidthKatakanaCodePoint(codePoint) {
return codePoint >= 0xff66 && codePoint <= 0xff9f;
}
// Build (once per candidate list) a first-character bucket index of the
// normalized name forms, so the pre-pass can reject a position with a
// single map hit instead of a backend round trip.
if (rawNameCandidates && nameCandidateIndex?.key !== rawNameCandidates.key) {
const byFirstChar = new Map();
for (const form of rawNameCandidates.forms) {
const normalized = typeof form === "string" ? convertKatakanaToHiragana(form.trim()) : "";
if (!normalized) { continue; }
const bucket = byFirstChar.get(normalized[0]);
if (bucket) { bucket.push(normalized); } else { byFirstChar.set(normalized[0], [normalized]); }
}
nameCandidateIndex = byFirstChar.size > 0 ? { key: rawNameCandidates.key, byFirstChar } : null;
} else if (!rawNameCandidates) {
nameCandidateIndex = null;
}
// Only meaningful when the installed list matches the media this scan is
// for; otherwise fall back to scanning every position.
const activeNameCandidateIndex =
nameCandidateKey !== null && nameCandidateIndex?.key === nameCandidateKey
? nameCandidateIndex
: null;
const normalizedText = activeNameCandidateIndex ? convertKatakanaToHiragana(text) : "";
function couldNameStartAt(position, codePoint) {
if (!activeNameCandidateIndex) { return true; }
if (isHalfwidthKatakanaCodePoint(codePoint)) { return true; }
const bucket = activeNameCandidateIndex.byFirstChar.get(normalizedText[position]);
if (!bucket) { return false; }
for (const form of bucket) {
if (normalizedText.startsWith(form, position)) { return true; }
}
return false;
}
// Greedy name pre-pass: character-name matches claim their spans before
// the left-to-right walk, so a longer generic match starting earlier
// (e.g. とヨー → 渡洋) cannot swallow the start of a name (ヨータ).
@@ -689,7 +749,7 @@ ${YOMITAN_SCANNING_HELPERS}
let namePos = 0;
while (namePos < text.length) {
const codePoint = text.codePointAt(namePos);
if (!isCodePointJapanese(codePoint)) {
if (!isCodePointJapanese(codePoint) || !couldNameStartAt(namePos, codePoint)) {
namePos += String.fromCodePoint(codePoint).length;
continue;
}
@@ -786,6 +846,36 @@ ${YOMITAN_SCANNING_HELPERS}
})();
`;
// Installs (or clears) the character-name candidate forms for the current
// media. Runs only when the list changes, not per line. Passing null restores
// the exhaustive every-position pre-pass.
export function buildYomitanScanNameCandidatesScript(
nameCandidates: { key: string; forms: string[] } | null,
): string {
if (!nameCandidates) {
return `
(() => {
if (typeof globalThis.__subminerYomitanScanSetNameCandidates !== "function") {
return false;
}
return globalThis.__subminerYomitanScanSetNameCandidates(null, null);
})();
`;
}
return `
(() => {
if (typeof globalThis.__subminerYomitanScanSetNameCandidates !== "function") {
return false;
}
return globalThis.__subminerYomitanScanSetNameCandidates(
${JSON.stringify(nameCandidates.key)},
${JSON.stringify(nameCandidates.forms)}
);
})();
`;
}
export function buildYomitanScanCallScript(params: YomitanScanRequestParams): string {
return `
(async () => {
+13
View File
@@ -487,6 +487,7 @@ import { createOverlayVisibilityRuntimeService } from './main/overlay-visibility
import { createDiscordPresenceRuntime } from './main/runtime/discord-presence-runtime';
import { createCharacterDictionaryRuntimeService } from './main/character-dictionary-runtime';
import { createCharacterDictionaryImageLookup } from './main/character-dictionary-runtime/image-lookup';
import { createCharacterNameCandidateLookup } from './main/character-dictionary-runtime/name-candidates';
import {
createCharacterDictionaryAutoSyncRuntimeService,
getCharacterDictionaryManagerSnapshot,
@@ -2562,6 +2563,13 @@ const characterDictionaryImageLookup = createCharacterDictionaryImageLookup({
getCurrentMediaId: () => characterDictionaryAutoSyncRuntime.getCurrentMediaId(),
});
// Lets the Yomitan scan runtime skip name lookups at positions where no
// character name can start; absent candidates just mean the exhaustive scan.
const characterNameCandidateLookup = createCharacterNameCandidateLookup({
userDataPath: USER_DATA_PATH,
getCurrentMediaId: () => characterDictionaryAutoSyncRuntime.getCurrentMediaId(),
});
const overlayVisibilityRuntime = createOverlayVisibilityRuntimeService(
createBuildOverlayVisibilityRuntimeMainDepsHandler({
getMainWindow: () => overlayManager.getMainWindow(),
@@ -4624,6 +4632,7 @@ const {
getCharacterNameImage: (term) => characterDictionaryImageLookup.get(term),
getCurrentCharacterDictionaryMediaId: () =>
characterDictionaryAutoSyncRuntime.getCurrentMediaId(),
getCharacterNameCandidates: () => characterNameCandidateLookup.get(),
getFrequencyDictionaryEnabled: () =>
getRuntimeBooleanOption(
'subtitle.annotation.frequency',
@@ -5679,6 +5688,8 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
try {
await characterDictionaryAutoSyncRuntime.runSyncNow();
characterDictionaryImageLookup.invalidate();
characterNameCandidateLookup.invalidate();
characterNameCandidateLookup.invalidate();
} catch (error) {
logger.warn('Failed to rebuild character dictionary after manager override:', error);
}
@@ -5710,6 +5721,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
try {
await characterDictionaryAutoSyncRuntime.runSyncNow();
characterDictionaryImageLookup.invalidate();
characterNameCandidateLookup.invalidate();
} catch (error) {
logger.warn('Failed to rebuild character dictionary after manager removal:', error);
}
@@ -5727,6 +5739,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
try {
await characterDictionaryAutoSyncRuntime.runSyncNow();
characterDictionaryImageLookup.invalidate();
characterNameCandidateLookup.invalidate();
} catch (error) {
logger.warn('Failed to rebuild character dictionary after manager reorder:', error);
}
@@ -0,0 +1,163 @@
import assert from 'node:assert/strict';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import test from 'node:test';
import { CHARACTER_DICTIONARY_FORMAT_VERSION } from './constants';
import { createCharacterNameCandidateLookup } from './name-candidates';
function writeSnapshot(outputDir: string, mediaId: number, entries: Array<[string, string]>): void {
const snapshotsDir = path.join(outputDir, 'snapshots');
fs.mkdirSync(snapshotsDir, { recursive: true });
fs.writeFileSync(
path.join(snapshotsDir, `anilist-${mediaId}.json`),
JSON.stringify({
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
mediaId,
mediaTitle: `title-${mediaId}`,
entryCount: entries.length,
updatedAt: 1,
termEntries: entries.map(([term, reading]) => [
term,
reading,
'name main',
'',
100,
[],
0,
'',
]),
images: [],
}),
);
}
function withTempDir<T>(run: (dir: string) => T): T {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-name-candidates-'));
try {
return run(dir);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
}
test('collects terms and readings for the current media', () => {
withTempDir((dir) => {
writeSnapshot(dir, 1, [
['ミナト', 'みなと'],
['湊', 'みなと'],
]);
writeSnapshot(dir, 2, [['カズマ', 'かずま']]);
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => 1,
});
const candidates = lookup.get();
assert.ok(candidates);
assert.deepEqual([...candidates.forms].sort(), ['みなと', 'ミナト', '湊'].sort());
// Deduplicated: both entries share the みなと reading.
assert.equal(candidates.forms.length, 3);
});
});
test('returns null without a media scope so the scanner stays exhaustive', () => {
withTempDir((dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => null,
});
assert.equal(lookup.get(), null);
});
});
test('returns null for a media with no cached snapshot', () => {
withTempDir((dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => 999,
});
assert.equal(lookup.get(), null);
});
});
test('key changes when the snapshot content changes', () => {
withTempDir((dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => 1,
});
const first = lookup.get();
writeSnapshot(dir, 1, [
['ミナト', 'みなと'],
['アクア', 'あくあ'],
]);
lookup.invalidate();
const second = lookup.get();
assert.ok(first && second);
assert.notEqual(first.key, second.key);
assert.equal(second.forms.length, 4);
});
});
// The lookup runs once per subtitle line, so it must not stat the snapshot
// directory every call. Asserted behaviorally: an unannounced on-disk change is
// invisible until the recheck interval elapses, which can only be true if the
// filesystem is not consulted per lookup.
test('does not re-read the snapshot directory on every lookup', () => {
withTempDir((dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
let nowMs = 1_000_000;
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => 1,
now: () => nowMs,
});
assert.equal(lookup.get()?.forms.length, 2);
writeSnapshot(dir, 1, [
['ミナト', 'みなと'],
['アクア', 'あくあ'],
]);
nowMs += 1000;
assert.equal(lookup.get()?.forms.length, 2, 'expected the cached list within the interval');
nowMs += 10_000;
assert.equal(lookup.get()?.forms.length, 4, 'expected a refresh past the interval');
});
});
test('invalidate picks up a snapshot change immediately', () => {
withTempDir((dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
let nowMs = 1_000_000;
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => 1,
now: () => nowMs,
});
assert.equal(lookup.get()?.forms.length, 2);
writeSnapshot(dir, 1, [
['ミナト', 'みなと'],
['アクア', 'あくあ'],
]);
nowMs += 1;
lookup.invalidate();
assert.equal(lookup.get()?.forms.length, 4);
});
});
@@ -0,0 +1,142 @@
import * as fs from 'fs';
import * as path from 'path';
import { readCachedSnapshots } from './cache';
import type { CharacterDictionarySnapshot } from './types';
// Candidate name forms for the greedy name pre-pass in the Yomitan scan
// runtime. The scanner otherwise has to ask the backend at every Japanese
// position, because a character name can start mid-token; knowing which forms
// exist lets it look up only where a name can actually begin.
//
// A form is any string Yomitan could match a character entry by: the term and
// its reading. Both come from the dictionary SubMiner generated, so the pair is
// the complete matchable set for an entry. Callers treat a missing list as
// "scan every position", so a stale or absent snapshot costs speed, never a
// missed name.
function getSnapshotsDir(outputDir: string): string {
return path.join(outputDir, 'snapshots');
}
function collectSnapshotNameForms(snapshot: CharacterDictionarySnapshot): string[] {
const forms = new Set<string>();
for (const entry of snapshot.termEntries) {
const term = typeof entry[0] === 'string' ? entry[0].trim() : '';
if (term) {
forms.add(term);
}
const reading = typeof entry[1] === 'string' ? entry[1].trim() : '';
if (reading) {
forms.add(reading);
}
}
return [...forms];
}
function getSnapshotDirectorySignature(outputDir: string): string {
let entries: fs.Dirent[] = [];
try {
entries = fs.readdirSync(getSnapshotsDir(outputDir), { withFileTypes: true });
} catch {
return '';
}
const parts: string[] = [];
for (const entry of entries) {
if (!entry.isFile() || !/^anilist-\d+\.json$/.test(entry.name)) {
continue;
}
try {
const stat = fs.statSync(path.join(getSnapshotsDir(outputDir), entry.name));
parts.push(`${entry.name}:${stat.mtimeMs}:${stat.size}`);
} catch {
// Ignore files that disappear during a refresh; the next lookup rebuilds.
}
}
return parts.sort().join('|');
}
export interface CharacterNameCandidateSet {
/** Identifies this exact form list, so the scan runtime can cache it. */
key: string;
forms: string[];
}
// This lookup is consulted once per subtitle line, so it must not stat the
// snapshot directory every time. Dictionary writes are rare and always call
// invalidate(), which forces the next lookup to re-read; the interval only
// bounds staleness from changes made behind our back.
const SNAPSHOT_SIGNATURE_RECHECK_INTERVAL_MS = 5000;
export function createCharacterNameCandidateLookup(deps: {
userDataPath?: string;
outputDir?: string;
getCurrentMediaId?: () => number | null | undefined;
now?: () => number;
}): {
get: (mediaId?: number | null) => CharacterNameCandidateSet | null;
invalidate: () => void;
} {
const outputDir =
deps.outputDir ??
(deps.userDataPath ? path.join(deps.userDataPath, 'character-dictionaries') : '');
const now = deps.now ?? (() => Date.now());
let signature: string | null = null;
let lastSignatureCheckAtMs = 0;
let formsByMediaId = new Map<number, string[]>();
function refreshIfNeeded(): void {
if (!outputDir) {
formsByMediaId = new Map<number, string[]>();
signature = '';
return;
}
const nowMs = now();
if (
signature !== null &&
nowMs - lastSignatureCheckAtMs < SNAPSHOT_SIGNATURE_RECHECK_INTERVAL_MS
) {
return;
}
lastSignatureCheckAtMs = nowMs;
const nextSignature = getSnapshotDirectorySignature(outputDir);
if (nextSignature === signature) {
return;
}
signature = nextSignature;
formsByMediaId = new Map<number, string[]>();
for (const snapshot of readCachedSnapshots(outputDir)) {
const forms = collectSnapshotNameForms(snapshot);
if (forms.length > 0) {
formsByMediaId.set(snapshot.mediaId, forms);
}
}
}
return {
get(mediaId?: number | null): CharacterNameCandidateSet | null {
refreshIfNeeded();
const rawMediaId = mediaId ?? deps.getCurrentMediaId?.() ?? null;
const normalizedMediaId =
typeof rawMediaId === 'number' && Number.isFinite(rawMediaId) && rawMediaId > 0
? Math.floor(rawMediaId)
: null;
// Without a media scope the pre-pass would need every character of every
// cached title, which is both slow to match and pointless: report no
// candidates so the scanner keeps its exhaustive behavior.
if (normalizedMediaId === null) {
return null;
}
const forms = formsByMediaId.get(normalizedMediaId);
if (!forms || forms.length === 0) {
return null;
}
return { key: `${signature ?? ''}:${normalizedMediaId}`, forms };
},
invalidate(): void {
signature = null;
lastSignatureCheckAtMs = 0;
},
};
}
@@ -9,6 +9,9 @@ type TokenizerMainDeps = TokenizerDepsRuntimeOptions & {
getCurrentCharacterDictionaryMediaId?: NonNullable<
TokenizerDepsRuntimeOptions['getCurrentCharacterDictionaryMediaId']
>;
getCharacterNameCandidates?: NonNullable<
TokenizerDepsRuntimeOptions['getCharacterNameCandidates']
>;
getFrequencyDictionaryEnabled: NonNullable<
TokenizerDepsRuntimeOptions['getFrequencyDictionaryEnabled']
>;
@@ -84,6 +87,11 @@ export function createBuildTokenizerDepsMainHandler(deps: TokenizerMainDeps) {
getCurrentCharacterDictionaryMediaId: () => deps.getCurrentCharacterDictionaryMediaId!(),
}
: {}),
...(deps.getCharacterNameCandidates
? {
getCharacterNameCandidates: () => deps.getCharacterNameCandidates!(),
}
: {}),
getFrequencyDictionaryEnabled: () => deps.getFrequencyDictionaryEnabled(),
getFrequencyDictionaryMatchMode: () => deps.getFrequencyDictionaryMatchMode(),
getFrequencyRank: (text: string) => deps.getFrequencyRank(text),