fix(dictionary): stop large character dictionaries from timing out (#189)

This commit is contained in:
2026-08-11 18:40:18 -07:00
committed by GitHub
parent 7b0fbdf254
commit ee25536d90
16 changed files with 1147 additions and 108 deletions
@@ -0,0 +1,90 @@
import type { CharacterDictionarySnapshotStageProgress } from '../character-dictionary-runtime';
export function buildSyncingMessage(mediaTitle: string): string {
return `Updating character dictionary for ${mediaTitle}...`;
}
export function buildCheckingMessage(mediaTitle: string): string {
return `Checking character dictionary for ${mediaTitle}...`;
}
export function buildGeneratingMessage(mediaTitle: string, detail?: string): string {
return detail
? `Generating character dictionary for ${mediaTitle} (${detail})...`
: `Generating character dictionary for ${mediaTitle}...`;
}
export function formatCharacterDictionaryProgressDetail(
progress: CharacterDictionarySnapshotStageProgress,
remainingMs: number | null,
): string {
if (progress.stage === 'saving') {
return 'saving snapshot';
}
if (progress.stage === 'names') {
return progress.total !== null && progress.total > 0
? `name ${progress.completed}/${progress.total}`
: `${progress.completed} names`;
}
if (progress.stage === 'images') {
const counted =
progress.total !== null && progress.total > 0
? `image ${progress.completed}/${progress.total}`
: `${progress.completed} images`;
return remainingMs !== null
? `${counted}, ~${formatRemainingDuration(remainingMs)} left`
: counted;
}
const page = typeof progress.page === 'number' ? `page ${progress.page}, ` : '';
return `${page}${progress.completed} characters`;
}
export function formatElapsedDuration(elapsedMs: number): string {
const totalSeconds = Math.max(0, Math.floor(elapsedMs / 1000));
const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60;
return minutes > 0 ? `${minutes}m ${String(seconds).padStart(2, '0')}s` : `${seconds}s`;
}
/** Coarser than the elapsed clock: an estimate that ticks every second reads as precision it lacks. */
export function formatRemainingDuration(remainingMs: number): string {
const totalSeconds = Math.max(1, Math.round(remainingMs / 1000));
if (totalSeconds >= 60) {
return `${Math.max(1, Math.round(totalSeconds / 60))}m`;
}
return `${Math.max(5, Math.ceil(totalSeconds / 5) * 5)}s`;
}
/**
* The elapsed clock is the part that proves the app is alive: a stalled network fetch freezes the
* counts, but the clock keeps moving.
*/
export function joinGeneratingDetail(detail: string | null, elapsedMs: number): string | undefined {
const parts = [detail, elapsedMs >= 5_000 ? formatElapsedDuration(elapsedMs) : null].filter(
(part): part is string => typeof part === 'string' && part.length > 0,
);
return parts.length > 0 ? parts.join(' · ') : undefined;
}
export function buildImportingMessage(mediaTitle: string, elapsedMs?: number): string {
const elapsed =
typeof elapsedMs === 'number' && elapsedMs >= 1000
? ` (${formatElapsedDuration(elapsedMs)})`
: '';
return `Importing character dictionary for ${mediaTitle}${elapsed}...`;
}
export function buildBuildingMessage(mediaTitle: string): string {
return `Building character dictionary for ${mediaTitle}...`;
}
export function buildReadyMessage(mediaTitle: string): string {
return `Character dictionary ready for ${mediaTitle}`;
}
export function buildFailedMessage(mediaTitle: string | null, errorMessage: string): string {
if (mediaTitle) {
return `Character dictionary sync failed for ${mediaTitle}: ${errorMessage}`;
}
return `Character dictionary sync failed: ${errorMessage}`;
}
@@ -214,3 +214,69 @@ test('auto sync notifications let startup sequencer own osd-system desktop deliv
assert.deepEqual(calls, ['osd:importing', 'desktop:SubMiner:importing']);
});
test('auto sync desktop notifications reuse one replace id across every phase', () => {
const replaceIds: Array<string | undefined> = [];
const deps = {
getNotificationType: () => 'system' as const,
showOsd: () => undefined,
showDesktopNotification: (_title: string, options: { body?: string; replaceId?: string }) => {
replaceIds.push(options.replaceId);
},
};
for (const phase of ['checking', 'generating', 'importing', 'ready'] as const) {
notifyCharacterDictionaryAutoSyncStatus(makeEvent(phase, phase), deps);
}
assert.deepEqual(replaceIds, [
'character-dictionary-auto-sync',
'character-dictionary-auto-sync',
'character-dictionary-auto-sync',
'character-dictionary-auto-sync',
]);
});
test('overlay-unavailable desktop fallback shares the same replace id', () => {
const replaceIds: Array<string | undefined> = [];
notifyCharacterDictionaryAutoSyncStatus(makeEvent('generating', 'generating'), {
getNotificationType: () => 'overlay',
showOsd: () => undefined,
showDesktopNotification: (_title, options) => {
replaceIds.push(options.replaceId);
},
});
assert.deepEqual(replaceIds, ['character-dictionary-auto-sync']);
});
test('startup lanes keep one desktop notification per lane', () => {
const calls: Array<{ body?: string; replaceId?: string }> = [];
const sequencer = createStartupOsdSequencer({
getNotificationType: () => 'system',
showOsd: () => undefined,
showDesktopNotification: (_title, options) => {
calls.push(options);
},
});
sequencer.markTokenizationReady();
notifyCharacterDictionaryAutoSyncStatus(makeEvent('generating', 'generating one'), {
getNotificationType: () => 'osd',
showOsd: () => undefined,
showDesktopNotification: () => undefined,
startupOsdSequencer: sequencer,
});
notifyCharacterDictionaryAutoSyncStatus(makeEvent('generating', 'generating two'), {
getNotificationType: () => 'osd',
showOsd: () => undefined,
showDesktopNotification: () => undefined,
startupOsdSequencer: sequencer,
});
assert.deepEqual(
calls.map((call) => call.replaceId),
['startup-status', 'startup-status'],
);
});
@@ -9,7 +9,7 @@ export interface CharacterDictionaryAutoSyncNotificationDeps {
getNotificationType: () => NotificationType | undefined;
showOsd: (message: string) => boolean | void;
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
showDesktopNotification: (title: string, options: { body?: string }) => void;
showDesktopNotification: (title: string, options: { body?: string; replaceId?: string }) => void;
startupOsdSequencer?: {
notifyCharacterDictionaryStatus: (
event: StartupOsdSequencerCharacterDictionaryEvent,
@@ -17,6 +17,10 @@ export interface CharacterDictionaryAutoSyncNotificationDeps {
};
}
// One live desktop notification for the whole sync: progress updates replace each other and the
// terminal ready/failed message replaces the last progress one, matching the overlay toast.
const CHARACTER_DICTIONARY_DESKTOP_NOTIFICATION_ID = 'character-dictionary-auto-sync';
function isTerminalPhase(phase: CharacterDictionaryAutoSyncNotificationEvent['phase']): boolean {
return phase === 'ready' || phase === 'failed';
}
@@ -53,7 +57,10 @@ export function notifyCharacterDictionaryAutoSyncStatus(
persistent: !isTerminalPhase(event.phase),
});
} else if (!shouldShowDesktop(type)) {
deps.showDesktopNotification('SubMiner', { body: event.message });
deps.showDesktopNotification('SubMiner', {
body: event.message,
replaceId: CHARACTER_DICTIONARY_DESKTOP_NOTIFICATION_ID,
});
}
}
@@ -69,6 +76,9 @@ export function notifyCharacterDictionaryAutoSyncStatus(
}
if (shouldShowDesktop(type) && !startupSequencerShown) {
deps.showDesktopNotification('SubMiner', { body: event.message });
deps.showDesktopNotification('SubMiner', {
body: event.message,
replaceId: CHARACTER_DICTIONARY_DESKTOP_NOTIFICATION_ID,
});
}
}
@@ -3,6 +3,7 @@ import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import test from 'node:test';
import { buildDictionaryZip } from '../character-dictionary-runtime/zip';
import {
createCharacterDictionaryAutoSyncRuntimeService,
getCharacterDictionaryManagerSnapshot,
@@ -14,6 +15,14 @@ function makeTempDir(): string {
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-char-dict-auto-sync-'));
}
async function waitUntil(predicate: () => boolean, label: string): Promise<void> {
for (let attempt = 0; attempt < 500; attempt += 1) {
if (predicate()) return;
await new Promise((resolve) => setTimeout(resolve, 0));
}
throw new Error(`Timed out waiting for ${label}`);
}
function createDeferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
let resolve!: (value: T) => void;
const promise = new Promise<T>((nextResolve) => {
@@ -187,7 +196,7 @@ test('auto sync imports merged dictionary and persists MRU state', async () => {
'[dictionary:auto-sync] syncing current anime snapshot',
'[dictionary:auto-sync] active AniList media set: 130298 - The Eminence in Shadow',
'[dictionary:auto-sync] rebuilding merged dictionary for active anime set',
'[dictionary:auto-sync] importing merged dictionary: /tmp/subminer-character-dictionary.zip',
'[dictionary:auto-sync] importing merged dictionary: /tmp/subminer-character-dictionary.zip (timeout 120000ms)',
'[dictionary:auto-sync] applying Yomitan settings for SubMiner Character Dictionary',
'[dictionary:auto-sync] synced AniList 130298: SubMiner Character Dictionary (2544 entries)',
]);
@@ -367,7 +376,14 @@ test('auto sync reimports existing merged zip without rebuilding on unchanged re
const userDataPath = makeTempDir();
const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
fs.mkdirSync(dictionariesDir, { recursive: true });
fs.writeFileSync(path.join(dictionariesDir, 'merged.zip'), 'cached-zip', 'utf8');
buildDictionaryZip(
path.join(dictionariesDir, 'merged.zip'),
'SubMiner Character Dictionary',
'Character names',
'rev-7',
[{ term: 'フリーレン', reading: 'フリーレン', role: 'main', glossary: [] } as never],
[],
);
const mergedBuilds: number[][] = [];
const imports: string[] = [];
let importedRevision: string | null = null;
@@ -958,11 +974,9 @@ test('auto sync emits building while merged dictionary generation is in flight',
});
const syncPromise = runtime.runSyncNow();
await Promise.resolve();
assert.equal(
events.some((event) => event.phase === 'building'),
true,
await waitUntil(
() => events.some((event) => event.phase === 'building'),
'the building status event',
);
buildDeferred.resolve({
@@ -1029,8 +1043,7 @@ test('auto sync waits for tokenization-ready gate before Yomitan mutations', asy
});
const syncPromise = runtime.runSyncNow();
await Promise.resolve();
await Promise.resolve();
await waitUntil(() => calls.includes('wait'), 'the tokenization-ready gate');
assert.deepEqual(calls, ['build', 'wait']);
@@ -1039,3 +1052,395 @@ test('auto sync waits for tokenization-ready gate before Yomitan mutations', asy
assert.deepEqual(calls, ['build', 'wait', 'info', 'import', 'settings']);
});
test('auto sync scales the import timeout with the merged dictionary size', async () => {
const userDataPath = makeTempDir();
const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
fs.mkdirSync(dictionariesDir, { recursive: true });
const zipPath = path.join(dictionariesDir, 'merged.zip');
// 2 MB of merged dictionary buys ~12s of import budget on top of the base.
fs.writeFileSync(zipPath, Buffer.alloc(2 * 1024 * 1024));
const events: Array<{ phase: string; message: string }> = [];
let importedRevision: string | null = null;
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath,
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async () => ({
mediaId: 21,
mediaTitle: 'ONE PIECE',
entryCount: 4000,
fromCache: false,
updatedAt: 1000,
}),
buildMergedDictionary: async () => ({
zipPath,
revision: 'rev-21',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 4000,
}),
getYomitanDictionaryInfo: async () =>
importedRevision
? [{ title: 'SubMiner Character Dictionary', revision: importedRevision }]
: [],
importYomitanDictionary: async () => {
// Far longer than the quick-operation budget, well inside the size-scaled one.
await new Promise((resolve) => setTimeout(resolve, 400));
importedRevision = 'rev-21';
return true;
},
deleteYomitanDictionary: async () => true,
upsertYomitanDictionarySettings: async () => true,
now: () => 1000,
// Comfortable for the stubs that resolve immediately, still far under the import's 400ms.
operationTimeoutMs: 100,
dictionaryImportTimeoutBaseMs: 20,
onSyncStatus: (event) => {
events.push({ phase: event.phase, message: event.message });
},
});
await runtime.runSyncNow();
assert.equal(
events.some((event) => event.phase === 'failed'),
false,
);
assert.deepEqual(events.at(-1), {
phase: 'ready',
message: 'Character dictionary ready for ONE PIECE',
});
});
test('auto sync reports the scaled budget when an import really does hang', async () => {
const userDataPath = makeTempDir();
const events: Array<{ phase: string; message: string }> = [];
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath,
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async () => ({
mediaId: 21,
mediaTitle: 'ONE PIECE',
entryCount: 4000,
fromCache: false,
updatedAt: 1000,
}),
buildMergedDictionary: async () => ({
zipPath: path.join(userDataPath, 'character-dictionaries', 'missing.zip'),
revision: 'rev-21',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 4000,
}),
getYomitanDictionaryInfo: async () => [],
importYomitanDictionary: () => new Promise<boolean>(() => {}),
deleteYomitanDictionary: async () => true,
upsertYomitanDictionarySettings: async () => true,
now: () => 1000,
dictionaryImportTimeoutBaseMs: 20,
onSyncStatus: (event) => {
events.push({ phase: event.phase, message: event.message });
},
});
await assert.rejects(
runtime.runSyncNow(),
/importYomitanDictionary\(missing\.zip\) timed out after 20ms/,
);
assert.equal(events.at(-1)?.phase, 'failed');
});
test('auto sync ticks the importing notification while the import runs', async () => {
const userDataPath = makeTempDir();
const events: Array<{ phase: string; message: string }> = [];
const scheduled: Array<() => void> = [];
const importDeferred = createDeferred<boolean>();
let clock = 1000;
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath,
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async () => ({
mediaId: 21,
mediaTitle: 'ONE PIECE',
entryCount: 4000,
fromCache: false,
updatedAt: 1000,
}),
buildMergedDictionary: async () => ({
zipPath: '/tmp/merged.zip',
revision: 'rev-21',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 4000,
}),
getYomitanDictionaryInfo: async () => [],
importYomitanDictionary: () => importDeferred.promise,
deleteYomitanDictionary: async () => true,
upsertYomitanDictionarySettings: async () => true,
now: () => clock,
schedule: (fn) => {
scheduled.push(fn);
return 0 as unknown as ReturnType<typeof setTimeout>;
},
clearSchedule: () => undefined,
onSyncStatus: (event) => {
events.push({ phase: event.phase, message: event.message });
},
});
const syncPromise = runtime.runSyncNow();
await waitUntil(
() => events.some((event) => event.phase === 'importing'),
'the importing status event',
);
clock = 1000 + 65_000;
// The importing heartbeat is the most recently scheduled tick.
scheduled.at(-1)!();
assert.deepEqual(events.at(-1), {
phase: 'importing',
message: 'Importing character dictionary for ONE PIECE (1m 05s)...',
});
importDeferred.resolve(true);
await syncPromise;
assert.equal(events.at(-1)?.phase, 'ready');
});
test('auto sync reports character and image counts while generating a snapshot', async () => {
const userDataPath = makeTempDir();
const events: Array<{ phase: string; message: string }> = [];
let clock = 1000;
let importedRevision: string | null = null;
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath,
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async (_targetPath, progress) => {
progress?.onGenerating?.({ mediaId: 21, mediaTitle: 'ONE PIECE' });
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'characters',
completed: 50,
total: null,
page: 12,
});
// Same stage, same clock tick: throttled away so a 33-page fetch cannot spam the overlay.
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'characters',
completed: 100,
total: null,
page: 13,
});
// A stage change always reports, throttle window or not.
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'images',
completed: 1,
total: 1220,
});
clock += 2000;
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'images',
completed: 240,
total: 1220,
});
clock += 6000;
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'names',
completed: 800,
total: 1220,
});
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'saving',
completed: 0,
total: null,
});
return {
mediaId: 21,
mediaTitle: 'ONE PIECE',
entryCount: 4000,
fromCache: false,
updatedAt: 1000,
};
},
buildMergedDictionary: async () => ({
zipPath: '/tmp/merged.zip',
revision: 'rev-21',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 4000,
}),
getYomitanDictionaryInfo: async () =>
importedRevision
? [{ title: 'SubMiner Character Dictionary', revision: importedRevision }]
: [],
importYomitanDictionary: async () => {
importedRevision = 'rev-21';
return true;
},
deleteYomitanDictionary: async () => true,
upsertYomitanDictionarySettings: async () => true,
now: () => clock,
onSyncStatus: (event) => {
events.push({ phase: event.phase, message: event.message });
},
});
await runtime.runSyncNow();
assert.deepEqual(
events.filter((event) => event.phase === 'generating').map((event) => event.message),
[
'Generating character dictionary for ONE PIECE...',
'Generating character dictionary for ONE PIECE (page 12, 50 characters)...',
'Generating character dictionary for ONE PIECE (image 1/1220)...',
'Generating character dictionary for ONE PIECE (image 240/1220, ~10s left)...',
'Generating character dictionary for ONE PIECE (name 800/1220 · 8s)...',
'Generating character dictionary for ONE PIECE (saving snapshot · 8s)...',
],
);
});
test('auto sync keeps the generating clock ticking when a stage stalls', async () => {
const userDataPath = makeTempDir();
const events: Array<{ phase: string; message: string }> = [];
const scheduled: Array<() => void> = [];
const snapshotDeferred = createDeferred<{
mediaId: number;
mediaTitle: string;
entryCount: number;
fromCache: boolean;
updatedAt: number;
}>();
let clock = 1000;
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath,
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async (_targetPath, progress) => {
progress?.onGenerating?.({ mediaId: 21, mediaTitle: 'ONE PIECE' });
progress?.onGenerateProgress?.({
mediaId: 21,
mediaTitle: 'ONE PIECE',
stage: 'images',
completed: 240,
total: 1220,
});
return await snapshotDeferred.promise;
},
buildMergedDictionary: async () => ({
zipPath: '/tmp/merged.zip',
revision: 'rev-21',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 4000,
}),
getYomitanDictionaryInfo: async () => [],
importYomitanDictionary: async () => true,
deleteYomitanDictionary: async () => true,
upsertYomitanDictionarySettings: async () => true,
now: () => clock,
schedule: (fn) => {
scheduled.push(fn);
return 0 as unknown as ReturnType<typeof setTimeout>;
},
clearSchedule: () => undefined,
onSyncStatus: (event) => {
events.push({ phase: event.phase, message: event.message });
},
});
const syncPromise = runtime.runSyncNow();
await waitUntil(() => scheduled.length > 0, 'the generating heartbeat');
// No further progress arrives: only the clock moves.
clock += 95_000;
scheduled.at(-1)!();
assert.deepEqual(events.at(-1), {
phase: 'generating',
message: 'Generating character dictionary for ONE PIECE (image 240/1220 · 1m 35s)...',
});
snapshotDeferred.resolve({
mediaId: 21,
mediaTitle: 'ONE PIECE',
entryCount: 4000,
fromCache: false,
updatedAt: 1000,
});
await syncPromise;
assert.equal(events.at(-1)?.phase, 'ready');
});
test('auto sync rebuilds instead of importing a cached merged ZIP with a mismatched revision', async () => {
const userDataPath = makeTempDir();
const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
fs.mkdirSync(dictionariesDir, { recursive: true });
const statePath = path.join(dictionariesDir, 'auto-sync-state.json');
fs.writeFileSync(
statePath,
JSON.stringify({
activeMediaIds: ['7 - Frieren'],
mergedRevision: 'rev-7',
mergedDictionaryTitle: 'SubMiner Character Dictionary',
}),
'utf8',
);
// Left over from an interrupted run: the archive on disk is not the revision state recorded.
buildDictionaryZip(
path.join(dictionariesDir, 'merged.zip'),
'SubMiner Character Dictionary',
'Character names',
'rev-stale',
[{ term: 'フリーレン', reading: 'フリーレン', role: 'main', glossary: [] } as never],
[],
);
const mergedBuilds: number[][] = [];
const imports: string[] = [];
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
userDataPath,
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
getOrCreateCurrentSnapshot: async () => ({
mediaId: 7,
mediaTitle: 'Frieren',
entryCount: 100,
fromCache: true,
updatedAt: 1000,
}),
buildMergedDictionary: async (mediaIds) => {
mergedBuilds.push([...mediaIds]);
return {
zipPath: '/tmp/rebuilt-merged.zip',
revision: 'rev-7',
dictionaryTitle: 'SubMiner Character Dictionary',
entryCount: 100,
};
},
// Yomitan does not have the dictionary, so the sync has to import despite the cached state.
getYomitanDictionaryInfo: async () => [],
importYomitanDictionary: async (zipPath) => {
imports.push(zipPath);
return true;
},
deleteYomitanDictionary: async () => true,
upsertYomitanDictionarySettings: async () => true,
now: () => 1000,
});
await runtime.runSyncNow();
assert.deepEqual(mergedBuilds, [[7]]);
assert.deepEqual(imports, ['/tmp/rebuilt-merged.zip']);
});
@@ -1,13 +1,29 @@
import * as fs from 'fs';
import * as path from 'path';
import { ensureDir } from '../../shared/fs-utils';
import {
buildBuildingMessage,
buildCheckingMessage,
buildFailedMessage,
buildGeneratingMessage,
buildImportingMessage,
buildReadyMessage,
formatCharacterDictionaryProgressDetail,
joinGeneratingDetail,
} from './character-dictionary-auto-sync-messages';
import { readDictionaryZipRevision } from '../character-dictionary-runtime/zip';
import type { AnilistCharacterDictionaryProfileScope } from '../../types';
import type {
CharacterDictionarySnapshotProgressCallbacks,
CharacterDictionarySnapshotResult,
CharacterDictionarySnapshotStageProgress,
MergedCharacterDictionaryBuildResult,
} from '../character-dictionary-runtime';
const DEFAULT_IMPORT_TIMEOUT_BASE_MS = 120_000;
const IMPORT_TIMEOUT_MS_PER_MB = 6_000;
const IMPORT_TIMEOUT_MAX_MS = 1_800_000;
type AutoSyncMediaEntry = {
mediaId: number;
label: string;
@@ -72,7 +88,15 @@ export interface CharacterDictionaryAutoSyncRuntimeDeps {
now: () => number;
schedule?: (fn: () => void, delayMs: number) => ReturnType<typeof setTimeout>;
clearSchedule?: (timer: ReturnType<typeof setTimeout>) => void;
/** Budget for the quick Yomitan queries (dictionary info, settings upsert). */
operationTimeoutMs?: number;
/**
* Base budget for the slow Yomitan mutations (delete + import). The effective budget grows with
* the merged ZIP size, because importing a large dictionary can take several minutes.
*/
dictionaryImportTimeoutBaseMs?: number;
heartbeatMs?: number;
progressThrottleMs?: number;
logInfo?: (message: string) => void;
logWarn?: (message: string) => void;
onSyncStatus?: (event: CharacterDictionaryAutoSyncStatusEvent) => void;
@@ -345,37 +369,6 @@ function sameMembership(left: number[], right: number[]): boolean {
return arraysEqual(leftSorted, rightSorted);
}
function buildSyncingMessage(mediaTitle: string): string {
return `Updating character dictionary for ${mediaTitle}...`;
}
function buildCheckingMessage(mediaTitle: string): string {
return `Checking character dictionary for ${mediaTitle}...`;
}
function buildGeneratingMessage(mediaTitle: string): string {
return `Generating character dictionary for ${mediaTitle}...`;
}
function buildImportingMessage(mediaTitle: string): string {
return `Importing character dictionary for ${mediaTitle}...`;
}
function buildBuildingMessage(mediaTitle: string): string {
return `Building character dictionary for ${mediaTitle}...`;
}
function buildReadyMessage(mediaTitle: string): string {
return `Character dictionary ready for ${mediaTitle}`;
}
function buildFailedMessage(mediaTitle: string | null, errorMessage: string): string {
if (mediaTitle) {
return `Character dictionary sync failed for ${mediaTitle}: ${errorMessage}`;
}
return `Character dictionary sync failed: ${errorMessage}`;
}
export function createCharacterDictionaryAutoSyncRuntimeService(
deps: CharacterDictionaryAutoSyncRuntimeDeps,
): {
@@ -389,21 +382,31 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
const clearSchedule = deps.clearSchedule ?? ((timer) => clearTimeout(timer));
const debounceMs = 800;
const operationTimeoutMs = Math.max(1, Math.floor(deps.operationTimeoutMs ?? 7_000));
const dictionaryImportTimeoutBaseMs = Math.max(
1,
Math.floor(deps.dictionaryImportTimeoutBaseMs ?? DEFAULT_IMPORT_TIMEOUT_BASE_MS),
);
const heartbeatMs = Math.max(1, Math.floor(deps.heartbeatMs ?? 5_000));
const progressThrottleMs = Math.max(0, Math.floor(deps.progressThrottleMs ?? 1_000));
let debounceTimer: ReturnType<typeof setTimeout> | null = null;
let syncInFlight = false;
let runQueued = false;
let activeCurrentMediaId: number | null = null;
const withOperationTimeout = async <T>(label: string, promise: Promise<T>): Promise<T> => {
const withTimeout = async <T>(
label: string,
promise: Promise<T>,
timeoutMs: number,
): Promise<T> => {
let timer: ReturnType<typeof setTimeout> | null = null;
try {
return await Promise.race([
promise,
new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => {
reject(new Error(`${label} timed out after ${operationTimeoutMs}ms`));
}, operationTimeoutMs);
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
}, timeoutMs);
}),
]);
} finally {
@@ -413,6 +416,57 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
}
};
const withOperationTimeout = <T>(label: string, promise: Promise<T>): Promise<T> =>
withTimeout(label, promise, operationTimeoutMs);
/**
* Importing a merged dictionary means Yomitan writing every term and image into IndexedDB, which
* scales with the ZIP: a single-season dictionary lands in seconds, One Piece takes minutes. Size
* the budget off the archive instead of failing a healthy import on a flat deadline.
*/
const resolveImportTimeoutMs = (zipPath: string | null): number => {
let bytes = 0;
if (zipPath) {
try {
bytes = fs.statSync(zipPath).size;
} catch {
bytes = 0;
}
}
const sizeAllowanceMs = (bytes / (1024 * 1024)) * IMPORT_TIMEOUT_MS_PER_MB;
return Math.min(
IMPORT_TIMEOUT_MAX_MS,
Math.max(
dictionaryImportTimeoutBaseMs,
Math.round(dictionaryImportTimeoutBaseMs + sizeAllowanceMs),
),
);
};
/** Keeps the persistent notification ticking so a multi-minute import never looks hung. */
const withHeartbeat = async <T>(
run: () => Promise<T>,
onTick: (elapsedMs: number) => void,
): Promise<T> => {
const startedAt = deps.now();
let stopped = false;
let timer: ReturnType<typeof setTimeout> | null = null;
const tick = (): void => {
if (stopped) return;
onTick(deps.now() - startedAt);
timer = schedule(tick, heartbeatMs);
};
timer = schedule(tick, heartbeatMs);
try {
return await run();
} finally {
stopped = true;
if (timer !== null) {
clearSchedule(timer);
}
}
};
const runSyncOnce = async (): Promise<void> => {
const config = deps.getConfig();
if (!config.enabled) {
@@ -422,33 +476,112 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
let currentMediaId: number | undefined;
let currentMediaTitle: string | null = null;
let lastProgressAt = Number.NEGATIVE_INFINITY;
let lastProgressStage: CharacterDictionarySnapshotStageProgress['stage'] | null = null;
let generating: {
mediaId: number;
mediaTitle: string;
startedAt: number;
detail: string | null;
} | null = null;
let imageRate: { startedAt: number; startCompleted: number } | null = null;
const emitGeneratingStatus = (): void => {
if (!generating) {
return;
}
deps.onSyncStatus?.({
phase: 'generating',
mediaId: generating.mediaId,
mediaTitle: generating.mediaTitle,
message: buildGeneratingMessage(
generating.mediaTitle,
joinGeneratingDetail(generating.detail, deps.now() - generating.startedAt),
),
});
};
// Image downloads are serial and evenly paced, so the observed rate predicts the tail well.
const estimateRemainingImageMs = (
progress: CharacterDictionarySnapshotStageProgress,
nowMs: number,
): number | null => {
if (progress.stage !== 'images' || progress.total === null || imageRate === null) {
return null;
}
const done = progress.completed - imageRate.startCompleted;
const elapsedMs = nowMs - imageRate.startedAt;
const remaining = progress.total - progress.completed;
if (done <= 0 || elapsedMs <= 0 || remaining <= 0) {
return null;
}
return Math.round((elapsedMs / done) * remaining);
};
try {
deps.logInfo?.('[dictionary:auto-sync] syncing current anime snapshot');
const snapshot = await deps.getOrCreateCurrentSnapshot(undefined, {
onChecking: ({ mediaId, mediaTitle }) => {
currentMediaId = mediaId;
currentMediaTitle = mediaTitle;
activeCurrentMediaId = mediaId;
deps.onSyncStatus?.({
phase: 'checking',
mediaId,
mediaTitle,
message: buildCheckingMessage(mediaTitle),
});
},
onGenerating: ({ mediaId, mediaTitle }) => {
currentMediaId = mediaId;
currentMediaTitle = mediaTitle;
activeCurrentMediaId = mediaId;
deps.onSyncStatus?.({
phase: 'generating',
mediaId,
mediaTitle,
message: buildGeneratingMessage(mediaTitle),
});
},
});
const snapshot = await withHeartbeat(
() =>
deps.getOrCreateCurrentSnapshot(undefined, {
onChecking: ({ mediaId, mediaTitle }) => {
currentMediaId = mediaId;
currentMediaTitle = mediaTitle;
activeCurrentMediaId = mediaId;
deps.onSyncStatus?.({
phase: 'checking',
mediaId,
mediaTitle,
message: buildCheckingMessage(mediaTitle),
});
},
onGenerating: ({ mediaId, mediaTitle }) => {
currentMediaId = mediaId;
currentMediaTitle = mediaTitle;
activeCurrentMediaId = mediaId;
lastProgressAt = Number.NEGATIVE_INFINITY;
lastProgressStage = null;
imageRate = null;
generating = { mediaId, mediaTitle, startedAt: deps.now(), detail: null };
emitGeneratingStatus();
},
// Long-running work (AniList character pages, then one image download per character
// and voice actor, then MeCab name splits) reports counts so the notification shows
// movement instead of a frozen "Generating..." for the minutes a large series takes.
onGenerateProgress: (progress) => {
const nowMs = deps.now();
if (!generating) {
generating = {
mediaId: progress.mediaId,
mediaTitle: progress.mediaTitle,
startedAt: nowMs,
detail: null,
};
}
if (progress.stage === 'images' && imageRate === null) {
imageRate = { startedAt: nowMs, startCompleted: progress.completed };
}
// Stage changes and the last item of a stage always report; the throttle only thins
// out the run of identical-looking updates in between.
const isFinal = progress.total !== null && progress.completed >= progress.total;
const isStageChange = progress.stage !== lastProgressStage;
if (!isFinal && !isStageChange && nowMs - lastProgressAt < progressThrottleMs) {
return;
}
lastProgressAt = nowMs;
lastProgressStage = progress.stage;
generating.mediaId = progress.mediaId;
generating.mediaTitle = progress.mediaTitle;
generating.detail = formatCharacterDictionaryProgressDetail(
progress,
estimateRemainingImageMs(progress, nowMs),
);
emitGeneratingStatus();
},
}),
// Ticks even when a step stalls, so the message keeps moving while the counts do not.
() => emitGeneratingStatus(),
);
generating = null;
currentMediaId = snapshot.mediaId;
currentMediaTitle = snapshot.mediaTitle;
activeCurrentMediaId = snapshot.mediaId;
@@ -531,33 +664,61 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
mediaTitle: snapshot.mediaTitle,
message: buildImportingMessage(snapshot.mediaTitle),
});
if (existing !== null) {
await withOperationTimeout(
`deleteYomitanDictionary(${dictionaryTitle})`,
deps.deleteYomitanDictionary(dictionaryTitle),
);
}
if (merged === null) {
const existingMergedZipPath = path.join(dictionariesDir, 'merged.zip');
if (fs.existsSync(existingMergedZipPath)) {
merged = {
zipPath: existingMergedZipPath,
revision,
dictionaryTitle,
entryCount: snapshot.entryCount,
};
} else {
merged = await deps.buildMergedDictionary(nextActiveMediaIdValues);
}
}
deps.logInfo?.(`[dictionary:auto-sync] importing merged dictionary: ${merged.zipPath}`);
const imported = await withOperationTimeout(
`importYomitanDictionary(${path.basename(merged.zipPath)})`,
deps.importYomitanDictionary(merged.zipPath),
await withHeartbeat(
async () => {
const importTimeoutMs = resolveImportTimeoutMs(
merged?.zipPath ?? path.join(dictionariesDir, 'merged.zip'),
);
if (existing !== null) {
await withTimeout(
`deleteYomitanDictionary(${dictionaryTitle})`,
deps.deleteYomitanDictionary(dictionaryTitle),
importTimeoutMs,
);
}
if (merged === null) {
// The cached archive only stands in for the recorded state when its own index.json
// agrees. A stale or half-written ZIP would be imported under the wrong revision,
// and every later sync would then see a revision mismatch and re-import it.
const existingMergedZipPath = path.join(dictionariesDir, 'merged.zip');
const existingMergedRevision = readDictionaryZipRevision(existingMergedZipPath);
if (existingMergedRevision === revision) {
merged = {
zipPath: existingMergedZipPath,
revision,
dictionaryTitle,
entryCount: snapshot.entryCount,
};
} else {
deps.logInfo?.(
`[dictionary:auto-sync] cached merged ZIP unusable (revision ${existingMergedRevision ?? 'unreadable'}, expected ${revision}); rebuilding`,
);
merged = await deps.buildMergedDictionary(nextActiveMediaIdValues);
}
}
const mergedZipPath = merged.zipPath;
const mergedImportTimeoutMs = resolveImportTimeoutMs(mergedZipPath);
deps.logInfo?.(
`[dictionary:auto-sync] importing merged dictionary: ${mergedZipPath} (timeout ${mergedImportTimeoutMs}ms)`,
);
const imported = await withTimeout(
`importYomitanDictionary(${path.basename(mergedZipPath)})`,
deps.importYomitanDictionary(mergedZipPath),
mergedImportTimeoutMs,
);
if (!imported) {
throw new Error(`Failed to import dictionary ZIP: ${merged.zipPath}`);
}
},
(elapsedMs) => {
deps.onSyncStatus?.({
phase: 'importing',
mediaId: snapshot.mediaId,
mediaTitle: snapshot.mediaTitle,
message: buildImportingMessage(snapshot.mediaTitle, elapsedMs),
});
},
);
if (!imported) {
throw new Error(`Failed to import dictionary ZIP: ${merged.zipPath}`);
}
changed = true;
}
+6 -2
View File
@@ -10,7 +10,7 @@ export interface StartupOsdSequencerDeps {
getNotificationType?: () => NotificationType | undefined;
showOsd: (message: string) => boolean | void;
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
showDesktopNotification?: (title: string, options: { body?: string }) => void;
showDesktopNotification?: (title: string, options: { body?: string; replaceId?: string }) => void;
}
interface StartupStatusNotificationOptions {
@@ -62,7 +62,11 @@ export function createStartupOsdSequencer(deps: StartupOsdSequencerDeps): {
shown = deps.showOsd(options.message) !== false || shown;
}
if (options.desktop !== false && shouldShowDesktop(type)) {
deps.showDesktopNotification?.('SubMiner', { body: options.message });
// Each startup lane keeps one live desktop notification instead of one per update.
deps.showDesktopNotification?.('SubMiner', {
body: options.message,
replaceId: options.id,
});
shown = true;
}
return shown;