mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-07 07:21:35 -07:00
fix(dictionary): stop large character dictionaries from timing out
- Size the Yomitan import/delete timeout off the merged ZIP (2min + 6s/MB, capped 30min) instead of the flat 7s query budget, so a big series like One Piece no longer fails mid-import - Report generation progress by stage (AniList page/character count, image N/total with ETA, name-split N/total, saving) instead of one static "Generating..." message - Add an elapsed-time heartbeat to generating/importing notifications, ticking every 5s even when a stage stalls, so a slow step reads as busy rather than hung
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
type: fixed
|
||||
area: character dictionary
|
||||
|
||||
- A large character dictionary no longer fails to install. The Yomitan import (and the delete that precedes it) used a flat 7 second budget, so a series like One Piece failed with `importYomitanDictionary(merged.zip) timed out after 7000ms` while the import was still healthy. The budget is now sized from the merged ZIP (2 minutes plus 6 seconds per MB, capped at 30 minutes); the quick queries keep the 7 second budget.
|
||||
- The "Generating character dictionary" notification now reports what it is doing instead of one static message for the whole run: the AniList page and character count while characters download (`page 12, 587 characters`), then `image 240/1220, ~4m left` per downloaded character/voice actor image, then `name 800/1220` for MeCab name splits and `saving snapshot` at the end. Updates are throttled to one per second, and every stage change and final item always reports.
|
||||
- Every long phase also carries an elapsed clock (`· 1m 35s`), refreshed every 5 seconds even when nothing else moves, so a stalled step is visibly different from a frozen app. Long imports tick the same way.
|
||||
- Image downloads are also logged every 100 files, and the import logs the timeout it computed.
|
||||
@@ -1903,7 +1903,7 @@ test('generateForCurrentMedia logs progress while resolving and rebuilding snaps
|
||||
'[dictionary] current anime guess: The Eminence in Shadow (episode 5)',
|
||||
'[dictionary] AniList match: The Eminence in Shadow -> AniList 130298',
|
||||
'[dictionary] snapshot miss for AniList 130298, fetching characters',
|
||||
'[dictionary] downloaded AniList character page 1 for AniList 130298',
|
||||
'[dictionary] downloaded AniList character page 1 for AniList 130298 (1 characters)',
|
||||
'[dictionary] downloading 1 images for AniList 130298',
|
||||
'[dictionary] stored snapshot for AniList 130298: 16 terms',
|
||||
'[dictionary] building ZIP for AniList 130298',
|
||||
|
||||
@@ -64,6 +64,7 @@ export type {
|
||||
CharacterDictionarySnapshotProgress,
|
||||
CharacterDictionarySnapshotProgressCallbacks,
|
||||
CharacterDictionarySnapshotResult,
|
||||
CharacterDictionarySnapshotStageProgress,
|
||||
MergedCharacterDictionaryBuildResult,
|
||||
} from './character-dictionary-runtime/types';
|
||||
|
||||
@@ -363,19 +364,28 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
deps.logInfo?.(`[dictionary] snapshot stale for AniList ${mediaId}: ${refreshReason}`);
|
||||
}
|
||||
|
||||
const progressMediaTitle = mediaTitleHint || `AniList ${mediaId}`;
|
||||
progress?.onGenerating?.({
|
||||
mediaId,
|
||||
mediaTitle: mediaTitleHint || `AniList ${mediaId}`,
|
||||
mediaTitle: progressMediaTitle,
|
||||
});
|
||||
deps.logInfo?.(`[dictionary] snapshot miss for AniList ${mediaId}, fetching characters`);
|
||||
|
||||
const { mediaTitle: fetchedMediaTitle, characters } = await fetchCharactersForMedia(
|
||||
mediaId,
|
||||
beforeRequest,
|
||||
(page) => {
|
||||
(page, charactersSoFar) => {
|
||||
deps.logInfo?.(
|
||||
`[dictionary] downloaded AniList character page ${page} for AniList ${mediaId}`,
|
||||
`[dictionary] downloaded AniList character page ${page} for AniList ${mediaId} (${charactersSoFar} characters)`,
|
||||
);
|
||||
progress?.onGenerateProgress?.({
|
||||
mediaId,
|
||||
mediaTitle: progressMediaTitle,
|
||||
stage: 'characters',
|
||||
completed: charactersSoFar,
|
||||
total: null,
|
||||
page,
|
||||
});
|
||||
},
|
||||
);
|
||||
if (characters.length === 0) {
|
||||
@@ -403,12 +413,26 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
);
|
||||
}
|
||||
let hasAttemptedImageDownload = false;
|
||||
let attemptedImageCount = 0;
|
||||
for (const entry of allImageUrls) {
|
||||
if (hasAttemptedImageDownload) {
|
||||
await sleepMs(CHARACTER_IMAGE_DOWNLOAD_DELAY_MS);
|
||||
}
|
||||
hasAttemptedImageDownload = true;
|
||||
const image = await downloadCharacterImage(entry.url, entry.id);
|
||||
attemptedImageCount += 1;
|
||||
progress?.onGenerateProgress?.({
|
||||
mediaId,
|
||||
mediaTitle: progressMediaTitle,
|
||||
stage: 'images',
|
||||
completed: attemptedImageCount,
|
||||
total: allImageUrls.length,
|
||||
});
|
||||
if (attemptedImageCount % 100 === 0) {
|
||||
deps.logInfo?.(
|
||||
`[dictionary] downloaded ${attemptedImageCount}/${allImageUrls.length} images for AniList ${mediaId}`,
|
||||
);
|
||||
}
|
||||
if (!image) continue;
|
||||
if (entry.kind === 'character') {
|
||||
imagesByCharacterId.set(entry.id, {
|
||||
@@ -425,11 +449,31 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
||||
|
||||
const nameSplitTokenizerAvailable = isNameSplitTokenizerAvailable();
|
||||
const resolvedNameSplits = nameSplitTokenizerAvailable
|
||||
? await resolveJapaneseNameSplits(characters, deps.tokenizeJapaneseName!, deps.logWarn)
|
||||
? await resolveJapaneseNameSplits(
|
||||
characters,
|
||||
deps.tokenizeJapaneseName!,
|
||||
deps.logWarn,
|
||||
(completed, total) => {
|
||||
progress?.onGenerateProgress?.({
|
||||
mediaId,
|
||||
mediaTitle: progressMediaTitle,
|
||||
stage: 'names',
|
||||
completed,
|
||||
total,
|
||||
});
|
||||
},
|
||||
)
|
||||
: undefined;
|
||||
const nameSplitSource =
|
||||
resolvedNameSplits && resolvedNameSplits.size > 0 ? 'mecab' : 'heuristic';
|
||||
|
||||
progress?.onGenerateProgress?.({
|
||||
mediaId,
|
||||
mediaTitle: progressMediaTitle,
|
||||
stage: 'saving',
|
||||
completed: 0,
|
||||
total: null,
|
||||
});
|
||||
const snapshot = buildSnapshotFromCharacters(
|
||||
mediaId,
|
||||
fetchedMediaTitle || mediaTitleHint || `AniList ${mediaId}`,
|
||||
|
||||
@@ -278,7 +278,7 @@ export async function fetchAniListMediaCandidateById(
|
||||
export async function fetchCharactersForMedia(
|
||||
mediaId: number,
|
||||
beforeRequest?: () => Promise<void>,
|
||||
onPageFetched?: (page: number) => void,
|
||||
onPageFetched?: (page: number, charactersSoFar: number) => void,
|
||||
): Promise<{
|
||||
mediaTitle: string;
|
||||
characters: CharacterRecord[];
|
||||
@@ -345,7 +345,6 @@ export async function fetchCharactersForMedia(
|
||||
},
|
||||
beforeRequest,
|
||||
);
|
||||
onPageFetched?.(page);
|
||||
|
||||
const media = data.Media;
|
||||
if (!media) {
|
||||
@@ -415,6 +414,8 @@ export async function fetchCharactersForMedia(
|
||||
});
|
||||
}
|
||||
|
||||
onPageFetched?.(page, characters.length);
|
||||
|
||||
const hasNextPage = Boolean(media.characters?.pageInfo?.hasNextPage);
|
||||
if (!hasNextPage) {
|
||||
break;
|
||||
|
||||
@@ -86,8 +86,10 @@ export async function resolveJapaneseNameSplits(
|
||||
characters: CharacterRecord[],
|
||||
tokenize: NameSplitTokenizer,
|
||||
logWarn?: (message: string) => void,
|
||||
onCharacterResolved?: (completed: number, total: number) => void,
|
||||
): Promise<Map<string, ResolvedNameSplit>> {
|
||||
const splits = new Map<string, ResolvedNameSplit>();
|
||||
let resolvedCharacters = 0;
|
||||
for (const character of characters) {
|
||||
const familyHintReading = buildReadingFromHint(character.lastNameHint?.trim() || '');
|
||||
const givenHintReading = buildReadingFromHint(character.firstNameHint?.trim() || '');
|
||||
@@ -113,6 +115,8 @@ export async function resolveJapaneseNameSplits(
|
||||
splits.set(name, { family, given });
|
||||
}
|
||||
}
|
||||
resolvedCharacters += 1;
|
||||
onCharacterResolved?.(resolvedCharacters, characters.length);
|
||||
}
|
||||
return splits;
|
||||
}
|
||||
|
||||
@@ -122,9 +122,24 @@ export type CharacterDictionarySnapshotProgress = {
|
||||
mediaTitle: string;
|
||||
};
|
||||
|
||||
export type CharacterDictionarySnapshotStage = 'characters' | 'images' | 'names' | 'saving';
|
||||
|
||||
/**
|
||||
* Fine-grained generation progress. `total` is null while the work size is still unknown (AniList
|
||||
* paginates characters, so the character count only settles on the last page).
|
||||
*/
|
||||
export type CharacterDictionarySnapshotStageProgress = CharacterDictionarySnapshotProgress & {
|
||||
stage: CharacterDictionarySnapshotStage;
|
||||
completed: number;
|
||||
total: number | null;
|
||||
/** AniList page currently being downloaded; only set during the `characters` stage. */
|
||||
page?: number;
|
||||
};
|
||||
|
||||
export type CharacterDictionarySnapshotProgressCallbacks = {
|
||||
onChecking?: (progress: CharacterDictionarySnapshotProgress) => void;
|
||||
onGenerating?: (progress: CharacterDictionarySnapshotProgress) => void;
|
||||
onGenerateProgress?: (progress: CharacterDictionarySnapshotStageProgress) => void;
|
||||
};
|
||||
|
||||
export type MergedCharacterDictionaryBuildResult = {
|
||||
|
||||
@@ -14,6 +14,14 @@ function makeTempDir(): string {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-char-dict-auto-sync-'));
|
||||
}
|
||||
|
||||
async function waitUntil(predicate: () => boolean, label: string): Promise<void> {
|
||||
for (let attempt = 0; attempt < 500; attempt += 1) {
|
||||
if (predicate()) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${label}`);
|
||||
}
|
||||
|
||||
function createDeferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>((nextResolve) => {
|
||||
@@ -187,7 +195,7 @@ test('auto sync imports merged dictionary and persists MRU state', async () => {
|
||||
'[dictionary:auto-sync] syncing current anime snapshot',
|
||||
'[dictionary:auto-sync] active AniList media set: 130298 - The Eminence in Shadow',
|
||||
'[dictionary:auto-sync] rebuilding merged dictionary for active anime set',
|
||||
'[dictionary:auto-sync] importing merged dictionary: /tmp/subminer-character-dictionary.zip',
|
||||
'[dictionary:auto-sync] importing merged dictionary: /tmp/subminer-character-dictionary.zip (timeout 120000ms)',
|
||||
'[dictionary:auto-sync] applying Yomitan settings for SubMiner Character Dictionary',
|
||||
'[dictionary:auto-sync] synced AniList 130298: SubMiner Character Dictionary (2544 entries)',
|
||||
]);
|
||||
@@ -958,11 +966,9 @@ test('auto sync emits building while merged dictionary generation is in flight',
|
||||
});
|
||||
|
||||
const syncPromise = runtime.runSyncNow();
|
||||
await Promise.resolve();
|
||||
|
||||
assert.equal(
|
||||
events.some((event) => event.phase === 'building'),
|
||||
true,
|
||||
await waitUntil(
|
||||
() => events.some((event) => event.phase === 'building'),
|
||||
'the building status event',
|
||||
);
|
||||
|
||||
buildDeferred.resolve({
|
||||
@@ -1029,8 +1035,7 @@ test('auto sync waits for tokenization-ready gate before Yomitan mutations', asy
|
||||
});
|
||||
|
||||
const syncPromise = runtime.runSyncNow();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await waitUntil(() => calls.includes('wait'), 'the tokenization-ready gate');
|
||||
|
||||
assert.deepEqual(calls, ['build', 'wait']);
|
||||
|
||||
@@ -1039,3 +1044,332 @@ test('auto sync waits for tokenization-ready gate before Yomitan mutations', asy
|
||||
|
||||
assert.deepEqual(calls, ['build', 'wait', 'info', 'import', 'settings']);
|
||||
});
|
||||
|
||||
test('auto sync scales the import timeout with the merged dictionary size', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
|
||||
fs.mkdirSync(dictionariesDir, { recursive: true });
|
||||
const zipPath = path.join(dictionariesDir, 'merged.zip');
|
||||
// 2 MB of merged dictionary buys ~12s of import budget on top of the base.
|
||||
fs.writeFileSync(zipPath, Buffer.alloc(2 * 1024 * 1024));
|
||||
const events: Array<{ phase: string; message: string }> = [];
|
||||
let importedRevision: string | null = null;
|
||||
|
||||
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
|
||||
userDataPath,
|
||||
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
|
||||
getOrCreateCurrentSnapshot: async () => ({
|
||||
mediaId: 21,
|
||||
mediaTitle: 'ONE PIECE',
|
||||
entryCount: 4000,
|
||||
fromCache: false,
|
||||
updatedAt: 1000,
|
||||
}),
|
||||
buildMergedDictionary: async () => ({
|
||||
zipPath,
|
||||
revision: 'rev-21',
|
||||
dictionaryTitle: 'SubMiner Character Dictionary',
|
||||
entryCount: 4000,
|
||||
}),
|
||||
getYomitanDictionaryInfo: async () =>
|
||||
importedRevision
|
||||
? [{ title: 'SubMiner Character Dictionary', revision: importedRevision }]
|
||||
: [],
|
||||
importYomitanDictionary: async () => {
|
||||
// Far longer than the quick-operation budget, well inside the size-scaled one.
|
||||
await new Promise((resolve) => setTimeout(resolve, 60));
|
||||
importedRevision = 'rev-21';
|
||||
return true;
|
||||
},
|
||||
deleteYomitanDictionary: async () => true,
|
||||
upsertYomitanDictionarySettings: async () => true,
|
||||
now: () => 1000,
|
||||
operationTimeoutMs: 5,
|
||||
dictionaryImportTimeoutBaseMs: 20,
|
||||
onSyncStatus: (event) => {
|
||||
events.push({ phase: event.phase, message: event.message });
|
||||
},
|
||||
});
|
||||
|
||||
await runtime.runSyncNow();
|
||||
|
||||
assert.equal(
|
||||
events.some((event) => event.phase === 'failed'),
|
||||
false,
|
||||
);
|
||||
assert.deepEqual(events.at(-1), {
|
||||
phase: 'ready',
|
||||
message: 'Character dictionary ready for ONE PIECE',
|
||||
});
|
||||
});
|
||||
|
||||
test('auto sync reports the scaled budget when an import really does hang', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const events: Array<{ phase: string; message: string }> = [];
|
||||
|
||||
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
|
||||
userDataPath,
|
||||
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
|
||||
getOrCreateCurrentSnapshot: async () => ({
|
||||
mediaId: 21,
|
||||
mediaTitle: 'ONE PIECE',
|
||||
entryCount: 4000,
|
||||
fromCache: false,
|
||||
updatedAt: 1000,
|
||||
}),
|
||||
buildMergedDictionary: async () => ({
|
||||
zipPath: path.join(userDataPath, 'character-dictionaries', 'missing.zip'),
|
||||
revision: 'rev-21',
|
||||
dictionaryTitle: 'SubMiner Character Dictionary',
|
||||
entryCount: 4000,
|
||||
}),
|
||||
getYomitanDictionaryInfo: async () => [],
|
||||
importYomitanDictionary: () => new Promise<boolean>(() => {}),
|
||||
deleteYomitanDictionary: async () => true,
|
||||
upsertYomitanDictionarySettings: async () => true,
|
||||
now: () => 1000,
|
||||
dictionaryImportTimeoutBaseMs: 20,
|
||||
onSyncStatus: (event) => {
|
||||
events.push({ phase: event.phase, message: event.message });
|
||||
},
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
runtime.runSyncNow(),
|
||||
/importYomitanDictionary\(missing\.zip\) timed out after 20ms/,
|
||||
);
|
||||
assert.equal(events.at(-1)?.phase, 'failed');
|
||||
});
|
||||
|
||||
test('auto sync ticks the importing notification while the import runs', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const events: Array<{ phase: string; message: string }> = [];
|
||||
const scheduled: Array<() => void> = [];
|
||||
const importDeferred = createDeferred<boolean>();
|
||||
let clock = 1000;
|
||||
|
||||
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
|
||||
userDataPath,
|
||||
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
|
||||
getOrCreateCurrentSnapshot: async () => ({
|
||||
mediaId: 21,
|
||||
mediaTitle: 'ONE PIECE',
|
||||
entryCount: 4000,
|
||||
fromCache: false,
|
||||
updatedAt: 1000,
|
||||
}),
|
||||
buildMergedDictionary: async () => ({
|
||||
zipPath: '/tmp/merged.zip',
|
||||
revision: 'rev-21',
|
||||
dictionaryTitle: 'SubMiner Character Dictionary',
|
||||
entryCount: 4000,
|
||||
}),
|
||||
getYomitanDictionaryInfo: async () => [],
|
||||
importYomitanDictionary: () => importDeferred.promise,
|
||||
deleteYomitanDictionary: async () => true,
|
||||
upsertYomitanDictionarySettings: async () => true,
|
||||
now: () => clock,
|
||||
schedule: (fn) => {
|
||||
scheduled.push(fn);
|
||||
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||
},
|
||||
clearSchedule: () => undefined,
|
||||
onSyncStatus: (event) => {
|
||||
events.push({ phase: event.phase, message: event.message });
|
||||
},
|
||||
});
|
||||
|
||||
const syncPromise = runtime.runSyncNow();
|
||||
await waitUntil(
|
||||
() => events.some((event) => event.phase === 'importing'),
|
||||
'the importing status event',
|
||||
);
|
||||
|
||||
clock = 1000 + 65_000;
|
||||
// The importing heartbeat is the most recently scheduled tick.
|
||||
scheduled.at(-1)!();
|
||||
|
||||
assert.deepEqual(events.at(-1), {
|
||||
phase: 'importing',
|
||||
message: 'Importing character dictionary for ONE PIECE (1m 05s)...',
|
||||
});
|
||||
|
||||
importDeferred.resolve(true);
|
||||
await syncPromise;
|
||||
assert.equal(events.at(-1)?.phase, 'ready');
|
||||
});
|
||||
|
||||
test('auto sync reports character and image counts while generating a snapshot', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const events: Array<{ phase: string; message: string }> = [];
|
||||
let clock = 1000;
|
||||
let importedRevision: string | null = null;
|
||||
|
||||
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
|
||||
userDataPath,
|
||||
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
|
||||
getOrCreateCurrentSnapshot: async (_targetPath, progress) => {
|
||||
progress?.onGenerating?.({ mediaId: 21, mediaTitle: 'ONE PIECE' });
|
||||
progress?.onGenerateProgress?.({
|
||||
mediaId: 21,
|
||||
mediaTitle: 'ONE PIECE',
|
||||
stage: 'characters',
|
||||
completed: 50,
|
||||
total: null,
|
||||
page: 12,
|
||||
});
|
||||
// Same stage, same clock tick: throttled away so a 33-page fetch cannot spam the overlay.
|
||||
progress?.onGenerateProgress?.({
|
||||
mediaId: 21,
|
||||
mediaTitle: 'ONE PIECE',
|
||||
stage: 'characters',
|
||||
completed: 100,
|
||||
total: null,
|
||||
page: 13,
|
||||
});
|
||||
// A stage change always reports, throttle window or not.
|
||||
progress?.onGenerateProgress?.({
|
||||
mediaId: 21,
|
||||
mediaTitle: 'ONE PIECE',
|
||||
stage: 'images',
|
||||
completed: 1,
|
||||
total: 1220,
|
||||
});
|
||||
clock += 2000;
|
||||
progress?.onGenerateProgress?.({
|
||||
mediaId: 21,
|
||||
mediaTitle: 'ONE PIECE',
|
||||
stage: 'images',
|
||||
completed: 240,
|
||||
total: 1220,
|
||||
});
|
||||
clock += 6000;
|
||||
progress?.onGenerateProgress?.({
|
||||
mediaId: 21,
|
||||
mediaTitle: 'ONE PIECE',
|
||||
stage: 'names',
|
||||
completed: 800,
|
||||
total: 1220,
|
||||
});
|
||||
progress?.onGenerateProgress?.({
|
||||
mediaId: 21,
|
||||
mediaTitle: 'ONE PIECE',
|
||||
stage: 'saving',
|
||||
completed: 0,
|
||||
total: null,
|
||||
});
|
||||
return {
|
||||
mediaId: 21,
|
||||
mediaTitle: 'ONE PIECE',
|
||||
entryCount: 4000,
|
||||
fromCache: false,
|
||||
updatedAt: 1000,
|
||||
};
|
||||
},
|
||||
buildMergedDictionary: async () => ({
|
||||
zipPath: '/tmp/merged.zip',
|
||||
revision: 'rev-21',
|
||||
dictionaryTitle: 'SubMiner Character Dictionary',
|
||||
entryCount: 4000,
|
||||
}),
|
||||
getYomitanDictionaryInfo: async () =>
|
||||
importedRevision
|
||||
? [{ title: 'SubMiner Character Dictionary', revision: importedRevision }]
|
||||
: [],
|
||||
importYomitanDictionary: async () => {
|
||||
importedRevision = 'rev-21';
|
||||
return true;
|
||||
},
|
||||
deleteYomitanDictionary: async () => true,
|
||||
upsertYomitanDictionarySettings: async () => true,
|
||||
now: () => clock,
|
||||
onSyncStatus: (event) => {
|
||||
events.push({ phase: event.phase, message: event.message });
|
||||
},
|
||||
});
|
||||
|
||||
await runtime.runSyncNow();
|
||||
|
||||
assert.deepEqual(
|
||||
events.filter((event) => event.phase === 'generating').map((event) => event.message),
|
||||
[
|
||||
'Generating character dictionary for ONE PIECE...',
|
||||
'Generating character dictionary for ONE PIECE (page 12, 50 characters)...',
|
||||
'Generating character dictionary for ONE PIECE (image 1/1220)...',
|
||||
'Generating character dictionary for ONE PIECE (image 240/1220, ~10s left)...',
|
||||
'Generating character dictionary for ONE PIECE (name 800/1220 · 8s)...',
|
||||
'Generating character dictionary for ONE PIECE (saving snapshot · 8s)...',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('auto sync keeps the generating clock ticking when a stage stalls', async () => {
|
||||
const userDataPath = makeTempDir();
|
||||
const events: Array<{ phase: string; message: string }> = [];
|
||||
const scheduled: Array<() => void> = [];
|
||||
const snapshotDeferred = createDeferred<{
|
||||
mediaId: number;
|
||||
mediaTitle: string;
|
||||
entryCount: number;
|
||||
fromCache: boolean;
|
||||
updatedAt: number;
|
||||
}>();
|
||||
let clock = 1000;
|
||||
|
||||
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
|
||||
userDataPath,
|
||||
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
|
||||
getOrCreateCurrentSnapshot: async (_targetPath, progress) => {
|
||||
progress?.onGenerating?.({ mediaId: 21, mediaTitle: 'ONE PIECE' });
|
||||
progress?.onGenerateProgress?.({
|
||||
mediaId: 21,
|
||||
mediaTitle: 'ONE PIECE',
|
||||
stage: 'images',
|
||||
completed: 240,
|
||||
total: 1220,
|
||||
});
|
||||
return await snapshotDeferred.promise;
|
||||
},
|
||||
buildMergedDictionary: async () => ({
|
||||
zipPath: '/tmp/merged.zip',
|
||||
revision: 'rev-21',
|
||||
dictionaryTitle: 'SubMiner Character Dictionary',
|
||||
entryCount: 4000,
|
||||
}),
|
||||
getYomitanDictionaryInfo: async () => [],
|
||||
importYomitanDictionary: async () => true,
|
||||
deleteYomitanDictionary: async () => true,
|
||||
upsertYomitanDictionarySettings: async () => true,
|
||||
now: () => clock,
|
||||
schedule: (fn) => {
|
||||
scheduled.push(fn);
|
||||
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||
},
|
||||
clearSchedule: () => undefined,
|
||||
onSyncStatus: (event) => {
|
||||
events.push({ phase: event.phase, message: event.message });
|
||||
},
|
||||
});
|
||||
|
||||
const syncPromise = runtime.runSyncNow();
|
||||
await waitUntil(() => scheduled.length > 0, 'the generating heartbeat');
|
||||
|
||||
// No further progress arrives: only the clock moves.
|
||||
clock += 95_000;
|
||||
scheduled.at(-1)!();
|
||||
|
||||
assert.deepEqual(events.at(-1), {
|
||||
phase: 'generating',
|
||||
message: 'Generating character dictionary for ONE PIECE (image 240/1220 · 1m 35s)...',
|
||||
});
|
||||
|
||||
snapshotDeferred.resolve({
|
||||
mediaId: 21,
|
||||
mediaTitle: 'ONE PIECE',
|
||||
entryCount: 4000,
|
||||
fromCache: false,
|
||||
updatedAt: 1000,
|
||||
});
|
||||
await syncPromise;
|
||||
assert.equal(events.at(-1)?.phase, 'ready');
|
||||
});
|
||||
|
||||
@@ -5,9 +5,14 @@ import type { AnilistCharacterDictionaryProfileScope } from '../../types';
|
||||
import type {
|
||||
CharacterDictionarySnapshotProgressCallbacks,
|
||||
CharacterDictionarySnapshotResult,
|
||||
CharacterDictionarySnapshotStageProgress,
|
||||
MergedCharacterDictionaryBuildResult,
|
||||
} from '../character-dictionary-runtime';
|
||||
|
||||
const DEFAULT_IMPORT_TIMEOUT_BASE_MS = 120_000;
|
||||
const IMPORT_TIMEOUT_MS_PER_MB = 6_000;
|
||||
const IMPORT_TIMEOUT_MAX_MS = 1_800_000;
|
||||
|
||||
type AutoSyncMediaEntry = {
|
||||
mediaId: number;
|
||||
label: string;
|
||||
@@ -72,7 +77,15 @@ export interface CharacterDictionaryAutoSyncRuntimeDeps {
|
||||
now: () => number;
|
||||
schedule?: (fn: () => void, delayMs: number) => ReturnType<typeof setTimeout>;
|
||||
clearSchedule?: (timer: ReturnType<typeof setTimeout>) => void;
|
||||
/** Budget for the quick Yomitan queries (dictionary info, settings upsert). */
|
||||
operationTimeoutMs?: number;
|
||||
/**
|
||||
* Base budget for the slow Yomitan mutations (delete + import). The effective budget grows with
|
||||
* the merged ZIP size, because importing a large dictionary can take several minutes.
|
||||
*/
|
||||
dictionaryImportTimeoutBaseMs?: number;
|
||||
heartbeatMs?: number;
|
||||
progressThrottleMs?: number;
|
||||
logInfo?: (message: string) => void;
|
||||
logWarn?: (message: string) => void;
|
||||
onSyncStatus?: (event: CharacterDictionaryAutoSyncStatusEvent) => void;
|
||||
@@ -353,12 +366,70 @@ function buildCheckingMessage(mediaTitle: string): string {
|
||||
return `Checking character dictionary for ${mediaTitle}...`;
|
||||
}
|
||||
|
||||
function buildGeneratingMessage(mediaTitle: string): string {
|
||||
return `Generating character dictionary for ${mediaTitle}...`;
|
||||
function buildGeneratingMessage(mediaTitle: string, detail?: string): string {
|
||||
return detail
|
||||
? `Generating character dictionary for ${mediaTitle} (${detail})...`
|
||||
: `Generating character dictionary for ${mediaTitle}...`;
|
||||
}
|
||||
|
||||
function buildImportingMessage(mediaTitle: string): string {
|
||||
return `Importing character dictionary for ${mediaTitle}...`;
|
||||
function formatCharacterDictionaryProgressDetail(
|
||||
progress: CharacterDictionarySnapshotStageProgress,
|
||||
remainingMs: number | null,
|
||||
): string {
|
||||
if (progress.stage === 'saving') {
|
||||
return 'saving snapshot';
|
||||
}
|
||||
if (progress.stage === 'names') {
|
||||
return progress.total !== null && progress.total > 0
|
||||
? `name ${progress.completed}/${progress.total}`
|
||||
: `${progress.completed} names`;
|
||||
}
|
||||
if (progress.stage === 'images') {
|
||||
const counted =
|
||||
progress.total !== null && progress.total > 0
|
||||
? `image ${progress.completed}/${progress.total}`
|
||||
: `${progress.completed} images`;
|
||||
return remainingMs !== null
|
||||
? `${counted}, ~${formatRemainingDuration(remainingMs)} left`
|
||||
: counted;
|
||||
}
|
||||
const page = typeof progress.page === 'number' ? `page ${progress.page}, ` : '';
|
||||
return `${page}${progress.completed} characters`;
|
||||
}
|
||||
|
||||
function formatElapsedDuration(elapsedMs: number): string {
|
||||
const totalSeconds = Math.max(0, Math.floor(elapsedMs / 1000));
|
||||
const minutes = Math.floor(totalSeconds / 60);
|
||||
const seconds = totalSeconds % 60;
|
||||
return minutes > 0 ? `${minutes}m ${String(seconds).padStart(2, '0')}s` : `${seconds}s`;
|
||||
}
|
||||
|
||||
/** Coarser than the elapsed clock: an estimate that ticks every second reads as precision it lacks. */
|
||||
function formatRemainingDuration(remainingMs: number): string {
|
||||
const totalSeconds = Math.max(1, Math.round(remainingMs / 1000));
|
||||
if (totalSeconds >= 60) {
|
||||
return `${Math.max(1, Math.round(totalSeconds / 60))}m`;
|
||||
}
|
||||
return `${Math.max(5, Math.ceil(totalSeconds / 5) * 5)}s`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The elapsed clock is the part that proves the app is alive: a stalled network fetch freezes the
|
||||
* counts, but the clock keeps moving.
|
||||
*/
|
||||
function joinGeneratingDetail(detail: string | null, elapsedMs: number): string | undefined {
|
||||
const parts = [detail, elapsedMs >= 5_000 ? formatElapsedDuration(elapsedMs) : null].filter(
|
||||
(part): part is string => typeof part === 'string' && part.length > 0,
|
||||
);
|
||||
return parts.length > 0 ? parts.join(' · ') : undefined;
|
||||
}
|
||||
|
||||
function buildImportingMessage(mediaTitle: string, elapsedMs?: number): string {
|
||||
const elapsed =
|
||||
typeof elapsedMs === 'number' && elapsedMs >= 1000
|
||||
? ` (${formatElapsedDuration(elapsedMs)})`
|
||||
: '';
|
||||
return `Importing character dictionary for ${mediaTitle}${elapsed}...`;
|
||||
}
|
||||
|
||||
function buildBuildingMessage(mediaTitle: string): string {
|
||||
@@ -389,21 +460,31 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
|
||||
const clearSchedule = deps.clearSchedule ?? ((timer) => clearTimeout(timer));
|
||||
const debounceMs = 800;
|
||||
const operationTimeoutMs = Math.max(1, Math.floor(deps.operationTimeoutMs ?? 7_000));
|
||||
const dictionaryImportTimeoutBaseMs = Math.max(
|
||||
1,
|
||||
Math.floor(deps.dictionaryImportTimeoutBaseMs ?? DEFAULT_IMPORT_TIMEOUT_BASE_MS),
|
||||
);
|
||||
const heartbeatMs = Math.max(1, Math.floor(deps.heartbeatMs ?? 5_000));
|
||||
const progressThrottleMs = Math.max(0, Math.floor(deps.progressThrottleMs ?? 1_000));
|
||||
|
||||
let debounceTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let syncInFlight = false;
|
||||
let runQueued = false;
|
||||
let activeCurrentMediaId: number | null = null;
|
||||
|
||||
const withOperationTimeout = async <T>(label: string, promise: Promise<T>): Promise<T> => {
|
||||
const withTimeout = async <T>(
|
||||
label: string,
|
||||
promise: Promise<T>,
|
||||
timeoutMs: number,
|
||||
): Promise<T> => {
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
try {
|
||||
return await Promise.race([
|
||||
promise,
|
||||
new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
reject(new Error(`${label} timed out after ${operationTimeoutMs}ms`));
|
||||
}, operationTimeoutMs);
|
||||
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
@@ -413,6 +494,57 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
|
||||
}
|
||||
};
|
||||
|
||||
const withOperationTimeout = <T>(label: string, promise: Promise<T>): Promise<T> =>
|
||||
withTimeout(label, promise, operationTimeoutMs);
|
||||
|
||||
/**
|
||||
* Importing a merged dictionary means Yomitan writing every term and image into IndexedDB, which
|
||||
* scales with the ZIP: a single-season dictionary lands in seconds, One Piece takes minutes. Size
|
||||
* the budget off the archive instead of failing a healthy import on a flat deadline.
|
||||
*/
|
||||
const resolveImportTimeoutMs = (zipPath: string | null): number => {
|
||||
let bytes = 0;
|
||||
if (zipPath) {
|
||||
try {
|
||||
bytes = fs.statSync(zipPath).size;
|
||||
} catch {
|
||||
bytes = 0;
|
||||
}
|
||||
}
|
||||
const sizeAllowanceMs = (bytes / (1024 * 1024)) * IMPORT_TIMEOUT_MS_PER_MB;
|
||||
return Math.min(
|
||||
IMPORT_TIMEOUT_MAX_MS,
|
||||
Math.max(
|
||||
dictionaryImportTimeoutBaseMs,
|
||||
Math.round(dictionaryImportTimeoutBaseMs + sizeAllowanceMs),
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
/** Keeps the persistent notification ticking so a multi-minute import never looks hung. */
|
||||
const withHeartbeat = async <T>(
|
||||
run: () => Promise<T>,
|
||||
onTick: (elapsedMs: number) => void,
|
||||
): Promise<T> => {
|
||||
const startedAt = deps.now();
|
||||
let stopped = false;
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
const tick = (): void => {
|
||||
if (stopped) return;
|
||||
onTick(deps.now() - startedAt);
|
||||
timer = schedule(tick, heartbeatMs);
|
||||
};
|
||||
timer = schedule(tick, heartbeatMs);
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
stopped = true;
|
||||
if (timer !== null) {
|
||||
clearSchedule(timer);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const runSyncOnce = async (): Promise<void> => {
|
||||
const config = deps.getConfig();
|
||||
if (!config.enabled) {
|
||||
@@ -422,33 +554,112 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
|
||||
|
||||
let currentMediaId: number | undefined;
|
||||
let currentMediaTitle: string | null = null;
|
||||
let lastProgressAt = Number.NEGATIVE_INFINITY;
|
||||
let lastProgressStage: CharacterDictionarySnapshotStageProgress['stage'] | null = null;
|
||||
let generating: {
|
||||
mediaId: number;
|
||||
mediaTitle: string;
|
||||
startedAt: number;
|
||||
detail: string | null;
|
||||
} | null = null;
|
||||
let imageRate: { startedAt: number; startCompleted: number } | null = null;
|
||||
|
||||
const emitGeneratingStatus = (): void => {
|
||||
if (!generating) {
|
||||
return;
|
||||
}
|
||||
deps.onSyncStatus?.({
|
||||
phase: 'generating',
|
||||
mediaId: generating.mediaId,
|
||||
mediaTitle: generating.mediaTitle,
|
||||
message: buildGeneratingMessage(
|
||||
generating.mediaTitle,
|
||||
joinGeneratingDetail(generating.detail, deps.now() - generating.startedAt),
|
||||
),
|
||||
});
|
||||
};
|
||||
|
||||
// Image downloads are serial and evenly paced, so the observed rate predicts the tail well.
|
||||
const estimateRemainingImageMs = (
|
||||
progress: CharacterDictionarySnapshotStageProgress,
|
||||
nowMs: number,
|
||||
): number | null => {
|
||||
if (progress.stage !== 'images' || progress.total === null || imageRate === null) {
|
||||
return null;
|
||||
}
|
||||
const done = progress.completed - imageRate.startCompleted;
|
||||
const elapsedMs = nowMs - imageRate.startedAt;
|
||||
const remaining = progress.total - progress.completed;
|
||||
if (done <= 0 || elapsedMs <= 0 || remaining <= 0) {
|
||||
return null;
|
||||
}
|
||||
return Math.round((elapsedMs / done) * remaining);
|
||||
};
|
||||
|
||||
try {
|
||||
deps.logInfo?.('[dictionary:auto-sync] syncing current anime snapshot');
|
||||
const snapshot = await deps.getOrCreateCurrentSnapshot(undefined, {
|
||||
onChecking: ({ mediaId, mediaTitle }) => {
|
||||
currentMediaId = mediaId;
|
||||
currentMediaTitle = mediaTitle;
|
||||
activeCurrentMediaId = mediaId;
|
||||
deps.onSyncStatus?.({
|
||||
phase: 'checking',
|
||||
mediaId,
|
||||
mediaTitle,
|
||||
message: buildCheckingMessage(mediaTitle),
|
||||
});
|
||||
},
|
||||
onGenerating: ({ mediaId, mediaTitle }) => {
|
||||
currentMediaId = mediaId;
|
||||
currentMediaTitle = mediaTitle;
|
||||
activeCurrentMediaId = mediaId;
|
||||
deps.onSyncStatus?.({
|
||||
phase: 'generating',
|
||||
mediaId,
|
||||
mediaTitle,
|
||||
message: buildGeneratingMessage(mediaTitle),
|
||||
});
|
||||
},
|
||||
});
|
||||
const snapshot = await withHeartbeat(
|
||||
() =>
|
||||
deps.getOrCreateCurrentSnapshot(undefined, {
|
||||
onChecking: ({ mediaId, mediaTitle }) => {
|
||||
currentMediaId = mediaId;
|
||||
currentMediaTitle = mediaTitle;
|
||||
activeCurrentMediaId = mediaId;
|
||||
deps.onSyncStatus?.({
|
||||
phase: 'checking',
|
||||
mediaId,
|
||||
mediaTitle,
|
||||
message: buildCheckingMessage(mediaTitle),
|
||||
});
|
||||
},
|
||||
onGenerating: ({ mediaId, mediaTitle }) => {
|
||||
currentMediaId = mediaId;
|
||||
currentMediaTitle = mediaTitle;
|
||||
activeCurrentMediaId = mediaId;
|
||||
lastProgressAt = Number.NEGATIVE_INFINITY;
|
||||
lastProgressStage = null;
|
||||
imageRate = null;
|
||||
generating = { mediaId, mediaTitle, startedAt: deps.now(), detail: null };
|
||||
emitGeneratingStatus();
|
||||
},
|
||||
// Long-running work (AniList character pages, then one image download per character
|
||||
// and voice actor, then MeCab name splits) reports counts so the notification shows
|
||||
// movement instead of a frozen "Generating..." for the minutes a large series takes.
|
||||
onGenerateProgress: (progress) => {
|
||||
const nowMs = deps.now();
|
||||
if (!generating) {
|
||||
generating = {
|
||||
mediaId: progress.mediaId,
|
||||
mediaTitle: progress.mediaTitle,
|
||||
startedAt: nowMs,
|
||||
detail: null,
|
||||
};
|
||||
}
|
||||
if (progress.stage === 'images' && imageRate === null) {
|
||||
imageRate = { startedAt: nowMs, startCompleted: progress.completed };
|
||||
}
|
||||
// Stage changes and the last item of a stage always report; the throttle only thins
|
||||
// out the run of identical-looking updates in between.
|
||||
const isFinal = progress.total !== null && progress.completed >= progress.total;
|
||||
const isStageChange = progress.stage !== lastProgressStage;
|
||||
if (!isFinal && !isStageChange && nowMs - lastProgressAt < progressThrottleMs) {
|
||||
return;
|
||||
}
|
||||
lastProgressAt = nowMs;
|
||||
lastProgressStage = progress.stage;
|
||||
generating.mediaId = progress.mediaId;
|
||||
generating.mediaTitle = progress.mediaTitle;
|
||||
generating.detail = formatCharacterDictionaryProgressDetail(
|
||||
progress,
|
||||
estimateRemainingImageMs(progress, nowMs),
|
||||
);
|
||||
emitGeneratingStatus();
|
||||
},
|
||||
}),
|
||||
// Ticks even when a step stalls, so the message keeps moving while the counts do not.
|
||||
() => emitGeneratingStatus(),
|
||||
);
|
||||
generating = null;
|
||||
currentMediaId = snapshot.mediaId;
|
||||
currentMediaTitle = snapshot.mediaTitle;
|
||||
activeCurrentMediaId = snapshot.mediaId;
|
||||
@@ -531,33 +742,54 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
|
||||
mediaTitle: snapshot.mediaTitle,
|
||||
message: buildImportingMessage(snapshot.mediaTitle),
|
||||
});
|
||||
if (existing !== null) {
|
||||
await withOperationTimeout(
|
||||
`deleteYomitanDictionary(${dictionaryTitle})`,
|
||||
deps.deleteYomitanDictionary(dictionaryTitle),
|
||||
);
|
||||
}
|
||||
if (merged === null) {
|
||||
const existingMergedZipPath = path.join(dictionariesDir, 'merged.zip');
|
||||
if (fs.existsSync(existingMergedZipPath)) {
|
||||
merged = {
|
||||
zipPath: existingMergedZipPath,
|
||||
revision,
|
||||
dictionaryTitle,
|
||||
entryCount: snapshot.entryCount,
|
||||
};
|
||||
} else {
|
||||
merged = await deps.buildMergedDictionary(nextActiveMediaIdValues);
|
||||
}
|
||||
}
|
||||
deps.logInfo?.(`[dictionary:auto-sync] importing merged dictionary: ${merged.zipPath}`);
|
||||
const imported = await withOperationTimeout(
|
||||
`importYomitanDictionary(${path.basename(merged.zipPath)})`,
|
||||
deps.importYomitanDictionary(merged.zipPath),
|
||||
await withHeartbeat(
|
||||
async () => {
|
||||
const importTimeoutMs = resolveImportTimeoutMs(
|
||||
merged?.zipPath ?? path.join(dictionariesDir, 'merged.zip'),
|
||||
);
|
||||
if (existing !== null) {
|
||||
await withTimeout(
|
||||
`deleteYomitanDictionary(${dictionaryTitle})`,
|
||||
deps.deleteYomitanDictionary(dictionaryTitle),
|
||||
importTimeoutMs,
|
||||
);
|
||||
}
|
||||
if (merged === null) {
|
||||
const existingMergedZipPath = path.join(dictionariesDir, 'merged.zip');
|
||||
if (fs.existsSync(existingMergedZipPath)) {
|
||||
merged = {
|
||||
zipPath: existingMergedZipPath,
|
||||
revision,
|
||||
dictionaryTitle,
|
||||
entryCount: snapshot.entryCount,
|
||||
};
|
||||
} else {
|
||||
merged = await deps.buildMergedDictionary(nextActiveMediaIdValues);
|
||||
}
|
||||
}
|
||||
const mergedZipPath = merged.zipPath;
|
||||
const mergedImportTimeoutMs = resolveImportTimeoutMs(mergedZipPath);
|
||||
deps.logInfo?.(
|
||||
`[dictionary:auto-sync] importing merged dictionary: ${mergedZipPath} (timeout ${mergedImportTimeoutMs}ms)`,
|
||||
);
|
||||
const imported = await withTimeout(
|
||||
`importYomitanDictionary(${path.basename(mergedZipPath)})`,
|
||||
deps.importYomitanDictionary(mergedZipPath),
|
||||
mergedImportTimeoutMs,
|
||||
);
|
||||
if (!imported) {
|
||||
throw new Error(`Failed to import dictionary ZIP: ${merged.zipPath}`);
|
||||
}
|
||||
},
|
||||
(elapsedMs) => {
|
||||
deps.onSyncStatus?.({
|
||||
phase: 'importing',
|
||||
mediaId: snapshot.mediaId,
|
||||
mediaTitle: snapshot.mediaTitle,
|
||||
message: buildImportingMessage(snapshot.mediaTitle, elapsedMs),
|
||||
});
|
||||
},
|
||||
);
|
||||
if (!imported) {
|
||||
throw new Error(`Failed to import dictionary ZIP: ${merged.zipPath}`);
|
||||
}
|
||||
changed = true;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user