mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-11 07:21:34 -07:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
155c27aac9
|
|||
|
0e8599dd1d
|
|||
|
8a56b11d0c
|
|||
|
11b6f84d41
|
|||
|
da2c50b89f
|
@@ -0,0 +1,7 @@
|
|||||||
|
type: fixed
|
||||||
|
area: character dictionary
|
||||||
|
|
||||||
|
- A large character dictionary no longer fails to install. The Yomitan import (and the delete that precedes it) used a flat 7 second budget, so a series like One Piece failed with `importYomitanDictionary(merged.zip) timed out after 7000ms` while the import was still healthy. The budget is now sized from the merged ZIP (2 minutes plus 6 seconds per MB, capped at 30 minutes); the quick queries keep the 7 second budget.
|
||||||
|
- The "Generating character dictionary" notification now reports what it is doing instead of one static message for the whole run: the AniList page and character count while characters download (`page 12, 587 characters`), then `image 240/1220, ~4m left` per downloaded character/voice actor image, then `name 800/1220` for MeCab name splits and `saving snapshot` at the end. Updates are throttled to one per second, and every stage change and final item always reports.
|
||||||
|
- Every long phase also carries an elapsed clock (`· 1m 35s`), refreshed every 5 seconds even when nothing else moves, so a stalled step is visibly different from a frozen app. Long imports tick the same way.
|
||||||
|
- Image downloads are also logged every 100 files, and the import logs the timeout it computed.
|
||||||
@@ -52,9 +52,15 @@ function resolveRuntimeDefaultNotificationIconPath(): string | null {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Live notifications keyed by `replaceId`. Electron exposes no native "replace this notification"
|
||||||
|
* flag, so a repeated status closes its predecessor instead of stacking a fresh toast per update.
|
||||||
|
*/
|
||||||
|
const notificationsByReplaceId = new Map<string, Electron.Notification>();
|
||||||
|
|
||||||
export function showDesktopNotification(
|
export function showDesktopNotification(
|
||||||
title: string,
|
title: string,
|
||||||
options: { body?: string; icon?: string },
|
options: { body?: string; icon?: string; replaceId?: string },
|
||||||
): void {
|
): void {
|
||||||
const notificationOptions: {
|
const notificationOptions: {
|
||||||
title: string;
|
title: string;
|
||||||
@@ -98,5 +104,15 @@ export function showDesktopNotification(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const notification = new Notification(notificationOptions);
|
const notification = new Notification(notificationOptions);
|
||||||
|
const replaceId = options.replaceId?.trim();
|
||||||
|
if (replaceId) {
|
||||||
|
notificationsByReplaceId.get(replaceId)?.close();
|
||||||
|
notificationsByReplaceId.set(replaceId, notification);
|
||||||
|
notification.once('close', () => {
|
||||||
|
if (notificationsByReplaceId.get(replaceId) === notification) {
|
||||||
|
notificationsByReplaceId.delete(replaceId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
notification.show();
|
notification.show();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1903,7 +1903,7 @@ test('generateForCurrentMedia logs progress while resolving and rebuilding snaps
|
|||||||
'[dictionary] current anime guess: The Eminence in Shadow (episode 5)',
|
'[dictionary] current anime guess: The Eminence in Shadow (episode 5)',
|
||||||
'[dictionary] AniList match: The Eminence in Shadow -> AniList 130298',
|
'[dictionary] AniList match: The Eminence in Shadow -> AniList 130298',
|
||||||
'[dictionary] snapshot miss for AniList 130298, fetching characters',
|
'[dictionary] snapshot miss for AniList 130298, fetching characters',
|
||||||
'[dictionary] downloaded AniList character page 1 for AniList 130298',
|
'[dictionary] downloaded AniList character page 1 for AniList 130298 (1 characters)',
|
||||||
'[dictionary] downloading 1 images for AniList 130298',
|
'[dictionary] downloading 1 images for AniList 130298',
|
||||||
'[dictionary] stored snapshot for AniList 130298: 16 terms',
|
'[dictionary] stored snapshot for AniList 130298: 16 terms',
|
||||||
'[dictionary] building ZIP for AniList 130298',
|
'[dictionary] building ZIP for AniList 130298',
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ export type {
|
|||||||
CharacterDictionarySnapshotProgress,
|
CharacterDictionarySnapshotProgress,
|
||||||
CharacterDictionarySnapshotProgressCallbacks,
|
CharacterDictionarySnapshotProgressCallbacks,
|
||||||
CharacterDictionarySnapshotResult,
|
CharacterDictionarySnapshotResult,
|
||||||
|
CharacterDictionarySnapshotStageProgress,
|
||||||
MergedCharacterDictionaryBuildResult,
|
MergedCharacterDictionaryBuildResult,
|
||||||
} from './character-dictionary-runtime/types';
|
} from './character-dictionary-runtime/types';
|
||||||
|
|
||||||
@@ -363,19 +364,28 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
deps.logInfo?.(`[dictionary] snapshot stale for AniList ${mediaId}: ${refreshReason}`);
|
deps.logInfo?.(`[dictionary] snapshot stale for AniList ${mediaId}: ${refreshReason}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const progressMediaTitle = mediaTitleHint || `AniList ${mediaId}`;
|
||||||
progress?.onGenerating?.({
|
progress?.onGenerating?.({
|
||||||
mediaId,
|
mediaId,
|
||||||
mediaTitle: mediaTitleHint || `AniList ${mediaId}`,
|
mediaTitle: progressMediaTitle,
|
||||||
});
|
});
|
||||||
deps.logInfo?.(`[dictionary] snapshot miss for AniList ${mediaId}, fetching characters`);
|
deps.logInfo?.(`[dictionary] snapshot miss for AniList ${mediaId}, fetching characters`);
|
||||||
|
|
||||||
const { mediaTitle: fetchedMediaTitle, characters } = await fetchCharactersForMedia(
|
const { mediaTitle: fetchedMediaTitle, characters } = await fetchCharactersForMedia(
|
||||||
mediaId,
|
mediaId,
|
||||||
beforeRequest,
|
beforeRequest,
|
||||||
(page) => {
|
(page, charactersSoFar) => {
|
||||||
deps.logInfo?.(
|
deps.logInfo?.(
|
||||||
`[dictionary] downloaded AniList character page ${page} for AniList ${mediaId}`,
|
`[dictionary] downloaded AniList character page ${page} for AniList ${mediaId} (${charactersSoFar} characters)`,
|
||||||
);
|
);
|
||||||
|
progress?.onGenerateProgress?.({
|
||||||
|
mediaId,
|
||||||
|
mediaTitle: progressMediaTitle,
|
||||||
|
stage: 'characters',
|
||||||
|
completed: charactersSoFar,
|
||||||
|
total: null,
|
||||||
|
page,
|
||||||
|
});
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
if (characters.length === 0) {
|
if (characters.length === 0) {
|
||||||
@@ -403,12 +413,26 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
let hasAttemptedImageDownload = false;
|
let hasAttemptedImageDownload = false;
|
||||||
|
let attemptedImageCount = 0;
|
||||||
for (const entry of allImageUrls) {
|
for (const entry of allImageUrls) {
|
||||||
if (hasAttemptedImageDownload) {
|
if (hasAttemptedImageDownload) {
|
||||||
await sleepMs(CHARACTER_IMAGE_DOWNLOAD_DELAY_MS);
|
await sleepMs(CHARACTER_IMAGE_DOWNLOAD_DELAY_MS);
|
||||||
}
|
}
|
||||||
hasAttemptedImageDownload = true;
|
hasAttemptedImageDownload = true;
|
||||||
const image = await downloadCharacterImage(entry.url, entry.id);
|
const image = await downloadCharacterImage(entry.url, entry.id);
|
||||||
|
attemptedImageCount += 1;
|
||||||
|
progress?.onGenerateProgress?.({
|
||||||
|
mediaId,
|
||||||
|
mediaTitle: progressMediaTitle,
|
||||||
|
stage: 'images',
|
||||||
|
completed: attemptedImageCount,
|
||||||
|
total: allImageUrls.length,
|
||||||
|
});
|
||||||
|
if (attemptedImageCount % 100 === 0) {
|
||||||
|
deps.logInfo?.(
|
||||||
|
`[dictionary] downloaded ${attemptedImageCount}/${allImageUrls.length} images for AniList ${mediaId}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
if (!image) continue;
|
if (!image) continue;
|
||||||
if (entry.kind === 'character') {
|
if (entry.kind === 'character') {
|
||||||
imagesByCharacterId.set(entry.id, {
|
imagesByCharacterId.set(entry.id, {
|
||||||
@@ -425,11 +449,31 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
|
|
||||||
const nameSplitTokenizerAvailable = isNameSplitTokenizerAvailable();
|
const nameSplitTokenizerAvailable = isNameSplitTokenizerAvailable();
|
||||||
const resolvedNameSplits = nameSplitTokenizerAvailable
|
const resolvedNameSplits = nameSplitTokenizerAvailable
|
||||||
? await resolveJapaneseNameSplits(characters, deps.tokenizeJapaneseName!, deps.logWarn)
|
? await resolveJapaneseNameSplits(
|
||||||
|
characters,
|
||||||
|
deps.tokenizeJapaneseName!,
|
||||||
|
deps.logWarn,
|
||||||
|
(completed, total) => {
|
||||||
|
progress?.onGenerateProgress?.({
|
||||||
|
mediaId,
|
||||||
|
mediaTitle: progressMediaTitle,
|
||||||
|
stage: 'names',
|
||||||
|
completed,
|
||||||
|
total,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
)
|
||||||
: undefined;
|
: undefined;
|
||||||
const nameSplitSource =
|
const nameSplitSource =
|
||||||
resolvedNameSplits && resolvedNameSplits.size > 0 ? 'mecab' : 'heuristic';
|
resolvedNameSplits && resolvedNameSplits.size > 0 ? 'mecab' : 'heuristic';
|
||||||
|
|
||||||
|
progress?.onGenerateProgress?.({
|
||||||
|
mediaId,
|
||||||
|
mediaTitle: progressMediaTitle,
|
||||||
|
stage: 'saving',
|
||||||
|
completed: 0,
|
||||||
|
total: null,
|
||||||
|
});
|
||||||
const snapshot = buildSnapshotFromCharacters(
|
const snapshot = buildSnapshotFromCharacters(
|
||||||
mediaId,
|
mediaId,
|
||||||
fetchedMediaTitle || mediaTitleHint || `AniList ${mediaId}`,
|
fetchedMediaTitle || mediaTitleHint || `AniList ${mediaId}`,
|
||||||
|
|||||||
@@ -278,7 +278,7 @@ export async function fetchAniListMediaCandidateById(
|
|||||||
export async function fetchCharactersForMedia(
|
export async function fetchCharactersForMedia(
|
||||||
mediaId: number,
|
mediaId: number,
|
||||||
beforeRequest?: () => Promise<void>,
|
beforeRequest?: () => Promise<void>,
|
||||||
onPageFetched?: (page: number) => void,
|
onPageFetched?: (page: number, charactersSoFar: number) => void,
|
||||||
): Promise<{
|
): Promise<{
|
||||||
mediaTitle: string;
|
mediaTitle: string;
|
||||||
characters: CharacterRecord[];
|
characters: CharacterRecord[];
|
||||||
@@ -345,7 +345,6 @@ export async function fetchCharactersForMedia(
|
|||||||
},
|
},
|
||||||
beforeRequest,
|
beforeRequest,
|
||||||
);
|
);
|
||||||
onPageFetched?.(page);
|
|
||||||
|
|
||||||
const media = data.Media;
|
const media = data.Media;
|
||||||
if (!media) {
|
if (!media) {
|
||||||
@@ -415,6 +414,8 @@ export async function fetchCharactersForMedia(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
onPageFetched?.(page, characters.length);
|
||||||
|
|
||||||
const hasNextPage = Boolean(media.characters?.pageInfo?.hasNextPage);
|
const hasNextPage = Boolean(media.characters?.pageInfo?.hasNextPage);
|
||||||
if (!hasNextPage) {
|
if (!hasNextPage) {
|
||||||
break;
|
break;
|
||||||
|
|||||||
@@ -86,8 +86,10 @@ export async function resolveJapaneseNameSplits(
|
|||||||
characters: CharacterRecord[],
|
characters: CharacterRecord[],
|
||||||
tokenize: NameSplitTokenizer,
|
tokenize: NameSplitTokenizer,
|
||||||
logWarn?: (message: string) => void,
|
logWarn?: (message: string) => void,
|
||||||
|
onCharacterResolved?: (completed: number, total: number) => void,
|
||||||
): Promise<Map<string, ResolvedNameSplit>> {
|
): Promise<Map<string, ResolvedNameSplit>> {
|
||||||
const splits = new Map<string, ResolvedNameSplit>();
|
const splits = new Map<string, ResolvedNameSplit>();
|
||||||
|
let resolvedCharacters = 0;
|
||||||
for (const character of characters) {
|
for (const character of characters) {
|
||||||
const familyHintReading = buildReadingFromHint(character.lastNameHint?.trim() || '');
|
const familyHintReading = buildReadingFromHint(character.lastNameHint?.trim() || '');
|
||||||
const givenHintReading = buildReadingFromHint(character.firstNameHint?.trim() || '');
|
const givenHintReading = buildReadingFromHint(character.firstNameHint?.trim() || '');
|
||||||
@@ -113,6 +115,8 @@ export async function resolveJapaneseNameSplits(
|
|||||||
splits.set(name, { family, given });
|
splits.set(name, { family, given });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
resolvedCharacters += 1;
|
||||||
|
onCharacterResolved?.(resolvedCharacters, characters.length);
|
||||||
}
|
}
|
||||||
return splits;
|
return splits;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -122,9 +122,24 @@ export type CharacterDictionarySnapshotProgress = {
|
|||||||
mediaTitle: string;
|
mediaTitle: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type CharacterDictionarySnapshotStage = 'characters' | 'images' | 'names' | 'saving';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fine-grained generation progress. `total` is null while the work size is still unknown (AniList
|
||||||
|
* paginates characters, so the character count only settles on the last page).
|
||||||
|
*/
|
||||||
|
export type CharacterDictionarySnapshotStageProgress = CharacterDictionarySnapshotProgress & {
|
||||||
|
stage: CharacterDictionarySnapshotStage;
|
||||||
|
completed: number;
|
||||||
|
total: number | null;
|
||||||
|
/** AniList page currently being downloaded; only set during the `characters` stage. */
|
||||||
|
page?: number;
|
||||||
|
};
|
||||||
|
|
||||||
export type CharacterDictionarySnapshotProgressCallbacks = {
|
export type CharacterDictionarySnapshotProgressCallbacks = {
|
||||||
onChecking?: (progress: CharacterDictionarySnapshotProgress) => void;
|
onChecking?: (progress: CharacterDictionarySnapshotProgress) => void;
|
||||||
onGenerating?: (progress: CharacterDictionarySnapshotProgress) => void;
|
onGenerating?: (progress: CharacterDictionarySnapshotProgress) => void;
|
||||||
|
onGenerateProgress?: (progress: CharacterDictionarySnapshotStageProgress) => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type MergedCharacterDictionaryBuildResult = {
|
export type MergedCharacterDictionaryBuildResult = {
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import fs from 'node:fs';
|
|||||||
import os from 'node:os';
|
import os from 'node:os';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
import { buildDictionaryZip } from './zip';
|
import { buildDictionaryZip, readDictionaryZipRevision } from './zip';
|
||||||
import type { CharacterDictionaryTermEntry } from './types';
|
import type { CharacterDictionaryTermEntry } from './types';
|
||||||
|
|
||||||
function makeTempDir(): string {
|
function makeTempDir(): string {
|
||||||
@@ -105,3 +105,63 @@ test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', ()
|
|||||||
cleanupDir(tempDir);
|
cleanupDir(tempDir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('readDictionaryZipRevision reads the built revision and rejects foreign archives', () => {
|
||||||
|
const dir = makeTempDir();
|
||||||
|
try {
|
||||||
|
const zipPath = path.join(dir, 'merged.zip');
|
||||||
|
buildDictionaryZip(
|
||||||
|
zipPath,
|
||||||
|
'SubMiner Character Dictionary',
|
||||||
|
'Character names',
|
||||||
|
'rev-42',
|
||||||
|
[
|
||||||
|
{
|
||||||
|
term: 'ルフィ',
|
||||||
|
reading: 'ルフィ',
|
||||||
|
role: 'main',
|
||||||
|
glossary: [],
|
||||||
|
} as unknown as CharacterDictionaryTermEntry,
|
||||||
|
],
|
||||||
|
[],
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(readDictionaryZipRevision(zipPath), 'rev-42');
|
||||||
|
assert.equal(readDictionaryZipRevision(path.join(dir, 'missing.zip')), null);
|
||||||
|
|
||||||
|
const archive = fs.readFileSync(zipPath);
|
||||||
|
const truncatedPath = path.join(dir, 'truncated.zip');
|
||||||
|
fs.writeFileSync(truncatedPath, archive.subarray(0, 40));
|
||||||
|
assert.equal(readDictionaryZipRevision(truncatedPath), null);
|
||||||
|
|
||||||
|
// An archive cut short after index.json still holds a readable revision, but importing it
|
||||||
|
// would hand Yomitan a half-written file: the missing end-of-central-directory record has to
|
||||||
|
// reject it. One byte off the end is enough to make the record incomplete.
|
||||||
|
for (const missingBytes of [1, 22, archive.length - 200]) {
|
||||||
|
const cutPath = path.join(dir, `cut-${missingBytes}.zip`);
|
||||||
|
fs.writeFileSync(cutPath, archive.subarray(0, archive.length - missingBytes));
|
||||||
|
assert.equal(readDictionaryZipRevision(cutPath), null, `cut of ${missingBytes} bytes`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Same size, corrupt directory: a record overwritten in place has to be rejected too.
|
||||||
|
const centralStart = archive.readUInt32LE(archive.length - 22 + 16);
|
||||||
|
const brokenSignaturePath = path.join(dir, 'broken-signature.zip');
|
||||||
|
const brokenSignature = Buffer.from(archive);
|
||||||
|
brokenSignature.writeUInt32LE(0xdeadbeef, centralStart);
|
||||||
|
fs.writeFileSync(brokenSignaturePath, brokenSignature);
|
||||||
|
assert.equal(readDictionaryZipRevision(brokenSignaturePath), null);
|
||||||
|
|
||||||
|
const brokenLengthPath = path.join(dir, 'broken-length.zip');
|
||||||
|
const brokenLength = Buffer.from(archive);
|
||||||
|
// Name length that runs the walk past the end of the directory.
|
||||||
|
brokenLength.writeUInt16LE(0xffff, centralStart + 28);
|
||||||
|
fs.writeFileSync(brokenLengthPath, brokenLength);
|
||||||
|
assert.equal(readDictionaryZipRevision(brokenLengthPath), null);
|
||||||
|
|
||||||
|
const foreignPath = path.join(dir, 'foreign.zip');
|
||||||
|
fs.writeFileSync(foreignPath, Buffer.from('not a zip at all', 'utf8'));
|
||||||
|
assert.equal(readDictionaryZipRevision(foreignPath), null);
|
||||||
|
} finally {
|
||||||
|
cleanupDir(dir);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import { writeStoredZip } from '../../shared/stored-zip';
|
import { readStoredZipFirstFile, writeStoredZip } from '../../shared/stored-zip';
|
||||||
import { ensureDir } from './fs-utils';
|
import { ensureDir } from './fs-utils';
|
||||||
import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } from './types';
|
import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } from './types';
|
||||||
|
|
||||||
@@ -31,6 +31,23 @@ function createTagBank(): Array<[string, string, number, string, number]> {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Revision recorded inside a built dictionary ZIP, or null when the archive is missing, truncated,
|
||||||
|
* or not one of ours. `index.json` is always the first entry written by {@link buildDictionaryZip}.
|
||||||
|
*/
|
||||||
|
export function readDictionaryZipRevision(zipPath: string): string | null {
|
||||||
|
const firstFile = readStoredZipFirstFile(zipPath);
|
||||||
|
if (!firstFile || firstFile.name !== 'index.json') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const index = JSON.parse(firstFile.data.toString('utf8')) as { revision?: unknown };
|
||||||
|
return typeof index.revision === 'string' && index.revision.length > 0 ? index.revision : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export function buildDictionaryZip(
|
export function buildDictionaryZip(
|
||||||
outputPath: string,
|
outputPath: string,
|
||||||
dictionaryTitle: string,
|
dictionaryTitle: string,
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
import type { CharacterDictionarySnapshotStageProgress } from '../character-dictionary-runtime';
|
||||||
|
|
||||||
|
export function buildSyncingMessage(mediaTitle: string): string {
|
||||||
|
return `Updating character dictionary for ${mediaTitle}...`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildCheckingMessage(mediaTitle: string): string {
|
||||||
|
return `Checking character dictionary for ${mediaTitle}...`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildGeneratingMessage(mediaTitle: string, detail?: string): string {
|
||||||
|
return detail
|
||||||
|
? `Generating character dictionary for ${mediaTitle} (${detail})...`
|
||||||
|
: `Generating character dictionary for ${mediaTitle}...`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatCharacterDictionaryProgressDetail(
|
||||||
|
progress: CharacterDictionarySnapshotStageProgress,
|
||||||
|
remainingMs: number | null,
|
||||||
|
): string {
|
||||||
|
if (progress.stage === 'saving') {
|
||||||
|
return 'saving snapshot';
|
||||||
|
}
|
||||||
|
if (progress.stage === 'names') {
|
||||||
|
return progress.total !== null && progress.total > 0
|
||||||
|
? `name ${progress.completed}/${progress.total}`
|
||||||
|
: `${progress.completed} names`;
|
||||||
|
}
|
||||||
|
if (progress.stage === 'images') {
|
||||||
|
const counted =
|
||||||
|
progress.total !== null && progress.total > 0
|
||||||
|
? `image ${progress.completed}/${progress.total}`
|
||||||
|
: `${progress.completed} images`;
|
||||||
|
return remainingMs !== null
|
||||||
|
? `${counted}, ~${formatRemainingDuration(remainingMs)} left`
|
||||||
|
: counted;
|
||||||
|
}
|
||||||
|
const page = typeof progress.page === 'number' ? `page ${progress.page}, ` : '';
|
||||||
|
return `${page}${progress.completed} characters`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatElapsedDuration(elapsedMs: number): string {
|
||||||
|
const totalSeconds = Math.max(0, Math.floor(elapsedMs / 1000));
|
||||||
|
const minutes = Math.floor(totalSeconds / 60);
|
||||||
|
const seconds = totalSeconds % 60;
|
||||||
|
return minutes > 0 ? `${minutes}m ${String(seconds).padStart(2, '0')}s` : `${seconds}s`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Coarser than the elapsed clock: an estimate that ticks every second reads as precision it lacks. */
|
||||||
|
export function formatRemainingDuration(remainingMs: number): string {
|
||||||
|
const totalSeconds = Math.max(1, Math.round(remainingMs / 1000));
|
||||||
|
if (totalSeconds >= 60) {
|
||||||
|
return `${Math.max(1, Math.round(totalSeconds / 60))}m`;
|
||||||
|
}
|
||||||
|
return `${Math.max(5, Math.ceil(totalSeconds / 5) * 5)}s`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The elapsed clock is the part that proves the app is alive: a stalled network fetch freezes the
|
||||||
|
* counts, but the clock keeps moving.
|
||||||
|
*/
|
||||||
|
export function joinGeneratingDetail(detail: string | null, elapsedMs: number): string | undefined {
|
||||||
|
const parts = [detail, elapsedMs >= 5_000 ? formatElapsedDuration(elapsedMs) : null].filter(
|
||||||
|
(part): part is string => typeof part === 'string' && part.length > 0,
|
||||||
|
);
|
||||||
|
return parts.length > 0 ? parts.join(' · ') : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildImportingMessage(mediaTitle: string, elapsedMs?: number): string {
|
||||||
|
const elapsed =
|
||||||
|
typeof elapsedMs === 'number' && elapsedMs >= 1000
|
||||||
|
? ` (${formatElapsedDuration(elapsedMs)})`
|
||||||
|
: '';
|
||||||
|
return `Importing character dictionary for ${mediaTitle}${elapsed}...`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildBuildingMessage(mediaTitle: string): string {
|
||||||
|
return `Building character dictionary for ${mediaTitle}...`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildReadyMessage(mediaTitle: string): string {
|
||||||
|
return `Character dictionary ready for ${mediaTitle}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildFailedMessage(mediaTitle: string | null, errorMessage: string): string {
|
||||||
|
if (mediaTitle) {
|
||||||
|
return `Character dictionary sync failed for ${mediaTitle}: ${errorMessage}`;
|
||||||
|
}
|
||||||
|
return `Character dictionary sync failed: ${errorMessage}`;
|
||||||
|
}
|
||||||
@@ -214,3 +214,69 @@ test('auto sync notifications let startup sequencer own osd-system desktop deliv
|
|||||||
|
|
||||||
assert.deepEqual(calls, ['osd:importing', 'desktop:SubMiner:importing']);
|
assert.deepEqual(calls, ['osd:importing', 'desktop:SubMiner:importing']);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('auto sync desktop notifications reuse one replace id across every phase', () => {
|
||||||
|
const replaceIds: Array<string | undefined> = [];
|
||||||
|
const deps = {
|
||||||
|
getNotificationType: () => 'system' as const,
|
||||||
|
showOsd: () => undefined,
|
||||||
|
showDesktopNotification: (_title: string, options: { body?: string; replaceId?: string }) => {
|
||||||
|
replaceIds.push(options.replaceId);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const phase of ['checking', 'generating', 'importing', 'ready'] as const) {
|
||||||
|
notifyCharacterDictionaryAutoSyncStatus(makeEvent(phase, phase), deps);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.deepEqual(replaceIds, [
|
||||||
|
'character-dictionary-auto-sync',
|
||||||
|
'character-dictionary-auto-sync',
|
||||||
|
'character-dictionary-auto-sync',
|
||||||
|
'character-dictionary-auto-sync',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('overlay-unavailable desktop fallback shares the same replace id', () => {
|
||||||
|
const replaceIds: Array<string | undefined> = [];
|
||||||
|
|
||||||
|
notifyCharacterDictionaryAutoSyncStatus(makeEvent('generating', 'generating'), {
|
||||||
|
getNotificationType: () => 'overlay',
|
||||||
|
showOsd: () => undefined,
|
||||||
|
showDesktopNotification: (_title, options) => {
|
||||||
|
replaceIds.push(options.replaceId);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.deepEqual(replaceIds, ['character-dictionary-auto-sync']);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('startup lanes keep one desktop notification per lane', () => {
|
||||||
|
const calls: Array<{ body?: string; replaceId?: string }> = [];
|
||||||
|
const sequencer = createStartupOsdSequencer({
|
||||||
|
getNotificationType: () => 'system',
|
||||||
|
showOsd: () => undefined,
|
||||||
|
showDesktopNotification: (_title, options) => {
|
||||||
|
calls.push(options);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
sequencer.markTokenizationReady();
|
||||||
|
notifyCharacterDictionaryAutoSyncStatus(makeEvent('generating', 'generating one'), {
|
||||||
|
getNotificationType: () => 'osd',
|
||||||
|
showOsd: () => undefined,
|
||||||
|
showDesktopNotification: () => undefined,
|
||||||
|
startupOsdSequencer: sequencer,
|
||||||
|
});
|
||||||
|
notifyCharacterDictionaryAutoSyncStatus(makeEvent('generating', 'generating two'), {
|
||||||
|
getNotificationType: () => 'osd',
|
||||||
|
showOsd: () => undefined,
|
||||||
|
showDesktopNotification: () => undefined,
|
||||||
|
startupOsdSequencer: sequencer,
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.deepEqual(
|
||||||
|
calls.map((call) => call.replaceId),
|
||||||
|
['startup-status', 'startup-status'],
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ export interface CharacterDictionaryAutoSyncNotificationDeps {
|
|||||||
getNotificationType: () => NotificationType | undefined;
|
getNotificationType: () => NotificationType | undefined;
|
||||||
showOsd: (message: string) => boolean | void;
|
showOsd: (message: string) => boolean | void;
|
||||||
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
|
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
|
||||||
showDesktopNotification: (title: string, options: { body?: string }) => void;
|
showDesktopNotification: (title: string, options: { body?: string; replaceId?: string }) => void;
|
||||||
startupOsdSequencer?: {
|
startupOsdSequencer?: {
|
||||||
notifyCharacterDictionaryStatus: (
|
notifyCharacterDictionaryStatus: (
|
||||||
event: StartupOsdSequencerCharacterDictionaryEvent,
|
event: StartupOsdSequencerCharacterDictionaryEvent,
|
||||||
@@ -17,6 +17,10 @@ export interface CharacterDictionaryAutoSyncNotificationDeps {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One live desktop notification for the whole sync: progress updates replace each other and the
|
||||||
|
// terminal ready/failed message replaces the last progress one, matching the overlay toast.
|
||||||
|
const CHARACTER_DICTIONARY_DESKTOP_NOTIFICATION_ID = 'character-dictionary-auto-sync';
|
||||||
|
|
||||||
function isTerminalPhase(phase: CharacterDictionaryAutoSyncNotificationEvent['phase']): boolean {
|
function isTerminalPhase(phase: CharacterDictionaryAutoSyncNotificationEvent['phase']): boolean {
|
||||||
return phase === 'ready' || phase === 'failed';
|
return phase === 'ready' || phase === 'failed';
|
||||||
}
|
}
|
||||||
@@ -53,7 +57,10 @@ export function notifyCharacterDictionaryAutoSyncStatus(
|
|||||||
persistent: !isTerminalPhase(event.phase),
|
persistent: !isTerminalPhase(event.phase),
|
||||||
});
|
});
|
||||||
} else if (!shouldShowDesktop(type)) {
|
} else if (!shouldShowDesktop(type)) {
|
||||||
deps.showDesktopNotification('SubMiner', { body: event.message });
|
deps.showDesktopNotification('SubMiner', {
|
||||||
|
body: event.message,
|
||||||
|
replaceId: CHARACTER_DICTIONARY_DESKTOP_NOTIFICATION_ID,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,6 +76,9 @@ export function notifyCharacterDictionaryAutoSyncStatus(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (shouldShowDesktop(type) && !startupSequencerShown) {
|
if (shouldShowDesktop(type) && !startupSequencerShown) {
|
||||||
deps.showDesktopNotification('SubMiner', { body: event.message });
|
deps.showDesktopNotification('SubMiner', {
|
||||||
|
body: event.message,
|
||||||
|
replaceId: CHARACTER_DICTIONARY_DESKTOP_NOTIFICATION_ID,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import * as fs from 'fs';
|
|||||||
import * as os from 'os';
|
import * as os from 'os';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
|
import { buildDictionaryZip } from '../character-dictionary-runtime/zip';
|
||||||
import {
|
import {
|
||||||
createCharacterDictionaryAutoSyncRuntimeService,
|
createCharacterDictionaryAutoSyncRuntimeService,
|
||||||
getCharacterDictionaryManagerSnapshot,
|
getCharacterDictionaryManagerSnapshot,
|
||||||
@@ -14,6 +15,14 @@ function makeTempDir(): string {
|
|||||||
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-char-dict-auto-sync-'));
|
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-char-dict-auto-sync-'));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function waitUntil(predicate: () => boolean, label: string): Promise<void> {
|
||||||
|
for (let attempt = 0; attempt < 500; attempt += 1) {
|
||||||
|
if (predicate()) return;
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
}
|
||||||
|
throw new Error(`Timed out waiting for ${label}`);
|
||||||
|
}
|
||||||
|
|
||||||
function createDeferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
|
function createDeferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
|
||||||
let resolve!: (value: T) => void;
|
let resolve!: (value: T) => void;
|
||||||
const promise = new Promise<T>((nextResolve) => {
|
const promise = new Promise<T>((nextResolve) => {
|
||||||
@@ -187,7 +196,7 @@ test('auto sync imports merged dictionary and persists MRU state', async () => {
|
|||||||
'[dictionary:auto-sync] syncing current anime snapshot',
|
'[dictionary:auto-sync] syncing current anime snapshot',
|
||||||
'[dictionary:auto-sync] active AniList media set: 130298 - The Eminence in Shadow',
|
'[dictionary:auto-sync] active AniList media set: 130298 - The Eminence in Shadow',
|
||||||
'[dictionary:auto-sync] rebuilding merged dictionary for active anime set',
|
'[dictionary:auto-sync] rebuilding merged dictionary for active anime set',
|
||||||
'[dictionary:auto-sync] importing merged dictionary: /tmp/subminer-character-dictionary.zip',
|
'[dictionary:auto-sync] importing merged dictionary: /tmp/subminer-character-dictionary.zip (timeout 120000ms)',
|
||||||
'[dictionary:auto-sync] applying Yomitan settings for SubMiner Character Dictionary',
|
'[dictionary:auto-sync] applying Yomitan settings for SubMiner Character Dictionary',
|
||||||
'[dictionary:auto-sync] synced AniList 130298: SubMiner Character Dictionary (2544 entries)',
|
'[dictionary:auto-sync] synced AniList 130298: SubMiner Character Dictionary (2544 entries)',
|
||||||
]);
|
]);
|
||||||
@@ -367,7 +376,14 @@ test('auto sync reimports existing merged zip without rebuilding on unchanged re
|
|||||||
const userDataPath = makeTempDir();
|
const userDataPath = makeTempDir();
|
||||||
const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
|
const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
|
||||||
fs.mkdirSync(dictionariesDir, { recursive: true });
|
fs.mkdirSync(dictionariesDir, { recursive: true });
|
||||||
fs.writeFileSync(path.join(dictionariesDir, 'merged.zip'), 'cached-zip', 'utf8');
|
buildDictionaryZip(
|
||||||
|
path.join(dictionariesDir, 'merged.zip'),
|
||||||
|
'SubMiner Character Dictionary',
|
||||||
|
'Character names',
|
||||||
|
'rev-7',
|
||||||
|
[{ term: 'フリーレン', reading: 'フリーレン', role: 'main', glossary: [] } as never],
|
||||||
|
[],
|
||||||
|
);
|
||||||
const mergedBuilds: number[][] = [];
|
const mergedBuilds: number[][] = [];
|
||||||
const imports: string[] = [];
|
const imports: string[] = [];
|
||||||
let importedRevision: string | null = null;
|
let importedRevision: string | null = null;
|
||||||
@@ -958,11 +974,9 @@ test('auto sync emits building while merged dictionary generation is in flight',
|
|||||||
});
|
});
|
||||||
|
|
||||||
const syncPromise = runtime.runSyncNow();
|
const syncPromise = runtime.runSyncNow();
|
||||||
await Promise.resolve();
|
await waitUntil(
|
||||||
|
() => events.some((event) => event.phase === 'building'),
|
||||||
assert.equal(
|
'the building status event',
|
||||||
events.some((event) => event.phase === 'building'),
|
|
||||||
true,
|
|
||||||
);
|
);
|
||||||
|
|
||||||
buildDeferred.resolve({
|
buildDeferred.resolve({
|
||||||
@@ -1029,8 +1043,7 @@ test('auto sync waits for tokenization-ready gate before Yomitan mutations', asy
|
|||||||
});
|
});
|
||||||
|
|
||||||
const syncPromise = runtime.runSyncNow();
|
const syncPromise = runtime.runSyncNow();
|
||||||
await Promise.resolve();
|
await waitUntil(() => calls.includes('wait'), 'the tokenization-ready gate');
|
||||||
await Promise.resolve();
|
|
||||||
|
|
||||||
assert.deepEqual(calls, ['build', 'wait']);
|
assert.deepEqual(calls, ['build', 'wait']);
|
||||||
|
|
||||||
@@ -1039,3 +1052,395 @@ test('auto sync waits for tokenization-ready gate before Yomitan mutations', asy
|
|||||||
|
|
||||||
assert.deepEqual(calls, ['build', 'wait', 'info', 'import', 'settings']);
|
assert.deepEqual(calls, ['build', 'wait', 'info', 'import', 'settings']);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('auto sync scales the import timeout with the merged dictionary size', async () => {
|
||||||
|
const userDataPath = makeTempDir();
|
||||||
|
const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
|
||||||
|
fs.mkdirSync(dictionariesDir, { recursive: true });
|
||||||
|
const zipPath = path.join(dictionariesDir, 'merged.zip');
|
||||||
|
// 2 MB of merged dictionary buys ~12s of import budget on top of the base.
|
||||||
|
fs.writeFileSync(zipPath, Buffer.alloc(2 * 1024 * 1024));
|
||||||
|
const events: Array<{ phase: string; message: string }> = [];
|
||||||
|
let importedRevision: string | null = null;
|
||||||
|
|
||||||
|
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
|
||||||
|
userDataPath,
|
||||||
|
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
|
||||||
|
getOrCreateCurrentSnapshot: async () => ({
|
||||||
|
mediaId: 21,
|
||||||
|
mediaTitle: 'ONE PIECE',
|
||||||
|
entryCount: 4000,
|
||||||
|
fromCache: false,
|
||||||
|
updatedAt: 1000,
|
||||||
|
}),
|
||||||
|
buildMergedDictionary: async () => ({
|
||||||
|
zipPath,
|
||||||
|
revision: 'rev-21',
|
||||||
|
dictionaryTitle: 'SubMiner Character Dictionary',
|
||||||
|
entryCount: 4000,
|
||||||
|
}),
|
||||||
|
getYomitanDictionaryInfo: async () =>
|
||||||
|
importedRevision
|
||||||
|
? [{ title: 'SubMiner Character Dictionary', revision: importedRevision }]
|
||||||
|
: [],
|
||||||
|
importYomitanDictionary: async () => {
|
||||||
|
// Far longer than the quick-operation budget, well inside the size-scaled one.
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 400));
|
||||||
|
importedRevision = 'rev-21';
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
deleteYomitanDictionary: async () => true,
|
||||||
|
upsertYomitanDictionarySettings: async () => true,
|
||||||
|
now: () => 1000,
|
||||||
|
// Comfortable for the stubs that resolve immediately, still far under the import's 400ms.
|
||||||
|
operationTimeoutMs: 100,
|
||||||
|
dictionaryImportTimeoutBaseMs: 20,
|
||||||
|
onSyncStatus: (event) => {
|
||||||
|
events.push({ phase: event.phase, message: event.message });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await runtime.runSyncNow();
|
||||||
|
|
||||||
|
assert.equal(
|
||||||
|
events.some((event) => event.phase === 'failed'),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
assert.deepEqual(events.at(-1), {
|
||||||
|
phase: 'ready',
|
||||||
|
message: 'Character dictionary ready for ONE PIECE',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('auto sync reports the scaled budget when an import really does hang', async () => {
|
||||||
|
const userDataPath = makeTempDir();
|
||||||
|
const events: Array<{ phase: string; message: string }> = [];
|
||||||
|
|
||||||
|
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
|
||||||
|
userDataPath,
|
||||||
|
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
|
||||||
|
getOrCreateCurrentSnapshot: async () => ({
|
||||||
|
mediaId: 21,
|
||||||
|
mediaTitle: 'ONE PIECE',
|
||||||
|
entryCount: 4000,
|
||||||
|
fromCache: false,
|
||||||
|
updatedAt: 1000,
|
||||||
|
}),
|
||||||
|
buildMergedDictionary: async () => ({
|
||||||
|
zipPath: path.join(userDataPath, 'character-dictionaries', 'missing.zip'),
|
||||||
|
revision: 'rev-21',
|
||||||
|
dictionaryTitle: 'SubMiner Character Dictionary',
|
||||||
|
entryCount: 4000,
|
||||||
|
}),
|
||||||
|
getYomitanDictionaryInfo: async () => [],
|
||||||
|
importYomitanDictionary: () => new Promise<boolean>(() => {}),
|
||||||
|
deleteYomitanDictionary: async () => true,
|
||||||
|
upsertYomitanDictionarySettings: async () => true,
|
||||||
|
now: () => 1000,
|
||||||
|
dictionaryImportTimeoutBaseMs: 20,
|
||||||
|
onSyncStatus: (event) => {
|
||||||
|
events.push({ phase: event.phase, message: event.message });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
runtime.runSyncNow(),
|
||||||
|
/importYomitanDictionary\(missing\.zip\) timed out after 20ms/,
|
||||||
|
);
|
||||||
|
assert.equal(events.at(-1)?.phase, 'failed');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('auto sync ticks the importing notification while the import runs', async () => {
|
||||||
|
const userDataPath = makeTempDir();
|
||||||
|
const events: Array<{ phase: string; message: string }> = [];
|
||||||
|
const scheduled: Array<() => void> = [];
|
||||||
|
const importDeferred = createDeferred<boolean>();
|
||||||
|
let clock = 1000;
|
||||||
|
|
||||||
|
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
|
||||||
|
userDataPath,
|
||||||
|
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
|
||||||
|
getOrCreateCurrentSnapshot: async () => ({
|
||||||
|
mediaId: 21,
|
||||||
|
mediaTitle: 'ONE PIECE',
|
||||||
|
entryCount: 4000,
|
||||||
|
fromCache: false,
|
||||||
|
updatedAt: 1000,
|
||||||
|
}),
|
||||||
|
buildMergedDictionary: async () => ({
|
||||||
|
zipPath: '/tmp/merged.zip',
|
||||||
|
revision: 'rev-21',
|
||||||
|
dictionaryTitle: 'SubMiner Character Dictionary',
|
||||||
|
entryCount: 4000,
|
||||||
|
}),
|
||||||
|
getYomitanDictionaryInfo: async () => [],
|
||||||
|
importYomitanDictionary: () => importDeferred.promise,
|
||||||
|
deleteYomitanDictionary: async () => true,
|
||||||
|
upsertYomitanDictionarySettings: async () => true,
|
||||||
|
now: () => clock,
|
||||||
|
schedule: (fn) => {
|
||||||
|
scheduled.push(fn);
|
||||||
|
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||||
|
},
|
||||||
|
clearSchedule: () => undefined,
|
||||||
|
onSyncStatus: (event) => {
|
||||||
|
events.push({ phase: event.phase, message: event.message });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const syncPromise = runtime.runSyncNow();
|
||||||
|
await waitUntil(
|
||||||
|
() => events.some((event) => event.phase === 'importing'),
|
||||||
|
'the importing status event',
|
||||||
|
);
|
||||||
|
|
||||||
|
clock = 1000 + 65_000;
|
||||||
|
// The importing heartbeat is the most recently scheduled tick.
|
||||||
|
scheduled.at(-1)!();
|
||||||
|
|
||||||
|
assert.deepEqual(events.at(-1), {
|
||||||
|
phase: 'importing',
|
||||||
|
message: 'Importing character dictionary for ONE PIECE (1m 05s)...',
|
||||||
|
});
|
||||||
|
|
||||||
|
importDeferred.resolve(true);
|
||||||
|
await syncPromise;
|
||||||
|
assert.equal(events.at(-1)?.phase, 'ready');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('auto sync reports character and image counts while generating a snapshot', async () => {
|
||||||
|
const userDataPath = makeTempDir();
|
||||||
|
const events: Array<{ phase: string; message: string }> = [];
|
||||||
|
let clock = 1000;
|
||||||
|
let importedRevision: string | null = null;
|
||||||
|
|
||||||
|
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
|
||||||
|
userDataPath,
|
||||||
|
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
|
||||||
|
getOrCreateCurrentSnapshot: async (_targetPath, progress) => {
|
||||||
|
progress?.onGenerating?.({ mediaId: 21, mediaTitle: 'ONE PIECE' });
|
||||||
|
progress?.onGenerateProgress?.({
|
||||||
|
mediaId: 21,
|
||||||
|
mediaTitle: 'ONE PIECE',
|
||||||
|
stage: 'characters',
|
||||||
|
completed: 50,
|
||||||
|
total: null,
|
||||||
|
page: 12,
|
||||||
|
});
|
||||||
|
// Same stage, same clock tick: throttled away so a 33-page fetch cannot spam the overlay.
|
||||||
|
progress?.onGenerateProgress?.({
|
||||||
|
mediaId: 21,
|
||||||
|
mediaTitle: 'ONE PIECE',
|
||||||
|
stage: 'characters',
|
||||||
|
completed: 100,
|
||||||
|
total: null,
|
||||||
|
page: 13,
|
||||||
|
});
|
||||||
|
// A stage change always reports, throttle window or not.
|
||||||
|
progress?.onGenerateProgress?.({
|
||||||
|
mediaId: 21,
|
||||||
|
mediaTitle: 'ONE PIECE',
|
||||||
|
stage: 'images',
|
||||||
|
completed: 1,
|
||||||
|
total: 1220,
|
||||||
|
});
|
||||||
|
clock += 2000;
|
||||||
|
progress?.onGenerateProgress?.({
|
||||||
|
mediaId: 21,
|
||||||
|
mediaTitle: 'ONE PIECE',
|
||||||
|
stage: 'images',
|
||||||
|
completed: 240,
|
||||||
|
total: 1220,
|
||||||
|
});
|
||||||
|
clock += 6000;
|
||||||
|
progress?.onGenerateProgress?.({
|
||||||
|
mediaId: 21,
|
||||||
|
mediaTitle: 'ONE PIECE',
|
||||||
|
stage: 'names',
|
||||||
|
completed: 800,
|
||||||
|
total: 1220,
|
||||||
|
});
|
||||||
|
progress?.onGenerateProgress?.({
|
||||||
|
mediaId: 21,
|
||||||
|
mediaTitle: 'ONE PIECE',
|
||||||
|
stage: 'saving',
|
||||||
|
completed: 0,
|
||||||
|
total: null,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
mediaId: 21,
|
||||||
|
mediaTitle: 'ONE PIECE',
|
||||||
|
entryCount: 4000,
|
||||||
|
fromCache: false,
|
||||||
|
updatedAt: 1000,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
buildMergedDictionary: async () => ({
|
||||||
|
zipPath: '/tmp/merged.zip',
|
||||||
|
revision: 'rev-21',
|
||||||
|
dictionaryTitle: 'SubMiner Character Dictionary',
|
||||||
|
entryCount: 4000,
|
||||||
|
}),
|
||||||
|
getYomitanDictionaryInfo: async () =>
|
||||||
|
importedRevision
|
||||||
|
? [{ title: 'SubMiner Character Dictionary', revision: importedRevision }]
|
||||||
|
: [],
|
||||||
|
importYomitanDictionary: async () => {
|
||||||
|
importedRevision = 'rev-21';
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
deleteYomitanDictionary: async () => true,
|
||||||
|
upsertYomitanDictionarySettings: async () => true,
|
||||||
|
now: () => clock,
|
||||||
|
onSyncStatus: (event) => {
|
||||||
|
events.push({ phase: event.phase, message: event.message });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await runtime.runSyncNow();
|
||||||
|
|
||||||
|
assert.deepEqual(
|
||||||
|
events.filter((event) => event.phase === 'generating').map((event) => event.message),
|
||||||
|
[
|
||||||
|
'Generating character dictionary for ONE PIECE...',
|
||||||
|
'Generating character dictionary for ONE PIECE (page 12, 50 characters)...',
|
||||||
|
'Generating character dictionary for ONE PIECE (image 1/1220)...',
|
||||||
|
'Generating character dictionary for ONE PIECE (image 240/1220, ~10s left)...',
|
||||||
|
'Generating character dictionary for ONE PIECE (name 800/1220 · 8s)...',
|
||||||
|
'Generating character dictionary for ONE PIECE (saving snapshot · 8s)...',
|
||||||
|
],
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('auto sync keeps the generating clock ticking when a stage stalls', async () => {
|
||||||
|
const userDataPath = makeTempDir();
|
||||||
|
const events: Array<{ phase: string; message: string }> = [];
|
||||||
|
const scheduled: Array<() => void> = [];
|
||||||
|
const snapshotDeferred = createDeferred<{
|
||||||
|
mediaId: number;
|
||||||
|
mediaTitle: string;
|
||||||
|
entryCount: number;
|
||||||
|
fromCache: boolean;
|
||||||
|
updatedAt: number;
|
||||||
|
}>();
|
||||||
|
let clock = 1000;
|
||||||
|
|
||||||
|
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
|
||||||
|
userDataPath,
|
||||||
|
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
|
||||||
|
getOrCreateCurrentSnapshot: async (_targetPath, progress) => {
|
||||||
|
progress?.onGenerating?.({ mediaId: 21, mediaTitle: 'ONE PIECE' });
|
||||||
|
progress?.onGenerateProgress?.({
|
||||||
|
mediaId: 21,
|
||||||
|
mediaTitle: 'ONE PIECE',
|
||||||
|
stage: 'images',
|
||||||
|
completed: 240,
|
||||||
|
total: 1220,
|
||||||
|
});
|
||||||
|
return await snapshotDeferred.promise;
|
||||||
|
},
|
||||||
|
buildMergedDictionary: async () => ({
|
||||||
|
zipPath: '/tmp/merged.zip',
|
||||||
|
revision: 'rev-21',
|
||||||
|
dictionaryTitle: 'SubMiner Character Dictionary',
|
||||||
|
entryCount: 4000,
|
||||||
|
}),
|
||||||
|
getYomitanDictionaryInfo: async () => [],
|
||||||
|
importYomitanDictionary: async () => true,
|
||||||
|
deleteYomitanDictionary: async () => true,
|
||||||
|
upsertYomitanDictionarySettings: async () => true,
|
||||||
|
now: () => clock,
|
||||||
|
schedule: (fn) => {
|
||||||
|
scheduled.push(fn);
|
||||||
|
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||||
|
},
|
||||||
|
clearSchedule: () => undefined,
|
||||||
|
onSyncStatus: (event) => {
|
||||||
|
events.push({ phase: event.phase, message: event.message });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const syncPromise = runtime.runSyncNow();
|
||||||
|
await waitUntil(() => scheduled.length > 0, 'the generating heartbeat');
|
||||||
|
|
||||||
|
// No further progress arrives: only the clock moves.
|
||||||
|
clock += 95_000;
|
||||||
|
scheduled.at(-1)!();
|
||||||
|
|
||||||
|
assert.deepEqual(events.at(-1), {
|
||||||
|
phase: 'generating',
|
||||||
|
message: 'Generating character dictionary for ONE PIECE (image 240/1220 · 1m 35s)...',
|
||||||
|
});
|
||||||
|
|
||||||
|
snapshotDeferred.resolve({
|
||||||
|
mediaId: 21,
|
||||||
|
mediaTitle: 'ONE PIECE',
|
||||||
|
entryCount: 4000,
|
||||||
|
fromCache: false,
|
||||||
|
updatedAt: 1000,
|
||||||
|
});
|
||||||
|
await syncPromise;
|
||||||
|
assert.equal(events.at(-1)?.phase, 'ready');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('auto sync rebuilds instead of importing a cached merged ZIP with a mismatched revision', async () => {
|
||||||
|
const userDataPath = makeTempDir();
|
||||||
|
const dictionariesDir = path.join(userDataPath, 'character-dictionaries');
|
||||||
|
fs.mkdirSync(dictionariesDir, { recursive: true });
|
||||||
|
const statePath = path.join(dictionariesDir, 'auto-sync-state.json');
|
||||||
|
fs.writeFileSync(
|
||||||
|
statePath,
|
||||||
|
JSON.stringify({
|
||||||
|
activeMediaIds: ['7 - Frieren'],
|
||||||
|
mergedRevision: 'rev-7',
|
||||||
|
mergedDictionaryTitle: 'SubMiner Character Dictionary',
|
||||||
|
}),
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
// Left over from an interrupted run: the archive on disk is not the revision state recorded.
|
||||||
|
buildDictionaryZip(
|
||||||
|
path.join(dictionariesDir, 'merged.zip'),
|
||||||
|
'SubMiner Character Dictionary',
|
||||||
|
'Character names',
|
||||||
|
'rev-stale',
|
||||||
|
[{ term: 'フリーレン', reading: 'フリーレン', role: 'main', glossary: [] } as never],
|
||||||
|
[],
|
||||||
|
);
|
||||||
|
const mergedBuilds: number[][] = [];
|
||||||
|
const imports: string[] = [];
|
||||||
|
|
||||||
|
const runtime = createCharacterDictionaryAutoSyncRuntimeService({
|
||||||
|
userDataPath,
|
||||||
|
getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }),
|
||||||
|
getOrCreateCurrentSnapshot: async () => ({
|
||||||
|
mediaId: 7,
|
||||||
|
mediaTitle: 'Frieren',
|
||||||
|
entryCount: 100,
|
||||||
|
fromCache: true,
|
||||||
|
updatedAt: 1000,
|
||||||
|
}),
|
||||||
|
buildMergedDictionary: async (mediaIds) => {
|
||||||
|
mergedBuilds.push([...mediaIds]);
|
||||||
|
return {
|
||||||
|
zipPath: '/tmp/rebuilt-merged.zip',
|
||||||
|
revision: 'rev-7',
|
||||||
|
dictionaryTitle: 'SubMiner Character Dictionary',
|
||||||
|
entryCount: 100,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
// Yomitan does not have the dictionary, so the sync has to import despite the cached state.
|
||||||
|
getYomitanDictionaryInfo: async () => [],
|
||||||
|
importYomitanDictionary: async (zipPath) => {
|
||||||
|
imports.push(zipPath);
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
deleteYomitanDictionary: async () => true,
|
||||||
|
upsertYomitanDictionarySettings: async () => true,
|
||||||
|
now: () => 1000,
|
||||||
|
});
|
||||||
|
|
||||||
|
await runtime.runSyncNow();
|
||||||
|
|
||||||
|
assert.deepEqual(mergedBuilds, [[7]]);
|
||||||
|
assert.deepEqual(imports, ['/tmp/rebuilt-merged.zip']);
|
||||||
|
});
|
||||||
|
|||||||
@@ -1,13 +1,29 @@
|
|||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import { ensureDir } from '../../shared/fs-utils';
|
import { ensureDir } from '../../shared/fs-utils';
|
||||||
|
import {
|
||||||
|
buildBuildingMessage,
|
||||||
|
buildCheckingMessage,
|
||||||
|
buildFailedMessage,
|
||||||
|
buildGeneratingMessage,
|
||||||
|
buildImportingMessage,
|
||||||
|
buildReadyMessage,
|
||||||
|
formatCharacterDictionaryProgressDetail,
|
||||||
|
joinGeneratingDetail,
|
||||||
|
} from './character-dictionary-auto-sync-messages';
|
||||||
|
import { readDictionaryZipRevision } from '../character-dictionary-runtime/zip';
|
||||||
import type { AnilistCharacterDictionaryProfileScope } from '../../types';
|
import type { AnilistCharacterDictionaryProfileScope } from '../../types';
|
||||||
import type {
|
import type {
|
||||||
CharacterDictionarySnapshotProgressCallbacks,
|
CharacterDictionarySnapshotProgressCallbacks,
|
||||||
CharacterDictionarySnapshotResult,
|
CharacterDictionarySnapshotResult,
|
||||||
|
CharacterDictionarySnapshotStageProgress,
|
||||||
MergedCharacterDictionaryBuildResult,
|
MergedCharacterDictionaryBuildResult,
|
||||||
} from '../character-dictionary-runtime';
|
} from '../character-dictionary-runtime';
|
||||||
|
|
||||||
|
const DEFAULT_IMPORT_TIMEOUT_BASE_MS = 120_000;
|
||||||
|
const IMPORT_TIMEOUT_MS_PER_MB = 6_000;
|
||||||
|
const IMPORT_TIMEOUT_MAX_MS = 1_800_000;
|
||||||
|
|
||||||
type AutoSyncMediaEntry = {
|
type AutoSyncMediaEntry = {
|
||||||
mediaId: number;
|
mediaId: number;
|
||||||
label: string;
|
label: string;
|
||||||
@@ -72,7 +88,15 @@ export interface CharacterDictionaryAutoSyncRuntimeDeps {
|
|||||||
now: () => number;
|
now: () => number;
|
||||||
schedule?: (fn: () => void, delayMs: number) => ReturnType<typeof setTimeout>;
|
schedule?: (fn: () => void, delayMs: number) => ReturnType<typeof setTimeout>;
|
||||||
clearSchedule?: (timer: ReturnType<typeof setTimeout>) => void;
|
clearSchedule?: (timer: ReturnType<typeof setTimeout>) => void;
|
||||||
|
/** Budget for the quick Yomitan queries (dictionary info, settings upsert). */
|
||||||
operationTimeoutMs?: number;
|
operationTimeoutMs?: number;
|
||||||
|
/**
|
||||||
|
* Base budget for the slow Yomitan mutations (delete + import). The effective budget grows with
|
||||||
|
* the merged ZIP size, because importing a large dictionary can take several minutes.
|
||||||
|
*/
|
||||||
|
dictionaryImportTimeoutBaseMs?: number;
|
||||||
|
heartbeatMs?: number;
|
||||||
|
progressThrottleMs?: number;
|
||||||
logInfo?: (message: string) => void;
|
logInfo?: (message: string) => void;
|
||||||
logWarn?: (message: string) => void;
|
logWarn?: (message: string) => void;
|
||||||
onSyncStatus?: (event: CharacterDictionaryAutoSyncStatusEvent) => void;
|
onSyncStatus?: (event: CharacterDictionaryAutoSyncStatusEvent) => void;
|
||||||
@@ -345,37 +369,6 @@ function sameMembership(left: number[], right: number[]): boolean {
|
|||||||
return arraysEqual(leftSorted, rightSorted);
|
return arraysEqual(leftSorted, rightSorted);
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildSyncingMessage(mediaTitle: string): string {
|
|
||||||
return `Updating character dictionary for ${mediaTitle}...`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildCheckingMessage(mediaTitle: string): string {
|
|
||||||
return `Checking character dictionary for ${mediaTitle}...`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildGeneratingMessage(mediaTitle: string): string {
|
|
||||||
return `Generating character dictionary for ${mediaTitle}...`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildImportingMessage(mediaTitle: string): string {
|
|
||||||
return `Importing character dictionary for ${mediaTitle}...`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildBuildingMessage(mediaTitle: string): string {
|
|
||||||
return `Building character dictionary for ${mediaTitle}...`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildReadyMessage(mediaTitle: string): string {
|
|
||||||
return `Character dictionary ready for ${mediaTitle}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildFailedMessage(mediaTitle: string | null, errorMessage: string): string {
|
|
||||||
if (mediaTitle) {
|
|
||||||
return `Character dictionary sync failed for ${mediaTitle}: ${errorMessage}`;
|
|
||||||
}
|
|
||||||
return `Character dictionary sync failed: ${errorMessage}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function createCharacterDictionaryAutoSyncRuntimeService(
|
export function createCharacterDictionaryAutoSyncRuntimeService(
|
||||||
deps: CharacterDictionaryAutoSyncRuntimeDeps,
|
deps: CharacterDictionaryAutoSyncRuntimeDeps,
|
||||||
): {
|
): {
|
||||||
@@ -389,21 +382,31 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
|
|||||||
const clearSchedule = deps.clearSchedule ?? ((timer) => clearTimeout(timer));
|
const clearSchedule = deps.clearSchedule ?? ((timer) => clearTimeout(timer));
|
||||||
const debounceMs = 800;
|
const debounceMs = 800;
|
||||||
const operationTimeoutMs = Math.max(1, Math.floor(deps.operationTimeoutMs ?? 7_000));
|
const operationTimeoutMs = Math.max(1, Math.floor(deps.operationTimeoutMs ?? 7_000));
|
||||||
|
const dictionaryImportTimeoutBaseMs = Math.max(
|
||||||
|
1,
|
||||||
|
Math.floor(deps.dictionaryImportTimeoutBaseMs ?? DEFAULT_IMPORT_TIMEOUT_BASE_MS),
|
||||||
|
);
|
||||||
|
const heartbeatMs = Math.max(1, Math.floor(deps.heartbeatMs ?? 5_000));
|
||||||
|
const progressThrottleMs = Math.max(0, Math.floor(deps.progressThrottleMs ?? 1_000));
|
||||||
|
|
||||||
let debounceTimer: ReturnType<typeof setTimeout> | null = null;
|
let debounceTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
let syncInFlight = false;
|
let syncInFlight = false;
|
||||||
let runQueued = false;
|
let runQueued = false;
|
||||||
let activeCurrentMediaId: number | null = null;
|
let activeCurrentMediaId: number | null = null;
|
||||||
|
|
||||||
const withOperationTimeout = async <T>(label: string, promise: Promise<T>): Promise<T> => {
|
const withTimeout = async <T>(
|
||||||
|
label: string,
|
||||||
|
promise: Promise<T>,
|
||||||
|
timeoutMs: number,
|
||||||
|
): Promise<T> => {
|
||||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||||
try {
|
try {
|
||||||
return await Promise.race([
|
return await Promise.race([
|
||||||
promise,
|
promise,
|
||||||
new Promise<never>((_resolve, reject) => {
|
new Promise<never>((_resolve, reject) => {
|
||||||
timer = setTimeout(() => {
|
timer = setTimeout(() => {
|
||||||
reject(new Error(`${label} timed out after ${operationTimeoutMs}ms`));
|
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
|
||||||
}, operationTimeoutMs);
|
}, timeoutMs);
|
||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
} finally {
|
} finally {
|
||||||
@@ -413,6 +416,57 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const withOperationTimeout = <T>(label: string, promise: Promise<T>): Promise<T> =>
|
||||||
|
withTimeout(label, promise, operationTimeoutMs);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Importing a merged dictionary means Yomitan writing every term and image into IndexedDB, which
|
||||||
|
* scales with the ZIP: a single-season dictionary lands in seconds, One Piece takes minutes. Size
|
||||||
|
* the budget off the archive instead of failing a healthy import on a flat deadline.
|
||||||
|
*/
|
||||||
|
const resolveImportTimeoutMs = (zipPath: string | null): number => {
|
||||||
|
let bytes = 0;
|
||||||
|
if (zipPath) {
|
||||||
|
try {
|
||||||
|
bytes = fs.statSync(zipPath).size;
|
||||||
|
} catch {
|
||||||
|
bytes = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const sizeAllowanceMs = (bytes / (1024 * 1024)) * IMPORT_TIMEOUT_MS_PER_MB;
|
||||||
|
return Math.min(
|
||||||
|
IMPORT_TIMEOUT_MAX_MS,
|
||||||
|
Math.max(
|
||||||
|
dictionaryImportTimeoutBaseMs,
|
||||||
|
Math.round(dictionaryImportTimeoutBaseMs + sizeAllowanceMs),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Keeps the persistent notification ticking so a multi-minute import never looks hung. */
|
||||||
|
const withHeartbeat = async <T>(
|
||||||
|
run: () => Promise<T>,
|
||||||
|
onTick: (elapsedMs: number) => void,
|
||||||
|
): Promise<T> => {
|
||||||
|
const startedAt = deps.now();
|
||||||
|
let stopped = false;
|
||||||
|
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
const tick = (): void => {
|
||||||
|
if (stopped) return;
|
||||||
|
onTick(deps.now() - startedAt);
|
||||||
|
timer = schedule(tick, heartbeatMs);
|
||||||
|
};
|
||||||
|
timer = schedule(tick, heartbeatMs);
|
||||||
|
try {
|
||||||
|
return await run();
|
||||||
|
} finally {
|
||||||
|
stopped = true;
|
||||||
|
if (timer !== null) {
|
||||||
|
clearSchedule(timer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const runSyncOnce = async (): Promise<void> => {
|
const runSyncOnce = async (): Promise<void> => {
|
||||||
const config = deps.getConfig();
|
const config = deps.getConfig();
|
||||||
if (!config.enabled) {
|
if (!config.enabled) {
|
||||||
@@ -422,33 +476,112 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
|
|||||||
|
|
||||||
let currentMediaId: number | undefined;
|
let currentMediaId: number | undefined;
|
||||||
let currentMediaTitle: string | null = null;
|
let currentMediaTitle: string | null = null;
|
||||||
|
let lastProgressAt = Number.NEGATIVE_INFINITY;
|
||||||
|
let lastProgressStage: CharacterDictionarySnapshotStageProgress['stage'] | null = null;
|
||||||
|
let generating: {
|
||||||
|
mediaId: number;
|
||||||
|
mediaTitle: string;
|
||||||
|
startedAt: number;
|
||||||
|
detail: string | null;
|
||||||
|
} | null = null;
|
||||||
|
let imageRate: { startedAt: number; startCompleted: number } | null = null;
|
||||||
|
|
||||||
|
const emitGeneratingStatus = (): void => {
|
||||||
|
if (!generating) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
deps.onSyncStatus?.({
|
||||||
|
phase: 'generating',
|
||||||
|
mediaId: generating.mediaId,
|
||||||
|
mediaTitle: generating.mediaTitle,
|
||||||
|
message: buildGeneratingMessage(
|
||||||
|
generating.mediaTitle,
|
||||||
|
joinGeneratingDetail(generating.detail, deps.now() - generating.startedAt),
|
||||||
|
),
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
// Image downloads are serial and evenly paced, so the observed rate predicts the tail well.
|
||||||
|
const estimateRemainingImageMs = (
|
||||||
|
progress: CharacterDictionarySnapshotStageProgress,
|
||||||
|
nowMs: number,
|
||||||
|
): number | null => {
|
||||||
|
if (progress.stage !== 'images' || progress.total === null || imageRate === null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const done = progress.completed - imageRate.startCompleted;
|
||||||
|
const elapsedMs = nowMs - imageRate.startedAt;
|
||||||
|
const remaining = progress.total - progress.completed;
|
||||||
|
if (done <= 0 || elapsedMs <= 0 || remaining <= 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return Math.round((elapsedMs / done) * remaining);
|
||||||
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
deps.logInfo?.('[dictionary:auto-sync] syncing current anime snapshot');
|
deps.logInfo?.('[dictionary:auto-sync] syncing current anime snapshot');
|
||||||
const snapshot = await deps.getOrCreateCurrentSnapshot(undefined, {
|
const snapshot = await withHeartbeat(
|
||||||
onChecking: ({ mediaId, mediaTitle }) => {
|
() =>
|
||||||
currentMediaId = mediaId;
|
deps.getOrCreateCurrentSnapshot(undefined, {
|
||||||
currentMediaTitle = mediaTitle;
|
onChecking: ({ mediaId, mediaTitle }) => {
|
||||||
activeCurrentMediaId = mediaId;
|
currentMediaId = mediaId;
|
||||||
deps.onSyncStatus?.({
|
currentMediaTitle = mediaTitle;
|
||||||
phase: 'checking',
|
activeCurrentMediaId = mediaId;
|
||||||
mediaId,
|
deps.onSyncStatus?.({
|
||||||
mediaTitle,
|
phase: 'checking',
|
||||||
message: buildCheckingMessage(mediaTitle),
|
mediaId,
|
||||||
});
|
mediaTitle,
|
||||||
},
|
message: buildCheckingMessage(mediaTitle),
|
||||||
onGenerating: ({ mediaId, mediaTitle }) => {
|
});
|
||||||
currentMediaId = mediaId;
|
},
|
||||||
currentMediaTitle = mediaTitle;
|
onGenerating: ({ mediaId, mediaTitle }) => {
|
||||||
activeCurrentMediaId = mediaId;
|
currentMediaId = mediaId;
|
||||||
deps.onSyncStatus?.({
|
currentMediaTitle = mediaTitle;
|
||||||
phase: 'generating',
|
activeCurrentMediaId = mediaId;
|
||||||
mediaId,
|
lastProgressAt = Number.NEGATIVE_INFINITY;
|
||||||
mediaTitle,
|
lastProgressStage = null;
|
||||||
message: buildGeneratingMessage(mediaTitle),
|
imageRate = null;
|
||||||
});
|
generating = { mediaId, mediaTitle, startedAt: deps.now(), detail: null };
|
||||||
},
|
emitGeneratingStatus();
|
||||||
});
|
},
|
||||||
|
// Long-running work (AniList character pages, then one image download per character
|
||||||
|
// and voice actor, then MeCab name splits) reports counts so the notification shows
|
||||||
|
// movement instead of a frozen "Generating..." for the minutes a large series takes.
|
||||||
|
onGenerateProgress: (progress) => {
|
||||||
|
const nowMs = deps.now();
|
||||||
|
if (!generating) {
|
||||||
|
generating = {
|
||||||
|
mediaId: progress.mediaId,
|
||||||
|
mediaTitle: progress.mediaTitle,
|
||||||
|
startedAt: nowMs,
|
||||||
|
detail: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (progress.stage === 'images' && imageRate === null) {
|
||||||
|
imageRate = { startedAt: nowMs, startCompleted: progress.completed };
|
||||||
|
}
|
||||||
|
// Stage changes and the last item of a stage always report; the throttle only thins
|
||||||
|
// out the run of identical-looking updates in between.
|
||||||
|
const isFinal = progress.total !== null && progress.completed >= progress.total;
|
||||||
|
const isStageChange = progress.stage !== lastProgressStage;
|
||||||
|
if (!isFinal && !isStageChange && nowMs - lastProgressAt < progressThrottleMs) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
lastProgressAt = nowMs;
|
||||||
|
lastProgressStage = progress.stage;
|
||||||
|
generating.mediaId = progress.mediaId;
|
||||||
|
generating.mediaTitle = progress.mediaTitle;
|
||||||
|
generating.detail = formatCharacterDictionaryProgressDetail(
|
||||||
|
progress,
|
||||||
|
estimateRemainingImageMs(progress, nowMs),
|
||||||
|
);
|
||||||
|
emitGeneratingStatus();
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
// Ticks even when a step stalls, so the message keeps moving while the counts do not.
|
||||||
|
() => emitGeneratingStatus(),
|
||||||
|
);
|
||||||
|
generating = null;
|
||||||
currentMediaId = snapshot.mediaId;
|
currentMediaId = snapshot.mediaId;
|
||||||
currentMediaTitle = snapshot.mediaTitle;
|
currentMediaTitle = snapshot.mediaTitle;
|
||||||
activeCurrentMediaId = snapshot.mediaId;
|
activeCurrentMediaId = snapshot.mediaId;
|
||||||
@@ -531,33 +664,61 @@ export function createCharacterDictionaryAutoSyncRuntimeService(
|
|||||||
mediaTitle: snapshot.mediaTitle,
|
mediaTitle: snapshot.mediaTitle,
|
||||||
message: buildImportingMessage(snapshot.mediaTitle),
|
message: buildImportingMessage(snapshot.mediaTitle),
|
||||||
});
|
});
|
||||||
if (existing !== null) {
|
await withHeartbeat(
|
||||||
await withOperationTimeout(
|
async () => {
|
||||||
`deleteYomitanDictionary(${dictionaryTitle})`,
|
const importTimeoutMs = resolveImportTimeoutMs(
|
||||||
deps.deleteYomitanDictionary(dictionaryTitle),
|
merged?.zipPath ?? path.join(dictionariesDir, 'merged.zip'),
|
||||||
);
|
);
|
||||||
}
|
if (existing !== null) {
|
||||||
if (merged === null) {
|
await withTimeout(
|
||||||
const existingMergedZipPath = path.join(dictionariesDir, 'merged.zip');
|
`deleteYomitanDictionary(${dictionaryTitle})`,
|
||||||
if (fs.existsSync(existingMergedZipPath)) {
|
deps.deleteYomitanDictionary(dictionaryTitle),
|
||||||
merged = {
|
importTimeoutMs,
|
||||||
zipPath: existingMergedZipPath,
|
);
|
||||||
revision,
|
}
|
||||||
dictionaryTitle,
|
if (merged === null) {
|
||||||
entryCount: snapshot.entryCount,
|
// The cached archive only stands in for the recorded state when its own index.json
|
||||||
};
|
// agrees. A stale or half-written ZIP would be imported under the wrong revision,
|
||||||
} else {
|
// and every later sync would then see a revision mismatch and re-import it.
|
||||||
merged = await deps.buildMergedDictionary(nextActiveMediaIdValues);
|
const existingMergedZipPath = path.join(dictionariesDir, 'merged.zip');
|
||||||
}
|
const existingMergedRevision = readDictionaryZipRevision(existingMergedZipPath);
|
||||||
}
|
if (existingMergedRevision === revision) {
|
||||||
deps.logInfo?.(`[dictionary:auto-sync] importing merged dictionary: ${merged.zipPath}`);
|
merged = {
|
||||||
const imported = await withOperationTimeout(
|
zipPath: existingMergedZipPath,
|
||||||
`importYomitanDictionary(${path.basename(merged.zipPath)})`,
|
revision,
|
||||||
deps.importYomitanDictionary(merged.zipPath),
|
dictionaryTitle,
|
||||||
|
entryCount: snapshot.entryCount,
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
deps.logInfo?.(
|
||||||
|
`[dictionary:auto-sync] cached merged ZIP unusable (revision ${existingMergedRevision ?? 'unreadable'}, expected ${revision}); rebuilding`,
|
||||||
|
);
|
||||||
|
merged = await deps.buildMergedDictionary(nextActiveMediaIdValues);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const mergedZipPath = merged.zipPath;
|
||||||
|
const mergedImportTimeoutMs = resolveImportTimeoutMs(mergedZipPath);
|
||||||
|
deps.logInfo?.(
|
||||||
|
`[dictionary:auto-sync] importing merged dictionary: ${mergedZipPath} (timeout ${mergedImportTimeoutMs}ms)`,
|
||||||
|
);
|
||||||
|
const imported = await withTimeout(
|
||||||
|
`importYomitanDictionary(${path.basename(mergedZipPath)})`,
|
||||||
|
deps.importYomitanDictionary(mergedZipPath),
|
||||||
|
mergedImportTimeoutMs,
|
||||||
|
);
|
||||||
|
if (!imported) {
|
||||||
|
throw new Error(`Failed to import dictionary ZIP: ${merged.zipPath}`);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
(elapsedMs) => {
|
||||||
|
deps.onSyncStatus?.({
|
||||||
|
phase: 'importing',
|
||||||
|
mediaId: snapshot.mediaId,
|
||||||
|
mediaTitle: snapshot.mediaTitle,
|
||||||
|
message: buildImportingMessage(snapshot.mediaTitle, elapsedMs),
|
||||||
|
});
|
||||||
|
},
|
||||||
);
|
);
|
||||||
if (!imported) {
|
|
||||||
throw new Error(`Failed to import dictionary ZIP: ${merged.zipPath}`);
|
|
||||||
}
|
|
||||||
changed = true;
|
changed = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ export interface StartupOsdSequencerDeps {
|
|||||||
getNotificationType?: () => NotificationType | undefined;
|
getNotificationType?: () => NotificationType | undefined;
|
||||||
showOsd: (message: string) => boolean | void;
|
showOsd: (message: string) => boolean | void;
|
||||||
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
|
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
|
||||||
showDesktopNotification?: (title: string, options: { body?: string }) => void;
|
showDesktopNotification?: (title: string, options: { body?: string; replaceId?: string }) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface StartupStatusNotificationOptions {
|
interface StartupStatusNotificationOptions {
|
||||||
@@ -62,7 +62,11 @@ export function createStartupOsdSequencer(deps: StartupOsdSequencerDeps): {
|
|||||||
shown = deps.showOsd(options.message) !== false || shown;
|
shown = deps.showOsd(options.message) !== false || shown;
|
||||||
}
|
}
|
||||||
if (options.desktop !== false && shouldShowDesktop(type)) {
|
if (options.desktop !== false && shouldShowDesktop(type)) {
|
||||||
deps.showDesktopNotification?.('SubMiner', { body: options.message });
|
// Each startup lane keeps one live desktop notification instead of one per update.
|
||||||
|
deps.showDesktopNotification?.('SubMiner', {
|
||||||
|
body: options.message,
|
||||||
|
replaceId: options.id,
|
||||||
|
});
|
||||||
shown = true;
|
shown = true;
|
||||||
}
|
}
|
||||||
return shown;
|
return shown;
|
||||||
|
|||||||
@@ -148,6 +148,145 @@ function createEndOfCentralDirectory(
|
|||||||
return end;
|
return end;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const LOCAL_FILE_HEADER_SIGNATURE = 0x04034b50;
|
||||||
|
const LOCAL_FILE_HEADER_SIZE = 30;
|
||||||
|
const END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054b50;
|
||||||
|
const END_OF_CENTRAL_DIRECTORY_SIZE = 22;
|
||||||
|
const CENTRAL_FILE_HEADER_SIGNATURE = 0x02014b50;
|
||||||
|
const CENTRAL_FILE_HEADER_SIZE = 46;
|
||||||
|
// 65535 entries with names of a few dozen bytes stay far under this; the cap only stops a corrupt
|
||||||
|
// record length from asking for an allocation the size of the archive.
|
||||||
|
const MAX_CENTRAL_DIRECTORY_BYTES = 16 * 1024 * 1024;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Walks every declared central-directory record, checking each signature and keeping the
|
||||||
|
* variable-length name/extra/comment fields inside the directory. The walk has to land exactly on
|
||||||
|
* the end of the directory, so a record that was overwritten in place fails even though the file
|
||||||
|
* kept its size.
|
||||||
|
*/
|
||||||
|
function isCentralDirectoryIntact(
|
||||||
|
fd: number,
|
||||||
|
centralStart: number,
|
||||||
|
centralSize: number,
|
||||||
|
entryCount: number,
|
||||||
|
): boolean {
|
||||||
|
if (centralSize === 0 || centralSize > MAX_CENTRAL_DIRECTORY_BYTES) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const central = Buffer.alloc(centralSize);
|
||||||
|
if (fs.readSync(fd, central, 0, centralSize, centralStart) !== centralSize) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let cursor = 0;
|
||||||
|
for (let index = 0; index < entryCount; index += 1) {
|
||||||
|
if (cursor + CENTRAL_FILE_HEADER_SIZE > centralSize) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (central.readUInt32LE(cursor) !== CENTRAL_FILE_HEADER_SIGNATURE) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const nameLength = central.readUInt16LE(cursor + 28);
|
||||||
|
const extraLength = central.readUInt16LE(cursor + 30);
|
||||||
|
const commentLength = central.readUInt16LE(cursor + 32);
|
||||||
|
cursor += CENTRAL_FILE_HEADER_SIZE + nameLength + extraLength + commentLength;
|
||||||
|
if (cursor > centralSize) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cursor === centralSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start of the central directory, or null when the archive is not a complete one of ours. The
|
||||||
|
* end-of-central-directory record is written last, so finding an intact one is what separates a
|
||||||
|
* finished archive from a half-written one.
|
||||||
|
*/
|
||||||
|
function readCentralDirectoryStart(fd: number, fileSize: number): number | null {
|
||||||
|
if (fileSize < END_OF_CENTRAL_DIRECTORY_SIZE) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const end = Buffer.alloc(END_OF_CENTRAL_DIRECTORY_SIZE);
|
||||||
|
const endOffset = fileSize - END_OF_CENTRAL_DIRECTORY_SIZE;
|
||||||
|
if (fs.readSync(fd, end, 0, end.length, endOffset) !== end.length) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
// writeStoredZip never writes an archive comment, so the record is exactly the last 22 bytes.
|
||||||
|
if (end.readUInt32LE(0) !== END_OF_CENTRAL_DIRECTORY_SIGNATURE || end.readUInt16LE(20) !== 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const entryCount = end.readUInt16LE(10);
|
||||||
|
if (entryCount === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const centralSize = end.readUInt32LE(12);
|
||||||
|
const centralStart = end.readUInt32LE(16);
|
||||||
|
if (centralStart + centralSize !== endOffset) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (!isCentralDirectoryIntact(fd, centralStart, centralSize, entryCount)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return centralStart;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads the first entry of an archive written by {@link writeStoredZip}: every entry is stored
|
||||||
|
* uncompressed with no extra field and no data descriptor, so the leading local header is enough.
|
||||||
|
* Returns null for anything it does not recognize, so callers treat a corrupt, truncated, or
|
||||||
|
* foreign archive the same as a missing one.
|
||||||
|
*/
|
||||||
|
export function readStoredZipFirstFile(zipPath: string): StoredZipFile | null {
|
||||||
|
let fd: number;
|
||||||
|
try {
|
||||||
|
fd = fs.openSync(zipPath, 'r');
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const fileSize = fs.fstatSync(fd).size;
|
||||||
|
const centralStart = readCentralDirectoryStart(fd, fileSize);
|
||||||
|
if (centralStart === null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const header = Buffer.alloc(LOCAL_FILE_HEADER_SIZE);
|
||||||
|
if (fs.readSync(fd, header, 0, header.length, 0) !== header.length) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (header.readUInt32LE(0) !== LOCAL_FILE_HEADER_SIGNATURE) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
// Compression method 0 (stored) is the only thing writeStoredZip emits.
|
||||||
|
if (header.readUInt16LE(8) !== 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const entrySize = header.readUInt32LE(18);
|
||||||
|
const nameLength = header.readUInt16LE(26);
|
||||||
|
const extraLength = header.readUInt16LE(28);
|
||||||
|
const dataOffset = LOCAL_FILE_HEADER_SIZE + nameLength + extraLength;
|
||||||
|
if (dataOffset + entrySize > centralStart) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const name = Buffer.alloc(nameLength);
|
||||||
|
if (
|
||||||
|
nameLength > 0 &&
|
||||||
|
fs.readSync(fd, name, 0, nameLength, LOCAL_FILE_HEADER_SIZE) !== nameLength
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const data = Buffer.alloc(entrySize);
|
||||||
|
if (entrySize > 0 && fs.readSync(fd, data, 0, entrySize, dataOffset) !== entrySize) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return { name: name.toString('utf8'), data };
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
fs.closeSync(fd);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function writeBuffer(fd: number, buffer: Buffer): void {
|
function writeBuffer(fd: number, buffer: Buffer): void {
|
||||||
let written = 0;
|
let written = 0;
|
||||||
while (written < buffer.length) {
|
while (written < buffer.length) {
|
||||||
|
|||||||
Reference in New Issue
Block a user