diff --git a/src/main/character-dictionary-runtime/zip.test.ts b/src/main/character-dictionary-runtime/zip.test.ts index b7f8ab40..2316a0ee 100644 --- a/src/main/character-dictionary-runtime/zip.test.ts +++ b/src/main/character-dictionary-runtime/zip.test.ts @@ -3,7 +3,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import test from 'node:test'; -import { buildDictionaryZip } from './zip'; +import { buildDictionaryZip, readDictionaryZipRevision } from './zip'; import type { CharacterDictionaryTermEntry } from './types'; function makeTempDir(): string { @@ -105,3 +105,38 @@ test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', () cleanupDir(tempDir); } }); + +test('readDictionaryZipRevision reads the built revision and rejects foreign archives', () => { + const dir = makeTempDir(); + try { + const zipPath = path.join(dir, 'merged.zip'); + buildDictionaryZip( + zipPath, + 'SubMiner Character Dictionary', + 'Character names', + 'rev-42', + [ + { + term: 'ルフィ', + reading: 'ルフィ', + role: 'main', + glossary: [], + } as unknown as CharacterDictionaryTermEntry, + ], + [], + ); + + assert.equal(readDictionaryZipRevision(zipPath), 'rev-42'); + assert.equal(readDictionaryZipRevision(path.join(dir, 'missing.zip')), null); + + const truncatedPath = path.join(dir, 'truncated.zip'); + fs.writeFileSync(truncatedPath, fs.readFileSync(zipPath).subarray(0, 40)); + assert.equal(readDictionaryZipRevision(truncatedPath), null); + + const foreignPath = path.join(dir, 'foreign.zip'); + fs.writeFileSync(foreignPath, Buffer.from('not a zip at all', 'utf8')); + assert.equal(readDictionaryZipRevision(foreignPath), null); + } finally { + cleanupDir(dir); + } +}); diff --git a/src/main/character-dictionary-runtime/zip.ts b/src/main/character-dictionary-runtime/zip.ts index a4b92458..759a7ce6 100644 --- a/src/main/character-dictionary-runtime/zip.ts +++ b/src/main/character-dictionary-runtime/zip.ts @@ -1,5 +1,5 @@ import * as path from 'path'; -import { writeStoredZip } from '../../shared/stored-zip'; +import { readStoredZipFirstFile, writeStoredZip } from '../../shared/stored-zip'; import { ensureDir } from './fs-utils'; import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } from './types'; @@ -31,6 +31,23 @@ function createTagBank(): Array<[string, string, number, string, number]> { ]; } +/** + * Revision recorded inside a built dictionary ZIP, or null when the archive is missing, truncated, + * or not one of ours. `index.json` is always the first entry written by {@link buildDictionaryZip}. + */ +export function readDictionaryZipRevision(zipPath: string): string | null { + const firstFile = readStoredZipFirstFile(zipPath); + if (!firstFile || firstFile.name !== 'index.json') { + return null; + } + try { + const index = JSON.parse(firstFile.data.toString('utf8')) as { revision?: unknown }; + return typeof index.revision === 'string' && index.revision.length > 0 ? index.revision : null; + } catch { + return null; + } +} + export function buildDictionaryZip( outputPath: string, dictionaryTitle: string, diff --git a/src/main/runtime/character-dictionary-auto-sync-messages.ts b/src/main/runtime/character-dictionary-auto-sync-messages.ts new file mode 100644 index 00000000..b5edcae8 --- /dev/null +++ b/src/main/runtime/character-dictionary-auto-sync-messages.ts @@ -0,0 +1,90 @@ +import type { CharacterDictionarySnapshotStageProgress } from '../character-dictionary-runtime'; + +export function buildSyncingMessage(mediaTitle: string): string { + return `Updating character dictionary for ${mediaTitle}...`; +} + +export function buildCheckingMessage(mediaTitle: string): string { + return `Checking character dictionary for ${mediaTitle}...`; +} + +export function buildGeneratingMessage(mediaTitle: string, detail?: string): string { + return detail + ? `Generating character dictionary for ${mediaTitle} (${detail})...` + : `Generating character dictionary for ${mediaTitle}...`; +} + +export function formatCharacterDictionaryProgressDetail( + progress: CharacterDictionarySnapshotStageProgress, + remainingMs: number | null, +): string { + if (progress.stage === 'saving') { + return 'saving snapshot'; + } + if (progress.stage === 'names') { + return progress.total !== null && progress.total > 0 + ? `name ${progress.completed}/${progress.total}` + : `${progress.completed} names`; + } + if (progress.stage === 'images') { + const counted = + progress.total !== null && progress.total > 0 + ? `image ${progress.completed}/${progress.total}` + : `${progress.completed} images`; + return remainingMs !== null + ? `${counted}, ~${formatRemainingDuration(remainingMs)} left` + : counted; + } + const page = typeof progress.page === 'number' ? `page ${progress.page}, ` : ''; + return `${page}${progress.completed} characters`; +} + +export function formatElapsedDuration(elapsedMs: number): string { + const totalSeconds = Math.max(0, Math.floor(elapsedMs / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return minutes > 0 ? `${minutes}m ${String(seconds).padStart(2, '0')}s` : `${seconds}s`; +} + +/** Coarser than the elapsed clock: an estimate that ticks every second reads as precision it lacks. */ +export function formatRemainingDuration(remainingMs: number): string { + const totalSeconds = Math.max(1, Math.round(remainingMs / 1000)); + if (totalSeconds >= 60) { + return `${Math.max(1, Math.round(totalSeconds / 60))}m`; + } + return `${Math.max(5, Math.ceil(totalSeconds / 5) * 5)}s`; +} + +/** + * The elapsed clock is the part that proves the app is alive: a stalled network fetch freezes the + * counts, but the clock keeps moving. + */ +export function joinGeneratingDetail(detail: string | null, elapsedMs: number): string | undefined { + const parts = [detail, elapsedMs >= 5_000 ? formatElapsedDuration(elapsedMs) : null].filter( + (part): part is string => typeof part === 'string' && part.length > 0, + ); + return parts.length > 0 ? parts.join(' · ') : undefined; +} + +export function buildImportingMessage(mediaTitle: string, elapsedMs?: number): string { + const elapsed = + typeof elapsedMs === 'number' && elapsedMs >= 1000 + ? ` (${formatElapsedDuration(elapsedMs)})` + : ''; + return `Importing character dictionary for ${mediaTitle}${elapsed}...`; +} + +export function buildBuildingMessage(mediaTitle: string): string { + return `Building character dictionary for ${mediaTitle}...`; +} + +export function buildReadyMessage(mediaTitle: string): string { + return `Character dictionary ready for ${mediaTitle}`; +} + +export function buildFailedMessage(mediaTitle: string | null, errorMessage: string): string { + if (mediaTitle) { + return `Character dictionary sync failed for ${mediaTitle}: ${errorMessage}`; + } + return `Character dictionary sync failed: ${errorMessage}`; +} diff --git a/src/main/runtime/character-dictionary-auto-sync.test.ts b/src/main/runtime/character-dictionary-auto-sync.test.ts index 329af5dd..35b34ace 100644 --- a/src/main/runtime/character-dictionary-auto-sync.test.ts +++ b/src/main/runtime/character-dictionary-auto-sync.test.ts @@ -3,6 +3,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import test from 'node:test'; +import { buildDictionaryZip } from '../character-dictionary-runtime/zip'; import { createCharacterDictionaryAutoSyncRuntimeService, getCharacterDictionaryManagerSnapshot, @@ -375,7 +376,14 @@ test('auto sync reimports existing merged zip without rebuilding on unchanged re const userDataPath = makeTempDir(); const dictionariesDir = path.join(userDataPath, 'character-dictionaries'); fs.mkdirSync(dictionariesDir, { recursive: true }); - fs.writeFileSync(path.join(dictionariesDir, 'merged.zip'), 'cached-zip', 'utf8'); + buildDictionaryZip( + path.join(dictionariesDir, 'merged.zip'), + 'SubMiner Character Dictionary', + 'Character names', + 'rev-7', + [{ term: 'フリーレン', reading: 'フリーレン', role: 'main', glossary: [] } as never], + [], + ); const mergedBuilds: number[][] = []; const imports: string[] = []; let importedRevision: string | null = null; @@ -1077,14 +1085,15 @@ test('auto sync scales the import timeout with the merged dictionary size', asyn : [], importYomitanDictionary: async () => { // Far longer than the quick-operation budget, well inside the size-scaled one. - await new Promise((resolve) => setTimeout(resolve, 60)); + await new Promise((resolve) => setTimeout(resolve, 400)); importedRevision = 'rev-21'; return true; }, deleteYomitanDictionary: async () => true, upsertYomitanDictionarySettings: async () => true, now: () => 1000, - operationTimeoutMs: 5, + // Comfortable for the stubs that resolve immediately, still far under the import's 400ms. + operationTimeoutMs: 100, dictionaryImportTimeoutBaseMs: 20, onSyncStatus: (event) => { events.push({ phase: event.phase, message: event.message }); @@ -1373,3 +1382,65 @@ test('auto sync keeps the generating clock ticking when a stage stalls', async ( await syncPromise; assert.equal(events.at(-1)?.phase, 'ready'); }); + +test('auto sync rebuilds instead of importing a cached merged ZIP with a mismatched revision', async () => { + const userDataPath = makeTempDir(); + const dictionariesDir = path.join(userDataPath, 'character-dictionaries'); + fs.mkdirSync(dictionariesDir, { recursive: true }); + const statePath = path.join(dictionariesDir, 'auto-sync-state.json'); + fs.writeFileSync( + statePath, + JSON.stringify({ + activeMediaIds: ['7 - Frieren'], + mergedRevision: 'rev-7', + mergedDictionaryTitle: 'SubMiner Character Dictionary', + }), + 'utf8', + ); + // Left over from an interrupted run: the archive on disk is not the revision state recorded. + buildDictionaryZip( + path.join(dictionariesDir, 'merged.zip'), + 'SubMiner Character Dictionary', + 'Character names', + 'rev-stale', + [{ term: 'フリーレン', reading: 'フリーレン', role: 'main', glossary: [] } as never], + [], + ); + const mergedBuilds: number[][] = []; + const imports: string[] = []; + + const runtime = createCharacterDictionaryAutoSyncRuntimeService({ + userDataPath, + getConfig: () => ({ enabled: true, maxLoaded: 3, profileScope: 'all' }), + getOrCreateCurrentSnapshot: async () => ({ + mediaId: 7, + mediaTitle: 'Frieren', + entryCount: 100, + fromCache: true, + updatedAt: 1000, + }), + buildMergedDictionary: async (mediaIds) => { + mergedBuilds.push([...mediaIds]); + return { + zipPath: '/tmp/rebuilt-merged.zip', + revision: 'rev-7', + dictionaryTitle: 'SubMiner Character Dictionary', + entryCount: 100, + }; + }, + // Yomitan does not have the dictionary, so the sync has to import despite the cached state. + getYomitanDictionaryInfo: async () => [], + importYomitanDictionary: async (zipPath) => { + imports.push(zipPath); + return true; + }, + deleteYomitanDictionary: async () => true, + upsertYomitanDictionarySettings: async () => true, + now: () => 1000, + }); + + await runtime.runSyncNow(); + + assert.deepEqual(mergedBuilds, [[7]]); + assert.deepEqual(imports, ['/tmp/rebuilt-merged.zip']); +}); diff --git a/src/main/runtime/character-dictionary-auto-sync.ts b/src/main/runtime/character-dictionary-auto-sync.ts index 7801128a..82e27ec6 100644 --- a/src/main/runtime/character-dictionary-auto-sync.ts +++ b/src/main/runtime/character-dictionary-auto-sync.ts @@ -1,6 +1,17 @@ import * as fs from 'fs'; import * as path from 'path'; import { ensureDir } from '../../shared/fs-utils'; +import { + buildBuildingMessage, + buildCheckingMessage, + buildFailedMessage, + buildGeneratingMessage, + buildImportingMessage, + buildReadyMessage, + formatCharacterDictionaryProgressDetail, + joinGeneratingDetail, +} from './character-dictionary-auto-sync-messages'; +import { readDictionaryZipRevision } from '../character-dictionary-runtime/zip'; import type { AnilistCharacterDictionaryProfileScope } from '../../types'; import type { CharacterDictionarySnapshotProgressCallbacks, @@ -358,95 +369,6 @@ function sameMembership(left: number[], right: number[]): boolean { return arraysEqual(leftSorted, rightSorted); } -function buildSyncingMessage(mediaTitle: string): string { - return `Updating character dictionary for ${mediaTitle}...`; -} - -function buildCheckingMessage(mediaTitle: string): string { - return `Checking character dictionary for ${mediaTitle}...`; -} - -function buildGeneratingMessage(mediaTitle: string, detail?: string): string { - return detail - ? `Generating character dictionary for ${mediaTitle} (${detail})...` - : `Generating character dictionary for ${mediaTitle}...`; -} - -function formatCharacterDictionaryProgressDetail( - progress: CharacterDictionarySnapshotStageProgress, - remainingMs: number | null, -): string { - if (progress.stage === 'saving') { - return 'saving snapshot'; - } - if (progress.stage === 'names') { - return progress.total !== null && progress.total > 0 - ? `name ${progress.completed}/${progress.total}` - : `${progress.completed} names`; - } - if (progress.stage === 'images') { - const counted = - progress.total !== null && progress.total > 0 - ? `image ${progress.completed}/${progress.total}` - : `${progress.completed} images`; - return remainingMs !== null - ? `${counted}, ~${formatRemainingDuration(remainingMs)} left` - : counted; - } - const page = typeof progress.page === 'number' ? `page ${progress.page}, ` : ''; - return `${page}${progress.completed} characters`; -} - -function formatElapsedDuration(elapsedMs: number): string { - const totalSeconds = Math.max(0, Math.floor(elapsedMs / 1000)); - const minutes = Math.floor(totalSeconds / 60); - const seconds = totalSeconds % 60; - return minutes > 0 ? `${minutes}m ${String(seconds).padStart(2, '0')}s` : `${seconds}s`; -} - -/** Coarser than the elapsed clock: an estimate that ticks every second reads as precision it lacks. */ -function formatRemainingDuration(remainingMs: number): string { - const totalSeconds = Math.max(1, Math.round(remainingMs / 1000)); - if (totalSeconds >= 60) { - return `${Math.max(1, Math.round(totalSeconds / 60))}m`; - } - return `${Math.max(5, Math.ceil(totalSeconds / 5) * 5)}s`; -} - -/** - * The elapsed clock is the part that proves the app is alive: a stalled network fetch freezes the - * counts, but the clock keeps moving. - */ -function joinGeneratingDetail(detail: string | null, elapsedMs: number): string | undefined { - const parts = [detail, elapsedMs >= 5_000 ? formatElapsedDuration(elapsedMs) : null].filter( - (part): part is string => typeof part === 'string' && part.length > 0, - ); - return parts.length > 0 ? parts.join(' · ') : undefined; -} - -function buildImportingMessage(mediaTitle: string, elapsedMs?: number): string { - const elapsed = - typeof elapsedMs === 'number' && elapsedMs >= 1000 - ? ` (${formatElapsedDuration(elapsedMs)})` - : ''; - return `Importing character dictionary for ${mediaTitle}${elapsed}...`; -} - -function buildBuildingMessage(mediaTitle: string): string { - return `Building character dictionary for ${mediaTitle}...`; -} - -function buildReadyMessage(mediaTitle: string): string { - return `Character dictionary ready for ${mediaTitle}`; -} - -function buildFailedMessage(mediaTitle: string | null, errorMessage: string): string { - if (mediaTitle) { - return `Character dictionary sync failed for ${mediaTitle}: ${errorMessage}`; - } - return `Character dictionary sync failed: ${errorMessage}`; -} - export function createCharacterDictionaryAutoSyncRuntimeService( deps: CharacterDictionaryAutoSyncRuntimeDeps, ): { @@ -755,8 +677,12 @@ export function createCharacterDictionaryAutoSyncRuntimeService( ); } if (merged === null) { + // The cached archive only stands in for the recorded state when its own index.json + // agrees. A stale or half-written ZIP would be imported under the wrong revision, + // and every later sync would then see a revision mismatch and re-import it. const existingMergedZipPath = path.join(dictionariesDir, 'merged.zip'); - if (fs.existsSync(existingMergedZipPath)) { + const existingMergedRevision = readDictionaryZipRevision(existingMergedZipPath); + if (existingMergedRevision === revision) { merged = { zipPath: existingMergedZipPath, revision, @@ -764,6 +690,9 @@ export function createCharacterDictionaryAutoSyncRuntimeService( entryCount: snapshot.entryCount, }; } else { + deps.logInfo?.( + `[dictionary:auto-sync] cached merged ZIP unusable (revision ${existingMergedRevision ?? 'unreadable'}, expected ${revision}); rebuilding`, + ); merged = await deps.buildMergedDictionary(nextActiveMediaIdValues); } } diff --git a/src/shared/stored-zip.ts b/src/shared/stored-zip.ts index 9c10ea8a..90d66eec 100644 --- a/src/shared/stored-zip.ts +++ b/src/shared/stored-zip.ts @@ -148,6 +148,62 @@ function createEndOfCentralDirectory( return end; } +const LOCAL_FILE_HEADER_SIGNATURE = 0x04034b50; +const LOCAL_FILE_HEADER_SIZE = 30; + +/** + * Reads the first entry of an archive written by {@link writeStoredZip}: every entry is stored + * uncompressed with no extra field and no data descriptor, so the leading local header is enough. + * Returns null for anything it does not recognize, so callers treat a corrupt or foreign archive + * the same as a missing one. + */ +export function readStoredZipFirstFile(zipPath: string): StoredZipFile | null { + let fd: number; + try { + fd = fs.openSync(zipPath, 'r'); + } catch { + return null; + } + + try { + const fileSize = fs.fstatSync(fd).size; + const header = Buffer.alloc(LOCAL_FILE_HEADER_SIZE); + if (fs.readSync(fd, header, 0, header.length, 0) !== header.length) { + return null; + } + if (header.readUInt32LE(0) !== LOCAL_FILE_HEADER_SIGNATURE) { + return null; + } + // Compression method 0 (stored) is the only thing writeStoredZip emits. + if (header.readUInt16LE(8) !== 0) { + return null; + } + const entrySize = header.readUInt32LE(18); + const nameLength = header.readUInt16LE(26); + const extraLength = header.readUInt16LE(28); + const dataOffset = LOCAL_FILE_HEADER_SIZE + nameLength + extraLength; + if (dataOffset + entrySize > fileSize) { + return null; + } + const name = Buffer.alloc(nameLength); + if ( + nameLength > 0 && + fs.readSync(fd, name, 0, nameLength, LOCAL_FILE_HEADER_SIZE) !== nameLength + ) { + return null; + } + const data = Buffer.alloc(entrySize); + if (entrySize > 0 && fs.readSync(fd, data, 0, entrySize, dataOffset) !== entrySize) { + return null; + } + return { name: name.toString('utf8'), data }; + } catch { + return null; + } finally { + fs.closeSync(fd); + } +} + function writeBuffer(fd: number, buffer: Buffer): void { let written = 0; while (written < buffer.length) {