fix(dictionary): prevent freezes and restore AppImage notifications (#205)

src/shared/stored-zip.test.ts

@@ -0,0 +1,74 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+import { readStoredZipFirstFile, writeStoredZip, writeStoredZipAsync } from './stored-zip';
+
+function makeTempDir(): string {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-stored-zip-'));
+}
+
+function readEntries(zipPath: string): Map<string, Buffer> {
+  const archive = fs.readFileSync(zipPath);
+  const entries = new Map<string, Buffer>();
+  let cursor = 0;
+
+  while (cursor + 4 <= archive.length) {
+    const signature = archive.readUInt32LE(cursor);
+    if (signature === 0x02014b50 || signature === 0x06054b50) {
+      break;
+    }
+    assert.equal(signature, 0x04034b50, `unexpected local file header at offset ${cursor}`);
+    const size = archive.readUInt32LE(cursor + 18);
+    const nameLength = archive.readUInt16LE(cursor + 26);
+    const extraLength = archive.readUInt16LE(cursor + 28);
+    const nameStart = cursor + 30;
+    const dataStart = nameStart + nameLength + extraLength;
+    entries.set(
+      archive.subarray(nameStart, nameStart + nameLength).toString('utf8'),
+      Buffer.from(archive.subarray(dataStart, dataStart + size)),
+    );
+    cursor = dataStart + size;
+  }
+
+  return entries;
+}
+
+// The async writer yields on a byte budget between entries, so an entry that is itself larger than
+// that budget is the case where the accounting could drift: the offsets, CRCs, and central
+// directory all have to come out identical to the synchronous writer.
+test('writeStoredZipAsync writes a correct archive when one entry exceeds the yield budget', async () => {
+  const dir = makeTempDir();
+  try {
+    // Comfortably past the writer's 8MB yield budget.
+    const oversized = Buffer.alloc(10 * 1024 * 1024);
+    for (let i = 0; i < oversized.length; i += 1) {
+      oversized[i] = i % 251;
+    }
+    const files = [
+      { name: 'index.json', data: Buffer.from('{"revision":"rev-1"}', 'utf8') },
+      { name: 'big.bin', data: oversized },
+      { name: 'after.txt', data: Buffer.from('written after the oversized entry', 'utf8') },
+    ];
+
+    const asyncPath = path.join(dir, 'async.zip');
+    const syncPath = path.join(dir, 'sync.zip');
+    const asyncResult = await writeStoredZipAsync(asyncPath, files);
+    const syncResult = writeStoredZip(syncPath, files);
+
+    assert.equal(asyncResult.entryCount, 3);
+    assert.deepEqual(asyncResult, syncResult);
+    // Byte-identical to the synchronous writer: yielding mid-archive changed no offset or CRC.
+    assert.ok(fs.readFileSync(asyncPath).equals(fs.readFileSync(syncPath)));
+
+    const entries = readEntries(asyncPath);
+    assert.deepEqual([...entries.keys()], ['index.json', 'big.bin', 'after.txt']);
+    assert.ok(entries.get('big.bin')!.equals(oversized));
+    assert.equal(entries.get('after.txt')!.toString('utf8'), 'written after the oversized entry');
+    // The trailing records still parse, which is what proves the archive is complete.
+    assert.equal(readStoredZipFirstFile(asyncPath)?.name, 'index.json');
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});

src/shared/stored-zip.ts

@@ -1,4 +1,5 @@
 import * as fs from 'fs';
+import * as zlib from 'zlib';
 
 type ZipEntry = {
   name: string;
@@ -36,10 +37,17 @@ const CRC32_TABLE = (() => {
   return table;
 })();
 
+// Native CRC32 (Node >= 20.15) runs at native throughput, which matters for the multi-hundred-MB
+// dictionary archives; the table loop stays as a fallback for runtimes without it.
+const nativeCrc32 = (zlib as { crc32?: (data: Uint8Array, value?: number) => number }).crc32;
+
 function crc32(data: Buffer): number {
+  if (typeof nativeCrc32 === 'function') {
+    return nativeCrc32(data) >>> 0;
+  }
   let crc = 0xffffffff;
-  for (const byte of data) {
-    crc = CRC32_TABLE[(crc ^ byte) & 0xff]! ^ (crc >>> 8);
+  for (let i = 0; i < data.length; i += 1) {
+    crc = CRC32_TABLE[(crc ^ data[i]!) & 0xff]! ^ (crc >>> 8);
   }
   return (crc ^ 0xffffffff) >>> 0;
 }
@@ -294,59 +302,70 @@ function writeBuffer(fd: number, buffer: Buffer): void {
   }
 }
 
+type ZipWriteState = {
+  entries: ZipEntry[];
+  offset: number;
+};
+
+/** Appends one stored entry (local header + data) and returns the bytes written. */
+function appendStoredZipFile(fd: number, state: ZipWriteState, file: StoredZipFile): number {
+  const fileName = Buffer.from(file.name, 'utf8');
+  const fileSize = file.data.length;
+  if (fileName.length > ZIP32_MAX_UINT16) {
+    throw new RangeError(`ZIP entry name too long: ${file.name}`);
+  }
+  if (fileSize > ZIP32_MAX_UINT32) {
+    throw new RangeError(`ZIP entry too large for ZIP32: ${file.name}`);
+  }
+  if (state.offset > ZIP32_MAX_UINT32) {
+    throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
+  }
+  const fileCrc32 = crc32(file.data);
+  const localHeader = createLocalFileHeader(fileName, fileCrc32, fileSize);
+  const nextOffset = state.offset + localHeader.length + fileSize;
+  if (nextOffset > ZIP32_MAX_UINT32) {
+    throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
+  }
+  writeBuffer(fd, localHeader);
+  writeBuffer(fd, file.data);
+  state.entries.push({
+    name: file.name,
+    crc32: fileCrc32,
+    size: fileSize,
+    localHeaderOffset: state.offset,
+  });
+  const written = nextOffset - state.offset;
+  state.offset = nextOffset;
+  return written;
+}
+
+function finishStoredZip(fd: number, state: ZipWriteState): void {
+  const centralStart = state.offset;
+  if (centralStart > ZIP32_MAX_UINT32) {
+    throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
+  }
+  for (const entry of state.entries) {
+    const centralHeader = createCentralDirectoryHeader(entry);
+    writeBuffer(fd, centralHeader);
+    state.offset += centralHeader.length;
+  }
+
+  const centralSize = state.offset - centralStart;
+  writeBuffer(fd, createEndOfCentralDirectory(state.entries.length, centralSize, centralStart));
+}
+
 export function writeStoredZip(
   outputPath: string,
   files: Iterable<StoredZipFile>,
 ): { entryCount: number } {
-  const entries: ZipEntry[] = [];
-  let offset = 0;
+  const state: ZipWriteState = { entries: [], offset: 0 };
   const fd = fs.openSync(outputPath, 'w');
 
   try {
     for (const file of files) {
-      const fileName = Buffer.from(file.name, 'utf8');
-      const fileSize = file.data.length;
-      if (fileName.length > ZIP32_MAX_UINT16) {
-        throw new RangeError(`ZIP entry name too long: ${file.name}`);
-      }
-      if (fileSize > ZIP32_MAX_UINT32) {
-        throw new RangeError(`ZIP entry too large for ZIP32: ${file.name}`);
-      }
-      if (offset > ZIP32_MAX_UINT32) {
-        throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
-      }
-      const fileCrc32 = crc32(file.data);
-      const localHeader = createLocalFileHeader(fileName, fileCrc32, fileSize);
-      const nextOffset = offset + localHeader.length + fileSize;
-      if (nextOffset > ZIP32_MAX_UINT32) {
-        throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
-      }
-      writeBuffer(fd, localHeader);
-      writeBuffer(fd, file.data);
-      entries.push({
-        name: file.name,
-        crc32: fileCrc32,
-        size: fileSize,
-        localHeaderOffset: offset,
-      });
-      if (nextOffset > ZIP32_MAX_UINT32) {
-        throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
-      }
-      offset = nextOffset;
+      appendStoredZipFile(fd, state, file);
     }
-
-    const centralStart = offset;
-    if (centralStart > ZIP32_MAX_UINT32) {
-      throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
-    }
-    for (const entry of entries) {
-      const centralHeader = createCentralDirectoryHeader(entry);
-      writeBuffer(fd, centralHeader);
-      offset += centralHeader.length;
-    }
-
-    const centralSize = offset - centralStart;
-    writeBuffer(fd, createEndOfCentralDirectory(entries.length, centralSize, centralStart));
+    finishStoredZip(fd, state);
   } catch (error) {
     fs.closeSync(fd);
     fs.rmSync(outputPath, { force: true });
@@ -354,5 +373,42 @@ export function writeStoredZip(
   }
 
   fs.closeSync(fd);
-  return { entryCount: entries.length };
+  return { entryCount: state.entries.length };
+}
+
+// Yielding roughly every 8MB keeps individual event-loop blocks in the low tens of milliseconds
+// while adding a negligible number of macrotask hops even for the largest merged dictionary.
+const ASYNC_ZIP_YIELD_BYTE_BUDGET = 8 * 1024 * 1024;
+
+/**
+ * Same archive as {@link writeStoredZip}, written without starving the event loop: entry
+ * generation, CRC, and writes proceed in byte-budgeted slices with a macrotask yield in between.
+ * Multi-hundred-MB dictionary archives previously blocked the main process long enough for the
+ * compositor to declare the app unresponsive.
+ */
+export async function writeStoredZipAsync(
+  outputPath: string,
+  files: Iterable<StoredZipFile>,
+): Promise<{ entryCount: number }> {
+  const state: ZipWriteState = { entries: [], offset: 0 };
+  const fd = fs.openSync(outputPath, 'w');
+
+  try {
+    let bytesSinceYield = 0;
+    for (const file of files) {
+      bytesSinceYield += appendStoredZipFile(fd, state, file);
+      if (bytesSinceYield >= ASYNC_ZIP_YIELD_BYTE_BUDGET) {
+        bytesSinceYield = 0;
+        await new Promise<void>((resolve) => setImmediate(resolve));
+      }
+    }
+    finishStoredZip(fd, state);
+  } catch (error) {
+    fs.closeSync(fd);
+    fs.rmSync(outputPath, { force: true });
+    throw error;
+  }
+
+  fs.closeSync(fd);
+  return { entryCount: state.entries.length };
 }