SubMiner/src/shared/stored-zip.ts

import * as fs from 'fs';
import * as zlib from 'zlib';

type ZipEntry = {
  name: string;
  crc32: number;
  size: number;
  localHeaderOffset: number;
};

const ZIP32_MAX_UINT16 = 0xffff;
const ZIP32_MAX_UINT32 = 0xffffffff;

export type StoredZipFile = {
  name: string;
  data: Buffer;
};

function writeUint32LE(buffer: Buffer, value: number, offset: number): number {
  const normalized = value >>> 0;
  buffer[offset] = normalized & 0xff;
  buffer[offset + 1] = (normalized >>> 8) & 0xff;
  buffer[offset + 2] = (normalized >>> 16) & 0xff;
  buffer[offset + 3] = (normalized >>> 24) & 0xff;
  return offset + 4;
}

const CRC32_TABLE = (() => {
  const table = new Uint32Array(256);
  for (let i = 0; i < 256; i += 1) {
    let crc = i;
    for (let j = 0; j < 8; j += 1) {
      crc = (crc & 1) !== 0 ? 0xedb88320 ^ (crc >>> 1) : crc >>> 1;
    }
    table[i] = crc >>> 0;
  }
  return table;
})();

// Native CRC32 (Node >= 20.15) runs at native throughput, which matters for the multi-hundred-MB
// dictionary archives; the table loop stays as a fallback for runtimes without it.
const nativeCrc32 = (zlib as { crc32?: (data: Uint8Array, value?: number) => number }).crc32;

function crc32(data: Buffer): number {
  if (typeof nativeCrc32 === 'function') {
    return nativeCrc32(data) >>> 0;
  }
  let crc = 0xffffffff;
  for (let i = 0; i < data.length; i += 1) {
    crc = CRC32_TABLE[(crc ^ data[i]!) & 0xff]! ^ (crc >>> 8);
  }
  return (crc ^ 0xffffffff) >>> 0;
}

function createLocalFileHeader(fileName: Buffer, fileCrc32: number, fileSize: number): Buffer {
  const local = Buffer.alloc(30 + fileName.length);
  let cursor = 0;
  writeUint32LE(local, 0x04034b50, cursor);
  cursor += 4;
  local.writeUInt16LE(20, cursor);
  cursor += 2;
  local.writeUInt16LE(0, cursor);
  cursor += 2;
  local.writeUInt16LE(0, cursor);
  cursor += 2;
  local.writeUInt16LE(0, cursor);
  cursor += 2;
  local.writeUInt16LE(0, cursor);
  cursor += 2;
  writeUint32LE(local, fileCrc32, cursor);
  cursor += 4;
  writeUint32LE(local, fileSize, cursor);
  cursor += 4;
  writeUint32LE(local, fileSize, cursor);
  cursor += 4;
  local.writeUInt16LE(fileName.length, cursor);
  cursor += 2;
  local.writeUInt16LE(0, cursor);
  cursor += 2;
  fileName.copy(local, cursor);
  return local;
}

function createCentralDirectoryHeader(entry: ZipEntry): Buffer {
  const fileName = Buffer.from(entry.name, 'utf8');
  const central = Buffer.alloc(46 + fileName.length);
  let cursor = 0;
  writeUint32LE(central, 0x02014b50, cursor);
  cursor += 4;
  central.writeUInt16LE(20, cursor);
  cursor += 2;
  central.writeUInt16LE(20, cursor);
  cursor += 2;
  central.writeUInt16LE(0, cursor);
  cursor += 2;
  central.writeUInt16LE(0, cursor);
  cursor += 2;
  central.writeUInt16LE(0, cursor);
  cursor += 2;
  central.writeUInt16LE(0, cursor);
  cursor += 2;
  writeUint32LE(central, entry.crc32, cursor);
  cursor += 4;
  writeUint32LE(central, entry.size, cursor);
  cursor += 4;
  writeUint32LE(central, entry.size, cursor);
  cursor += 4;
  central.writeUInt16LE(fileName.length, cursor);
  cursor += 2;
  central.writeUInt16LE(0, cursor);
  cursor += 2;
  central.writeUInt16LE(0, cursor);
  cursor += 2;
  central.writeUInt16LE(0, cursor);
  cursor += 2;
  central.writeUInt16LE(0, cursor);
  cursor += 2;
  writeUint32LE(central, 0, cursor);
  cursor += 4;
  writeUint32LE(central, entry.localHeaderOffset, cursor);
  cursor += 4;
  fileName.copy(central, cursor);
  return central;
}

function createEndOfCentralDirectory(
  entriesLength: number,
  centralSize: number,
  centralStart: number,
): Buffer {
  if (
    entriesLength > ZIP32_MAX_UINT16 ||
    centralSize > ZIP32_MAX_UINT32 ||
    centralStart > ZIP32_MAX_UINT32
  ) {
    throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
  }

  const end = Buffer.alloc(22);
  let cursor = 0;
  writeUint32LE(end, 0x06054b50, cursor);
  cursor += 4;
  end.writeUInt16LE(0, cursor);
  cursor += 2;
  end.writeUInt16LE(0, cursor);
  cursor += 2;
  end.writeUInt16LE(entriesLength, cursor);
  cursor += 2;
  end.writeUInt16LE(entriesLength, cursor);
  cursor += 2;
  writeUint32LE(end, centralSize, cursor);
  cursor += 4;
  writeUint32LE(end, centralStart, cursor);
  cursor += 4;
  end.writeUInt16LE(0, cursor);
  return end;
}

const LOCAL_FILE_HEADER_SIGNATURE = 0x04034b50;
const LOCAL_FILE_HEADER_SIZE = 30;
const END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054b50;
const END_OF_CENTRAL_DIRECTORY_SIZE = 22;
const CENTRAL_FILE_HEADER_SIGNATURE = 0x02014b50;
const CENTRAL_FILE_HEADER_SIZE = 46;
// 65535 entries with names of a few dozen bytes stay far under this; the cap only stops a corrupt
// record length from asking for an allocation the size of the archive.
const MAX_CENTRAL_DIRECTORY_BYTES = 16 * 1024 * 1024;

/**
 * Walks every declared central-directory record, checking each signature and keeping the
 * variable-length name/extra/comment fields inside the directory. The walk has to land exactly on
 * the end of the directory, so a record that was overwritten in place fails even though the file
 * kept its size.
 */
function isCentralDirectoryIntact(
  fd: number,
  centralStart: number,
  centralSize: number,
  entryCount: number,
): boolean {
  if (centralSize === 0 || centralSize > MAX_CENTRAL_DIRECTORY_BYTES) {
    return false;
  }
  const central = Buffer.alloc(centralSize);
  if (fs.readSync(fd, central, 0, centralSize, centralStart) !== centralSize) {
    return false;
  }

  let cursor = 0;
  for (let index = 0; index < entryCount; index += 1) {
    if (cursor + CENTRAL_FILE_HEADER_SIZE > centralSize) {
      return false;
    }
    if (central.readUInt32LE(cursor) !== CENTRAL_FILE_HEADER_SIGNATURE) {
      return false;
    }
    const nameLength = central.readUInt16LE(cursor + 28);
    const extraLength = central.readUInt16LE(cursor + 30);
    const commentLength = central.readUInt16LE(cursor + 32);
    cursor += CENTRAL_FILE_HEADER_SIZE + nameLength + extraLength + commentLength;
    if (cursor > centralSize) {
      return false;
    }
  }
  return cursor === centralSize;
}

/**
 * Start of the central directory, or null when the archive is not a complete one of ours. The
 * end-of-central-directory record is written last, so finding an intact one is what separates a
 * finished archive from a half-written one.
 */
function readCentralDirectoryStart(fd: number, fileSize: number): number | null {
  if (fileSize < END_OF_CENTRAL_DIRECTORY_SIZE) {
    return null;
  }
  const end = Buffer.alloc(END_OF_CENTRAL_DIRECTORY_SIZE);
  const endOffset = fileSize - END_OF_CENTRAL_DIRECTORY_SIZE;
  if (fs.readSync(fd, end, 0, end.length, endOffset) !== end.length) {
    return null;
  }
  // writeStoredZip never writes an archive comment, so the record is exactly the last 22 bytes.
  if (end.readUInt32LE(0) !== END_OF_CENTRAL_DIRECTORY_SIGNATURE || end.readUInt16LE(20) !== 0) {
    return null;
  }
  const entryCount = end.readUInt16LE(10);
  if (entryCount === 0) {
    return null;
  }
  const centralSize = end.readUInt32LE(12);
  const centralStart = end.readUInt32LE(16);
  if (centralStart + centralSize !== endOffset) {
    return null;
  }
  if (!isCentralDirectoryIntact(fd, centralStart, centralSize, entryCount)) {
    return null;
  }
  return centralStart;
}

/**
 * Reads the first entry of an archive written by {@link writeStoredZip}: every entry is stored
 * uncompressed with no extra field and no data descriptor, so the leading local header is enough.
 * Returns null for anything it does not recognize, so callers treat a corrupt, truncated, or
 * foreign archive the same as a missing one.
 */
export function readStoredZipFirstFile(zipPath: string): StoredZipFile | null {
  let fd: number;
  try {
    fd = fs.openSync(zipPath, 'r');
  } catch {
    return null;
  }

  try {
    const fileSize = fs.fstatSync(fd).size;
    const centralStart = readCentralDirectoryStart(fd, fileSize);
    if (centralStart === null) {
      return null;
    }
    const header = Buffer.alloc(LOCAL_FILE_HEADER_SIZE);
    if (fs.readSync(fd, header, 0, header.length, 0) !== header.length) {
      return null;
    }
    if (header.readUInt32LE(0) !== LOCAL_FILE_HEADER_SIGNATURE) {
      return null;
    }
    // Compression method 0 (stored) is the only thing writeStoredZip emits.
    if (header.readUInt16LE(8) !== 0) {
      return null;
    }
    const entrySize = header.readUInt32LE(18);
    const nameLength = header.readUInt16LE(26);
    const extraLength = header.readUInt16LE(28);
    const dataOffset = LOCAL_FILE_HEADER_SIZE + nameLength + extraLength;
    if (dataOffset + entrySize > centralStart) {
      return null;
    }
    const name = Buffer.alloc(nameLength);
    if (
      nameLength > 0 &&
      fs.readSync(fd, name, 0, nameLength, LOCAL_FILE_HEADER_SIZE) !== nameLength
    ) {
      return null;
    }
    const data = Buffer.alloc(entrySize);
    if (entrySize > 0 && fs.readSync(fd, data, 0, entrySize, dataOffset) !== entrySize) {
      return null;
    }
    return { name: name.toString('utf8'), data };
  } catch {
    return null;
  } finally {
    fs.closeSync(fd);
  }
}

function writeBuffer(fd: number, buffer: Buffer): void {
  let written = 0;
  while (written < buffer.length) {
    written += fs.writeSync(fd, buffer, written, buffer.length - written);
  }
}

type ZipWriteState = {
  entries: ZipEntry[];
  offset: number;
};

/** Appends one stored entry (local header + data) and returns the bytes written. */
function appendStoredZipFile(fd: number, state: ZipWriteState, file: StoredZipFile): number {
  const fileName = Buffer.from(file.name, 'utf8');
  const fileSize = file.data.length;
  if (fileName.length > ZIP32_MAX_UINT16) {
    throw new RangeError(`ZIP entry name too long: ${file.name}`);
  }
  if (fileSize > ZIP32_MAX_UINT32) {
    throw new RangeError(`ZIP entry too large for ZIP32: ${file.name}`);
  }
  if (state.offset > ZIP32_MAX_UINT32) {
    throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
  }
  const fileCrc32 = crc32(file.data);
  const localHeader = createLocalFileHeader(fileName, fileCrc32, fileSize);
  const nextOffset = state.offset + localHeader.length + fileSize;
  if (nextOffset > ZIP32_MAX_UINT32) {
    throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
  }
  writeBuffer(fd, localHeader);
  writeBuffer(fd, file.data);
  state.entries.push({
    name: file.name,
    crc32: fileCrc32,
    size: fileSize,
    localHeaderOffset: state.offset,
  });
  const written = nextOffset - state.offset;
  state.offset = nextOffset;
  return written;
}

function finishStoredZip(fd: number, state: ZipWriteState): void {
  const centralStart = state.offset;
  if (centralStart > ZIP32_MAX_UINT32) {
    throw new RangeError('Archive exceeds ZIP32 limits (Zip64 not implemented)');
  }
  for (const entry of state.entries) {
    const centralHeader = createCentralDirectoryHeader(entry);
    writeBuffer(fd, centralHeader);
    state.offset += centralHeader.length;
  }

  const centralSize = state.offset - centralStart;
  writeBuffer(fd, createEndOfCentralDirectory(state.entries.length, centralSize, centralStart));
}

export function writeStoredZip(
  outputPath: string,
  files: Iterable<StoredZipFile>,
): { entryCount: number } {
  const state: ZipWriteState = { entries: [], offset: 0 };
  const fd = fs.openSync(outputPath, 'w');

  try {
    for (const file of files) {
      appendStoredZipFile(fd, state, file);
    }
    finishStoredZip(fd, state);
  } catch (error) {
    fs.closeSync(fd);
    fs.rmSync(outputPath, { force: true });
    throw error;
  }

  fs.closeSync(fd);
  return { entryCount: state.entries.length };
}

// Yielding roughly every 8MB keeps individual event-loop blocks in the low tens of milliseconds
// while adding a negligible number of macrotask hops even for the largest merged dictionary.
const ASYNC_ZIP_YIELD_BYTE_BUDGET = 8 * 1024 * 1024;

/**
 * Same archive as {@link writeStoredZip}, written without starving the event loop: entry
 * generation, CRC, and writes proceed in byte-budgeted slices with a macrotask yield in between.
 * Multi-hundred-MB dictionary archives previously blocked the main process long enough for the
 * compositor to declare the app unresponsive.
 */
export async function writeStoredZipAsync(
  outputPath: string,
  files: Iterable<StoredZipFile>,
): Promise<{ entryCount: number }> {
  const state: ZipWriteState = { entries: [], offset: 0 };
  const fd = fs.openSync(outputPath, 'w');

  try {
    let bytesSinceYield = 0;
    for (const file of files) {
      bytesSinceYield += appendStoredZipFile(fd, state, file);
      if (bytesSinceYield >= ASYNC_ZIP_YIELD_BYTE_BUDGET) {
        bytesSinceYield = 0;
        await new Promise<void>((resolve) => setImmediate(resolve));
      }
    }
    finishStoredZip(fd, state);
  } catch (error) {
    fs.closeSync(fd);
    fs.rmSync(outputPath, { force: true });
    throw error;
  }

  fs.closeSync(fd);
  return { entryCount: state.entries.length };
}