import * as fs from 'fs'; import * as path from 'path'; import { ensureDir } from './fs-utils'; import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } from './types'; type ZipEntry = { name: string; crc32: number; size: number; localHeaderOffset: number; }; function writeUint32LE(buffer: Buffer, value: number, offset: number): number { const normalized = value >>> 0; buffer[offset] = normalized & 0xff; buffer[offset + 1] = (normalized >>> 8) & 0xff; buffer[offset + 2] = (normalized >>> 16) & 0xff; buffer[offset + 3] = (normalized >>> 24) & 0xff; return offset + 4; } export function buildDictionaryTitle(mediaId: number): string { return `SubMiner Character Dictionary (AniList ${mediaId})`; } function createIndex( dictionaryTitle: string, description: string, revision: string, ): Record { return { title: dictionaryTitle, revision, format: 3, author: 'SubMiner', description, }; } function createTagBank(): Array<[string, string, number, string, number]> { return [ ['name', 'partOfSpeech', 0, 'Character name', 0], ['main', 'name', 0, 'Protagonist', 0], ['primary', 'name', 0, 'Main character', 0], ['side', 'name', 0, 'Side character', 0], ['appears', 'name', 0, 'Minor appearance', 0], ]; } const CRC32_TABLE = (() => { const table = new Uint32Array(256); for (let i = 0; i < 256; i += 1) { let crc = i; for (let j = 0; j < 8; j += 1) { crc = (crc & 1) !== 0 ? 0xedb88320 ^ (crc >>> 1) : crc >>> 1; } table[i] = crc >>> 0; } return table; })(); function crc32(data: Buffer): number { let crc = 0xffffffff; for (const byte of data) { crc = CRC32_TABLE[(crc ^ byte) & 0xff]! ^ (crc >>> 8); } return (crc ^ 0xffffffff) >>> 0; } function createLocalFileHeader(fileName: Buffer, fileCrc32: number, fileSize: number): Buffer { const local = Buffer.alloc(30 + fileName.length); let cursor = 0; writeUint32LE(local, 0x04034b50, cursor); cursor += 4; local.writeUInt16LE(20, cursor); cursor += 2; local.writeUInt16LE(0, cursor); cursor += 2; local.writeUInt16LE(0, cursor); cursor += 2; local.writeUInt16LE(0, cursor); cursor += 2; local.writeUInt16LE(0, cursor); cursor += 2; writeUint32LE(local, fileCrc32, cursor); cursor += 4; writeUint32LE(local, fileSize, cursor); cursor += 4; writeUint32LE(local, fileSize, cursor); cursor += 4; local.writeUInt16LE(fileName.length, cursor); cursor += 2; local.writeUInt16LE(0, cursor); cursor += 2; fileName.copy(local, cursor); return local; } function createCentralDirectoryHeader(entry: ZipEntry): Buffer { const fileName = Buffer.from(entry.name, 'utf8'); const central = Buffer.alloc(46 + fileName.length); let cursor = 0; writeUint32LE(central, 0x02014b50, cursor); cursor += 4; central.writeUInt16LE(20, cursor); cursor += 2; central.writeUInt16LE(20, cursor); cursor += 2; central.writeUInt16LE(0, cursor); cursor += 2; central.writeUInt16LE(0, cursor); cursor += 2; central.writeUInt16LE(0, cursor); cursor += 2; central.writeUInt16LE(0, cursor); cursor += 2; writeUint32LE(central, entry.crc32, cursor); cursor += 4; writeUint32LE(central, entry.size, cursor); cursor += 4; writeUint32LE(central, entry.size, cursor); cursor += 4; central.writeUInt16LE(fileName.length, cursor); cursor += 2; central.writeUInt16LE(0, cursor); cursor += 2; central.writeUInt16LE(0, cursor); cursor += 2; central.writeUInt16LE(0, cursor); cursor += 2; central.writeUInt16LE(0, cursor); cursor += 2; writeUint32LE(central, 0, cursor); cursor += 4; writeUint32LE(central, entry.localHeaderOffset, cursor); cursor += 4; fileName.copy(central, cursor); return central; } function createEndOfCentralDirectory(entriesLength: number, centralSize: number, centralStart: number): Buffer { const end = Buffer.alloc(22); let cursor = 0; writeUint32LE(end, 0x06054b50, cursor); cursor += 4; end.writeUInt16LE(0, cursor); cursor += 2; end.writeUInt16LE(0, cursor); cursor += 2; end.writeUInt16LE(entriesLength, cursor); cursor += 2; end.writeUInt16LE(entriesLength, cursor); cursor += 2; writeUint32LE(end, centralSize, cursor); cursor += 4; writeUint32LE(end, centralStart, cursor); cursor += 4; end.writeUInt16LE(0, cursor); return end; } function writeBuffer(fd: number, buffer: Buffer): void { let written = 0; while (written < buffer.length) { written += fs.writeSync(fd, buffer, written, buffer.length - written); } } function writeStoredZip(outputPath: string, files: Iterable<{ name: string; data: Buffer }>): void { const entries: ZipEntry[] = []; let offset = 0; const fd = fs.openSync(outputPath, 'w'); try { for (const file of files) { const fileName = Buffer.from(file.name, 'utf8'); const fileSize = file.data.length; const fileCrc32 = crc32(file.data); const localHeader = createLocalFileHeader(fileName, fileCrc32, fileSize); writeBuffer(fd, localHeader); writeBuffer(fd, file.data); entries.push({ name: file.name, crc32: fileCrc32, size: fileSize, localHeaderOffset: offset, }); offset += localHeader.length + fileSize; } const centralStart = offset; for (const entry of entries) { const centralHeader = createCentralDirectoryHeader(entry); writeBuffer(fd, centralHeader); offset += centralHeader.length; } const centralSize = offset - centralStart; writeBuffer(fd, createEndOfCentralDirectory(entries.length, centralSize, centralStart)); } catch (error) { fs.closeSync(fd); fs.rmSync(outputPath, { force: true }); throw error; } fs.closeSync(fd); } export function buildDictionaryZip( outputPath: string, dictionaryTitle: string, description: string, revision: string, termEntries: CharacterDictionaryTermEntry[], images: CharacterDictionarySnapshotImage[], ): { zipPath: string; entryCount: number } { ensureDir(path.dirname(outputPath)); function* zipFiles(): Iterable<{ name: string; data: Buffer }> { yield { name: 'index.json', data: Buffer.from( JSON.stringify(createIndex(dictionaryTitle, description, revision), null, 2), 'utf8', ), }; yield { name: 'tag_bank_1.json', data: Buffer.from(JSON.stringify(createTagBank()), 'utf8'), }; for (const image of images) { yield { name: image.path, data: Buffer.from(image.dataBase64, 'base64'), }; } const entriesPerBank = 10_000; for (let i = 0; i < termEntries.length; i += entriesPerBank) { yield { name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`, data: Buffer.from(JSON.stringify(termEntries.slice(i, i + entriesPerBank)), 'utf8'), }; } } writeStoredZip(outputPath, zipFiles()); return { zipPath: outputPath, entryCount: termEntries.length }; }