SubMiner/vendor/hachidori/extension/custom-dictionary.js

/*
 * Shared source and archive rules for Hachidori's managed custom dictionary.
 *
 * This module deliberately has no Chrome or engine dependencies so Settings,
 * the background worker, and both engine runtimes use the same representation.
 *
 * SPDX-License-Identifier: GPL-3.0-or-later
 */

export const CUSTOM_DICTIONARY_ID = "e4c2e20a1a964b6cbd4ae3f87643c1f0";
export const CUSTOM_DICTIONARY_TITLE = "Hachidori Custom Dictionary";
export const CUSTOM_DICTIONARY_SOURCE_KEY = "customDictionarySource";
export const CUSTOM_DICTIONARY_SOURCE_SCHEMA_VERSION = 1;
export const EMPTY_CUSTOM_DICTIONARY_SEMANTIC_REVISION =
  "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945";

export function customDictionaryMetadataMatches(value, semanticRevision, entryCount) {
  return value?.title === CUSTOM_DICTIONARY_TITLE && value.revision === semanticRevision
    && value.termCount === entryCount
    && ["frequencyCount", "pitchCount", "kanjiCount", "mediaCount"].every(key => value[key] === 0)
    && value.isUpdatable === false && value.indexUrl === null && value.downloadUrl === null && value.language === "ja";
}

export function assertCustomDictionaryCommit(dictionaries) {
  const customIndexes = dictionaries.flatMap((dictionary, index) =>
    dictionary?.id === CUSTOM_DICTIONARY_ID ? [index] : []);
  if (customIndexes.length > 1) {
    throw new Error("the custom dictionary state contains duplicate managed packages");
  }
  if (customIndexes.length === 0) return;
  const custom = dictionaries[customIndexes[0]];
  if (customIndexes[0] !== 0
      || custom?.title !== CUSTOM_DICTIONARY_TITLE
      || custom?.enabled !== true) {
    throw new Error("the managed custom dictionary must stay enabled and first");
  }
  if (dictionaries.some((dictionary, index) =>
    index !== customIndexes[0] && dictionary?.title === CUSTOM_DICTIONARY_TITLE)) {
    throw new Error(`a dictionary named ${CUSTOM_DICTIONARY_TITLE} is already installed`);
  }
}

export function assertCustomSourceState(dictionaries, semanticRevision, entryCount) {
  const custom = dictionaries.filter((dictionary) => dictionary?.id === CUSTOM_DICTIONARY_ID);
  if (entryCount === 0) {
    if (custom.length !== 0) {
      throw new Error("a zero-entry custom source cannot retain its managed package");
    }
    return;
  }
  assertCustomDictionaryCommit(dictionaries);
  const entry = custom[0];
  if (custom.length !== 1
      || !customDictionaryMetadataMatches(entry, semanticRevision, entryCount)) {
    throw new Error("the custom dictionary package does not match its source semantics");
  }
}

const TERM_BANK_SIZE = 1_000;
const ZIP_UTF8_FLAG = 0x0800;
const ZIP_STORE = 0;
const ZIP_DATE = 0x21;
const ZIP_VERSION = 20;
const UINT16_MAX = 0xffff;
const UINT32_MAX = 0xff_ff_ff_ff;
const encoder = new TextEncoder();

function sourceText(value) {
  if (typeof value !== "string") {
    throw new TypeError("the custom dictionary source must be text");
  }
  return value;
}

export function emptyCustomDictionaryDocument() {
  return {
    schemaVersion: CUSTOM_DICTIONARY_SOURCE_SCHEMA_VERSION,
    revision: 0,
    semanticRevision: EMPTY_CUSTOM_DICTIONARY_SEMANTIC_REVISION,
    text: "",
  };
}

export function normaliseCustomDictionaryDocument(value) {
  if (value === null || value === undefined) {
    return emptyCustomDictionaryDocument();
  }
  if (!value || typeof value !== "object" || Array.isArray(value)) {
    throw new TypeError("the custom dictionary document is invalid");
  }
  if (value.schemaVersion !== CUSTOM_DICTIONARY_SOURCE_SCHEMA_VERSION) {
    throw new Error(`unsupported custom dictionary source schema ${String(value.schemaVersion)}`);
  }
  if (!Number.isInteger(value.revision) || value.revision < 0) {
    throw new TypeError("the custom dictionary document revision is invalid");
  }
  if (typeof value.text !== "string"
      || typeof value.semanticRevision !== "string"
      || !/^[0-9a-f]{64}$/u.test(value.semanticRevision)) {
    throw new TypeError("the custom dictionary document content is invalid");
  }
  return {
    schemaVersion: CUSTOM_DICTIONARY_SOURCE_SCHEMA_VERSION,
    revision: value.revision,
    semanticRevision: value.semanticRevision,
    text: value.text,
  };
}

function decodeDefinition(value) {
  if (!value.includes("\\")) return value;
  let decoded = "";
  for (let index = 0; index < value.length; index += 1) {
    const character = value[index];
    if (character !== "\\" || index + 1 >= value.length) {
      decoded += character;
      continue;
    }
    const next = value[index + 1];
    if (next === "\\") {
      decoded += "\\";
      index += 1;
    } else if (next === "n") {
      decoded += "\n";
      index += 1;
    } else {
      decoded += character;
    }
  }
  return decoded;
}

function normaliseEntry(value) {
  if (!value || typeof value !== "object" || Array.isArray(value)) {
    throw new TypeError("the custom dictionary entry must be an object");
  }
  const term = typeof value.term === "string" ? value.term.trim() : "";
  const reading = typeof value.reading === "string" ? value.reading.trim() : "";
  const definition = typeof value.definition === "string"
    ? value.definition.trim().replace(/\r\n?|\n/gu, "\n")
    : "";
  if (term === "") throw new TypeError("the custom dictionary term is empty");
  if (term.startsWith("#")) {
    throw new TypeError("the custom dictionary term cannot begin with #");
  }
  if (term.includes(",") || /[\r\n]/u.test(term)) {
    throw new TypeError("the custom dictionary term cannot contain a comma or newline");
  }
  if (reading === "") throw new TypeError("the custom dictionary reading is empty");
  if (reading.includes(",") || /[\r\n]/u.test(reading)) {
    throw new TypeError("the custom dictionary reading cannot contain a comma or newline");
  }
  if (definition === "") throw new TypeError("the custom dictionary definition is empty");
  return { term, reading, definition };
}

export function parseCustomDictionary(value) {
  const source = sourceText(value);
  const parsedSource = source.startsWith("\ufeff") ? source.slice(1) : source;
  const entries = [];
  const errors = [];
  const lines = parsedSource.split(/\r\n|\r|\n/u);
  for (let index = 0; index < lines.length; index += 1) {
    const line = lines[index];
    const trimmed = line.trim();
    if (trimmed === "" || line.trimStart().startsWith("#")) continue;

    const firstComma = line.indexOf(",");
    const secondComma = firstComma < 0 ? -1 : line.indexOf(",", firstComma + 1);
    if (firstComma < 0 || secondComma < 0) {
      errors.push({ lineNumber: index + 1, reason: "expected two commas" });
      continue;
    }
    const term = line.slice(0, firstComma).trim();
    const reading = line.slice(firstComma + 1, secondComma).trim();
    const encodedDefinition = line.slice(secondComma + 1).trim();
    const definition = decodeDefinition(encodedDefinition);
    if (term === "") {
      errors.push({ lineNumber: index + 1, reason: "term is empty" });
    } else if (reading === "") {
      errors.push({ lineNumber: index + 1, reason: "reading is empty" });
    } else if (definition.trim() === "") {
      errors.push({ lineNumber: index + 1, reason: "definition is empty" });
    } else {
      entries.push({ term, reading, definition });
    }
  }
  return { entries, errors };
}

export function serializeCustomDictionaryEntry(value) {
  const { term, reading, definition } = normaliseEntry(value);
  const encodedDefinition = definition
    .replaceAll("\\", "\\\\")
    .replaceAll("\n", String.raw`\n`);
  return `${term}, ${reading}, ${encodedDefinition}`;
}

export function appendCustomDictionaryEntry(value, entry) {
  const source = sourceText(value);
  const newline = source.includes("\r\n") ? "\r\n" : "\n";
  const separator = source === "" || /[\r\n]$/u.test(source) ? "" : newline;
  return `${source}${separator}${serializeCustomDictionaryEntry(entry)}${newline}`;
}

export async function customDictionarySemanticRevision(values) {
  if (!Array.isArray(values)) {
    throw new TypeError("the custom dictionary entries must be a list");
  }
  const bytes = encoder.encode(JSON.stringify(values.map(normaliseEntry)));
  const digest = new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", bytes));
  return Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join("");
}

const crcTable = (() => {
  const table = new Uint32Array(256);
  for (let index = 0; index < table.length; index += 1) {
    let value = index;
    for (let bit = 0; bit < 8; bit += 1) {
      value = (value & 1) === 0
        ? value >>> 1
        : 0xed_b8_83_20 ^ (value >>> 1);
    }
    table[index] = value >>> 0;
  }
  return table;
})();

function crc32(bytes) {
  let crc = UINT32_MAX;
  let index = 0;
  while (index < bytes.length) {
    crc = crcTable[(crc ^ bytes[index]) & 0xff] ^ (crc >>> 8);
    index += 1;
  }
  return (crc ^ UINT32_MAX) >>> 0;
}

function zipFile(name, value) {
  const nameBytes = encoder.encode(name);
  const data = encoder.encode(JSON.stringify(value));
  if (nameBytes.byteLength > UINT16_MAX || data.byteLength > UINT32_MAX) {
    throw new RangeError("the custom dictionary exceeds the classic ZIP representation");
  }
  return { nameBytes, data, crc: crc32(data), localOffset: 0 };
}

function checkedZipSize(value) {
  if (!Number.isSafeInteger(value) || value > UINT32_MAX) {
    throw new RangeError("the custom dictionary exceeds the classic ZIP representation");
  }
  return value;
}

function writeLocalHeader(view, offset, file) {
  view.setUint32(offset, 0x04_03_4b_50, true);
  view.setUint16(offset + 4, ZIP_VERSION, true);
  view.setUint16(offset + 6, ZIP_UTF8_FLAG, true);
  view.setUint16(offset + 8, ZIP_STORE, true);
  view.setUint16(offset + 10, 0, true);
  view.setUint16(offset + 12, ZIP_DATE, true);
  view.setUint32(offset + 14, file.crc, true);
  view.setUint32(offset + 18, file.data.byteLength, true);
  view.setUint32(offset + 22, file.data.byteLength, true);
  view.setUint16(offset + 26, file.nameBytes.byteLength, true);
  view.setUint16(offset + 28, 0, true);
}

function writeCentralHeader(view, offset, file) {
  view.setUint32(offset, 0x02_01_4b_50, true);
  view.setUint16(offset + 4, ZIP_VERSION, true);
  view.setUint16(offset + 6, ZIP_VERSION, true);
  view.setUint16(offset + 8, ZIP_UTF8_FLAG, true);
  view.setUint16(offset + 10, ZIP_STORE, true);
  view.setUint16(offset + 12, 0, true);
  view.setUint16(offset + 14, ZIP_DATE, true);
  view.setUint32(offset + 16, file.crc, true);
  view.setUint32(offset + 20, file.data.byteLength, true);
  view.setUint32(offset + 24, file.data.byteLength, true);
  view.setUint16(offset + 28, file.nameBytes.byteLength, true);
  view.setUint16(offset + 30, 0, true);
  view.setUint16(offset + 32, 0, true);
  view.setUint16(offset + 34, 0, true);
  view.setUint16(offset + 36, 0, true);
  view.setUint32(offset + 38, (0o100644 << 16) >>> 0, true);
  view.setUint32(offset + 42, file.localOffset, true);
}

function writeEndRecord(view, offset, entryCount, centralSize, centralOffset) {
  view.setUint32(offset, 0x06_05_4b_50, true);
  view.setUint16(offset + 4, 0, true);
  view.setUint16(offset + 6, 0, true);
  view.setUint16(offset + 8, entryCount, true);
  view.setUint16(offset + 10, entryCount, true);
  view.setUint32(offset + 12, centralSize, true);
  view.setUint32(offset + 16, centralOffset, true);
  view.setUint16(offset + 20, 0, true);
}

export function buildCustomDictionaryZip(values, revision) {
  if (!Array.isArray(values) || values.length === 0) {
    throw new TypeError("the custom dictionary needs at least one entry");
  }
  if (typeof revision !== "string" || revision === "") {
    throw new TypeError("the custom dictionary needs a semantic revision");
  }
  const entries = values.map(normaliseEntry);
  const files = [zipFile("index.json", {
    title: CUSTOM_DICTIONARY_TITLE,
    format: 3,
    revision,
    sequenced: true,
    author: "Hachidori",
    description: "Personal entries managed by Hachidori",
    sourceLanguage: "ja",
    targetLanguage: "en",
  })];
  for (let offset = 0; offset < entries.length; offset += TERM_BANK_SIZE) {
    const rows = entries.slice(offset, offset + TERM_BANK_SIZE).map((entry, index) => [
      entry.term,
      entry.reading,
      "",
      "",
      0,
      [entry.definition],
      offset + index + 1,
      "",
    ]);
    files.push(zipFile(`term_bank_${files.length}.json`, rows));
  }
  if (files.length > UINT16_MAX) {
    throw new RangeError("the custom dictionary exceeds the classic ZIP representation");
  }

  let localSize = 0;
  for (const file of files) {
    file.localOffset = localSize;
    localSize = checkedZipSize(localSize + 30 + file.nameBytes.byteLength + file.data.byteLength);
  }
  const centralSize = checkedZipSize(files.reduce(
    (size, file) => size + 46 + file.nameBytes.byteLength,
    0,
  ));
  const totalSize = checkedZipSize(localSize + centralSize + 22);
  const bytes = new Uint8Array(totalSize);
  const view = new DataView(bytes.buffer);

  let cursor = 0;
  for (const file of files) {
    writeLocalHeader(view, cursor, file);
    cursor += 30;
    bytes.set(file.nameBytes, cursor);
    cursor += file.nameBytes.byteLength;
    bytes.set(file.data, cursor);
    cursor += file.data.byteLength;
  }
  for (const file of files) {
    writeCentralHeader(view, cursor, file);
    cursor += 46;
    bytes.set(file.nameBytes, cursor);
    cursor += file.nameBytes.byteLength;
  }
  writeEndRecord(view, cursor, files.length, centralSize, localSize);
  return bytes;
}