mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-09-22 17:16:19 -07:00
feat(dictionary): add Hachidori backend support
- Add backend selection, setup gating, Anki integration, and external host support - Add launcher flags, documentation, packaging, and focused tests - Open on-demand overlay modals on the first attempt
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
// Proves the vendored lzokay and gumbo-parser libraries compile, link and do
|
||||
// what the MDX importer needs from them: an LZO1X round trip and an HTML
|
||||
// fragment parse that yields the expected tree.
|
||||
#include <lzokay.hpp>
|
||||
#include <nokogiri_gumbo.h>
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
int failures = 0;
|
||||
|
||||
void check(bool ok, const char* what) {
|
||||
if (!ok) {
|
||||
std::printf("FAIL %s\n", what);
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
|
||||
void test_lzokay() {
|
||||
std::string text;
|
||||
for (int i = 0; i < 200; ++i) {
|
||||
text += "見出し語の説明文 " + std::to_string(i % 7) + " repeated text, ";
|
||||
}
|
||||
std::vector<uint8_t> compressed(lzokay::compress_worst_size(text.size()));
|
||||
size_t compressed_size = 0;
|
||||
auto result = lzokay::compress(reinterpret_cast<const uint8_t*>(text.data()), text.size(), compressed.data(),
|
||||
compressed.size(), compressed_size);
|
||||
check(result == lzokay::EResult::Success, "lzokay compress");
|
||||
check(compressed_size < text.size() / 2, "lzokay compressed the repetitive text");
|
||||
|
||||
std::vector<uint8_t> decompressed(text.size());
|
||||
size_t decompressed_size = 0;
|
||||
result = lzokay::decompress(compressed.data(), compressed_size, decompressed.data(), decompressed.size(),
|
||||
decompressed_size);
|
||||
check(result == lzokay::EResult::Success, "lzokay decompress");
|
||||
check(decompressed_size == text.size() &&
|
||||
std::memcmp(decompressed.data(), text.data(), text.size()) == 0,
|
||||
"lzokay round trip");
|
||||
|
||||
// A too-small output buffer is reported, not written past.
|
||||
std::vector<uint8_t> small(16);
|
||||
result = lzokay::decompress(compressed.data(), compressed_size, small.data(), small.size(), decompressed_size);
|
||||
check(result == lzokay::EResult::OutputOverrun, "lzokay output overrun detected");
|
||||
}
|
||||
|
||||
const GumboNode* first_element_child(const GumboNode* node) {
|
||||
for (unsigned int i = 0; i < node->v.element.children.length; ++i) {
|
||||
auto* child = static_cast<const GumboNode*>(node->v.element.children.data[i]);
|
||||
if (child->type == GUMBO_NODE_ELEMENT) {
|
||||
return child;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void test_gumbo() {
|
||||
const char* html = "<p class=\"x\">見<b>出</b>し&<br>語</p><img src=\"a.png\">";
|
||||
GumboOptions options = kGumboDefaultOptions;
|
||||
options.fragment_context = "body";
|
||||
GumboOutput* output = gumbo_parse_with_options(&options, html, std::strlen(html));
|
||||
check(output != nullptr, "gumbo parse");
|
||||
if (!output) {
|
||||
return;
|
||||
}
|
||||
// Fragment parsing puts the children directly under the synthetic <html> root.
|
||||
const GumboNode* root = output->root;
|
||||
check(root->type == GUMBO_NODE_ELEMENT && root->v.element.tag == GUMBO_TAG_HTML, "gumbo root is <html>");
|
||||
const GumboNode* p = first_element_child(root);
|
||||
check(p && p->v.element.tag == GUMBO_TAG_P, "gumbo first child is <p>");
|
||||
if (p) {
|
||||
const GumboAttribute* cls = gumbo_get_attribute(&p->v.element.attributes, "class");
|
||||
check(cls && std::strcmp(cls->value, "x") == 0, "gumbo attribute value");
|
||||
check(p->v.element.children.length == 5, "gumbo <p> has text, <b>, text, <br>, text");
|
||||
auto* text = static_cast<const GumboNode*>(p->v.element.children.data[2]);
|
||||
check(text->type == GUMBO_NODE_TEXT && std::strcmp(text->v.text.text, "し&") == 0,
|
||||
"gumbo decodes character references");
|
||||
}
|
||||
gumbo_destroy_output(output);
|
||||
}
|
||||
}
|
||||
|
||||
int main() {
|
||||
test_lzokay();
|
||||
test_gumbo();
|
||||
if (failures == 0) {
|
||||
std::printf("ok\n");
|
||||
}
|
||||
return failures == 0 ? 0 : 1;
|
||||
}
|
||||
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
+338
@@ -0,0 +1,338 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Writes the MDX/MDD fixtures used by mdict_reader_test and mdict_test.
|
||||
|
||||
A small MDict writer (format per
|
||||
https://github.com/zhansliu/writemdict/blob/master/fileformat.md) rather than
|
||||
a dependency on writemdict, which is not on PyPI. Supports engine versions
|
||||
1.2 and 2.0, UTF-8 and UTF-16 text, compression 0 (stored), 1 (LZO1X, written
|
||||
as a single literal run, which is a valid stream) and 2 (zlib), and the
|
||||
Encrypted=2 key-index cipher. Every fixture is a few KB and deterministic.
|
||||
|
||||
python3 tests/fixtures/mdict/gen_fixtures.py
|
||||
"""
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- RIPEMD-128
|
||||
def _rol(x, n):
|
||||
return ((x << n) | (x >> (32 - n))) & 0xFFFFFFFF
|
||||
|
||||
|
||||
_RL = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
7, 4, 13, 1, 10, 6, 15, 3, 12, 0, 9, 5, 2, 14, 11, 8,
|
||||
3, 10, 14, 4, 9, 15, 8, 1, 2, 7, 0, 6, 13, 11, 5, 12,
|
||||
1, 9, 11, 10, 0, 8, 12, 4, 13, 3, 7, 15, 14, 5, 6, 2]
|
||||
_RR = [5, 14, 7, 0, 9, 2, 11, 4, 13, 6, 15, 8, 1, 10, 3, 12,
|
||||
6, 11, 3, 7, 0, 13, 5, 10, 14, 15, 8, 12, 4, 9, 1, 2,
|
||||
15, 5, 1, 3, 7, 14, 6, 9, 11, 8, 12, 2, 10, 0, 4, 13,
|
||||
8, 6, 4, 1, 3, 11, 15, 0, 5, 12, 2, 13, 9, 7, 10, 14]
|
||||
_SL = [11, 14, 15, 12, 5, 8, 7, 9, 11, 13, 14, 15, 6, 7, 9, 8,
|
||||
7, 6, 8, 13, 11, 9, 7, 15, 7, 12, 15, 9, 11, 7, 13, 12,
|
||||
11, 13, 6, 7, 14, 9, 13, 15, 14, 8, 13, 6, 5, 12, 7, 5,
|
||||
11, 12, 14, 15, 14, 15, 9, 8, 9, 14, 5, 6, 8, 6, 5, 12]
|
||||
_SR = [8, 9, 9, 11, 13, 15, 15, 5, 7, 7, 8, 11, 14, 14, 12, 6,
|
||||
9, 13, 15, 7, 12, 8, 9, 11, 7, 7, 12, 7, 6, 15, 13, 11,
|
||||
9, 7, 15, 11, 8, 6, 6, 14, 12, 13, 5, 14, 13, 13, 7, 5,
|
||||
15, 5, 8, 11, 14, 14, 6, 14, 6, 9, 12, 9, 12, 5, 15, 8]
|
||||
_KL = [0x00000000, 0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC]
|
||||
_KR = [0x50A28BE6, 0x5C4DD124, 0x6D703EF3, 0x00000000]
|
||||
|
||||
|
||||
def _f(r, x, y, z):
|
||||
if r == 0:
|
||||
return x ^ y ^ z
|
||||
if r == 1:
|
||||
return (x & y) | (~x & z)
|
||||
if r == 2:
|
||||
return (x | ~y) ^ z
|
||||
return (x & z) | (y & ~z)
|
||||
|
||||
|
||||
def ripemd128(data):
|
||||
h = [0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476]
|
||||
msg = bytearray(data) + b"\x80"
|
||||
while len(msg) % 64 != 56:
|
||||
msg += b"\x00"
|
||||
msg += struct.pack("<Q", len(data) * 8)
|
||||
for off in range(0, len(msg), 64):
|
||||
x = list(struct.unpack("<16I", msg[off:off + 64]))
|
||||
al, bl, cl, dl = h
|
||||
ar, br, cr, dr = h
|
||||
for j in range(64):
|
||||
r = j // 16
|
||||
t = _rol((al + _f(r, bl, cl, dl) + x[_RL[j]] + _KL[r]) & 0xFFFFFFFF, _SL[j])
|
||||
al, dl, cl, bl = dl, cl, bl, t
|
||||
t = _rol((ar + _f(3 - r, br, cr, dr) + x[_RR[j]] + _KR[r]) & 0xFFFFFFFF, _SR[j])
|
||||
ar, dr, cr, br = dr, cr, br, t
|
||||
t = (h[1] + cl + dr) & 0xFFFFFFFF
|
||||
h[1] = (h[2] + dl + ar) & 0xFFFFFFFF
|
||||
h[2] = (h[3] + al + br) & 0xFFFFFFFF
|
||||
h[3] = (h[0] + bl + cr) & 0xFFFFFFFF
|
||||
h[0] = t
|
||||
return struct.pack("<4I", *h)
|
||||
|
||||
|
||||
assert ripemd128(b"abc").hex() == "c14a12199c66e4ba84636b0f69144c77"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ helpers
|
||||
def lzo_literal_stream(data):
|
||||
"""A valid LZO1X stream that stores `data` as one literal run.
|
||||
|
||||
First-byte shortcut: 18..255 copies (byte - 17) literals. Longer runs use
|
||||
the regular long-literal instruction 0x00 with zero-byte length extension.
|
||||
0x11 0x00 0x00 is the end-of-stream marker (M4 with distance 16384).
|
||||
"""
|
||||
n = len(data)
|
||||
end = b"\x11\x00\x00"
|
||||
if n == 0:
|
||||
return end
|
||||
if n <= 238:
|
||||
return bytes([17 + n]) + data + end
|
||||
rest = n - 18
|
||||
zeros = (rest - 1) // 255
|
||||
last = rest - 255 * zeros
|
||||
return b"\x00" + b"\x00" * zeros + bytes([last]) + data + end
|
||||
|
||||
|
||||
def frame(payload, compression):
|
||||
"""MDict block framing: LE u32 compression, BE adler32 of payload, packed payload."""
|
||||
if compression == 0:
|
||||
packed = payload
|
||||
elif compression == 1:
|
||||
packed = lzo_literal_stream(payload)
|
||||
elif compression == 2:
|
||||
packed = zlib.compress(payload, 9)
|
||||
else:
|
||||
raise ValueError(compression)
|
||||
return struct.pack("<I", compression) + struct.pack(">I", zlib.adler32(payload) & 0xFFFFFFFF) + packed
|
||||
|
||||
|
||||
def encrypt_key_index(framed):
|
||||
key = ripemd128(framed[4:8] + struct.pack("<L", 0x3695))
|
||||
body = bytearray(framed[8:])
|
||||
previous = 0x36
|
||||
# Inverse of readmdict's _fast_decrypt: plaintext byte p -> stored byte b with
|
||||
# decrypt(b) = swap(b) ^ prev ^ i ^ key[i], prev = previous stored byte.
|
||||
out = bytearray()
|
||||
for i, p in enumerate(body):
|
||||
t = p ^ previous ^ (i & 0xFF) ^ key[i % 16]
|
||||
b = ((t >> 4) | (t << 4)) & 0xFF
|
||||
out.append(b)
|
||||
previous = b
|
||||
return framed[:8] + bytes(out)
|
||||
|
||||
|
||||
def xml_escape(s):
|
||||
return (s.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- writer
|
||||
def write_mdict(path, entries, *, version="2.0", encoding="UTF-8", compression=2, encrypted=0,
|
||||
kind="mdx", fmt="Html", title="Fixture", description="", stylesheet="",
|
||||
block_entries=3, record_block_bytes=200, extra_attrs=None, corrupt=None):
|
||||
"""entries: list of (key, value) with value str for mdx, bytes for mdd.
|
||||
|
||||
corrupt: None or one of 'truncate', 'record_adler', 'huge_block' to produce a
|
||||
malformed file for the negative tests.
|
||||
"""
|
||||
v2 = float(version) >= 2.0
|
||||
num = ">Q" if v2 else ">I"
|
||||
width = 8 if v2 else 4
|
||||
text_enc = "utf-16-le" if (kind == "mdd" or encoding.upper().startswith("UTF-16")) else "utf-8"
|
||||
term = b"\x00\x00" if text_enc == "utf-16-le" else b"\x00"
|
||||
|
||||
# Records, in key order (MDict stores keys sorted; we keep caller order,
|
||||
# which the fixtures keep sorted where the reader cares).
|
||||
records = []
|
||||
for _, value in entries:
|
||||
if kind == "mdd":
|
||||
records.append(value)
|
||||
else:
|
||||
records.append(value.encode(text_enc) + term)
|
||||
|
||||
# Record blocks.
|
||||
record_blocks = []
|
||||
cur = b""
|
||||
for rec in records:
|
||||
if cur and len(cur) + len(rec) > record_block_bytes:
|
||||
record_blocks.append(cur)
|
||||
cur = b""
|
||||
cur += rec
|
||||
if cur or not record_blocks:
|
||||
record_blocks.append(cur)
|
||||
|
||||
# Key blocks with record offsets into the concatenated record space.
|
||||
offsets = []
|
||||
off = 0
|
||||
for rec in records:
|
||||
offsets.append(off)
|
||||
off += len(rec)
|
||||
keys = []
|
||||
for (key, _), o in zip(entries, offsets):
|
||||
keys.append((key, o))
|
||||
key_blocks = [keys[i:i + block_entries] for i in range(0, len(keys), block_entries)]
|
||||
|
||||
key_block_bytes = []
|
||||
for block in key_blocks:
|
||||
raw = b"".join(struct.pack(num, o) + k.encode(text_enc) + term for k, o in block)
|
||||
key_block_bytes.append((raw, frame(raw, compression)))
|
||||
|
||||
# Key-block index.
|
||||
size_fmt = ">H" if v2 else ">B"
|
||||
index = b""
|
||||
for block, (raw, packed) in zip(key_blocks, key_block_bytes):
|
||||
first = block[0][0].encode(text_enc)
|
||||
last = block[-1][0].encode(text_enc)
|
||||
unit = 2 if text_enc == "utf-16-le" else 1
|
||||
index += struct.pack(num, len(block))
|
||||
index += struct.pack(size_fmt, len(first) // unit) + first + (term if v2 else b"")
|
||||
index += struct.pack(size_fmt, len(last) // unit) + last + (term if v2 else b"")
|
||||
index += struct.pack(num, len(packed)) + struct.pack(num, len(raw))
|
||||
if v2:
|
||||
index_packed = frame(index, 2)
|
||||
if encrypted & 2:
|
||||
index_packed = encrypt_key_index(index_packed)
|
||||
else:
|
||||
index_packed = index
|
||||
|
||||
key_blocks_packed = b"".join(p for _, p in key_block_bytes)
|
||||
if v2:
|
||||
key_header = struct.pack(num, len(key_blocks)) + struct.pack(num, len(keys)) + struct.pack(num, len(index))
|
||||
key_header += struct.pack(num, len(index_packed)) + struct.pack(num, len(key_blocks_packed))
|
||||
key_header += struct.pack(">I", zlib.adler32(key_header) & 0xFFFFFFFF)
|
||||
else:
|
||||
key_header = struct.pack(num, len(key_blocks)) + struct.pack(num, len(keys))
|
||||
key_header += struct.pack(num, len(index_packed)) + struct.pack(num, len(key_blocks_packed))
|
||||
|
||||
# Record section.
|
||||
record_packed = []
|
||||
record_index = b""
|
||||
for i, blk in enumerate(record_blocks):
|
||||
framed = frame(blk, compression)
|
||||
if corrupt == "record_adler" and i == 0:
|
||||
framed = framed[:4] + struct.pack(">I", (struct.unpack(">I", framed[4:8])[0] ^ 1)) + framed[8:]
|
||||
record_packed.append(framed)
|
||||
declared_unpacked = len(blk)
|
||||
if corrupt == "huge_block" and i == 0:
|
||||
declared_unpacked = 1 << 40
|
||||
record_index += struct.pack(num, len(framed)) + struct.pack(num, declared_unpacked)
|
||||
record_blocks_packed = b"".join(record_packed)
|
||||
record_header = struct.pack(num, len(record_blocks)) + struct.pack(num, len(keys))
|
||||
record_header += struct.pack(num, len(record_index)) + struct.pack(num, len(record_blocks_packed))
|
||||
|
||||
# Header.
|
||||
attrs = {
|
||||
"GeneratedByEngineVersion": version,
|
||||
"RequiredEngineVersion": version,
|
||||
"Format": fmt,
|
||||
"KeyCaseSensitive": "No",
|
||||
"StripKey": "Yes",
|
||||
"Encrypted": str(encrypted),
|
||||
"RegisterBy": "EMail",
|
||||
"Description": description,
|
||||
"Title": title,
|
||||
"Encoding": "UTF-16" if encoding.upper().startswith("UTF-16") else encoding,
|
||||
"CreationDate": "2020-1-1",
|
||||
"Compact": "Yes",
|
||||
"Compat": "Yes",
|
||||
"Left2Right": "Yes",
|
||||
"DataSourceFormat": "107",
|
||||
"StyleSheet": stylesheet,
|
||||
}
|
||||
if kind == "mdd":
|
||||
attrs["Encoding"] = ""
|
||||
if extra_attrs:
|
||||
attrs.update(extra_attrs)
|
||||
root = "Library_Data" if kind == "mdd" else "Dictionary"
|
||||
header_text = "<%s %s/>\r\n" % (root, " ".join('%s="%s"' % (k, xml_escape(v)) for k, v in attrs.items()))
|
||||
header_bytes = header_text.encode("utf-16-le") + b"\x00\x00"
|
||||
header = struct.pack(">I", len(header_bytes)) + header_bytes
|
||||
header += struct.pack("<I", zlib.adler32(header_bytes) & 0xFFFFFFFF)
|
||||
|
||||
data = header + key_header + index_packed + key_blocks_packed + record_header + record_index + record_blocks_packed
|
||||
if corrupt == "truncate":
|
||||
data = data[:-(len(record_blocks_packed) // 2 + 1)]
|
||||
with open(path, "wb") as f:
|
||||
f.write(data)
|
||||
return len(data)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- fixtures
|
||||
HTML_ENTRIES = [
|
||||
("@@@LINK_target", "<div>target of a link</div>"),
|
||||
("alias", "@@@LINK=@@@LINK_target\r\n"),
|
||||
("dup", '<p class="a">first dup</p>'),
|
||||
("dup", '<p class="a">second dup</p>'),
|
||||
("entry", '<b>bold</b> <i>italic</i> <a href="entry://alias">alias</a> <a href="sound://a.spx">snd</a>'
|
||||
'<img src="../evil.png"><style>.inline-x { color: blue; }</style>'),
|
||||
("missing-alias", "@@@LINK=nowhere"),
|
||||
("ruby", '<table><tr><td><ruby>漢<rt>かん</rt></ruby></td></tr></table><img src="img/pic.png">'),
|
||||
("食べる", "`1`to eat`2` (ichidan)"),
|
||||
("見出し", "<span style=\"color:red;font-size:12px\">見出し語</span>"),
|
||||
]
|
||||
|
||||
TEXT_ENTRIES = [
|
||||
("alpha", "first definition"),
|
||||
("beta", "second\ndefinition with newline"),
|
||||
("gamma", "third"),
|
||||
("日本語", "Japanese text \"quoted\""),
|
||||
]
|
||||
|
||||
PNG = bytes.fromhex(
|
||||
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489"
|
||||
"0000000d49444154789c63f8ffff3f0005fe02fea72d5a5e0000000049454e44ae426082"
|
||||
)
|
||||
|
||||
MDD_ENTRIES = [
|
||||
("\\a.spx", b"not really speex"),
|
||||
("\\img\\pic.png", PNG),
|
||||
("\\style.css", ".mdx-red { color: red; }\n".encode("utf-8")),
|
||||
("\\..\\evil.png", b"traversal"),
|
||||
# UTF-16LE without a BOM and with non-ASCII text, as some MDD authors save it.
|
||||
("\\utf16.css", ".u16::before { content: \"\u2192\"; }\n".encode("utf-16-le")),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
out = lambda name: os.path.join(HERE, name) # noqa: E731
|
||||
sizes = {}
|
||||
sizes["v2_utf8_zlib_text.mdx"] = write_mdict(
|
||||
out("v2_utf8_zlib_text.mdx"), TEXT_ENTRIES, version="2.0", encoding="UTF-8", compression=2,
|
||||
fmt="Text", title="Text Fixture", description="A <b>text</b> fixture & entities")
|
||||
sizes["v2_utf8_lzo_html.mdx"] = write_mdict(
|
||||
out("v2_utf8_lzo_html.mdx"), HTML_ENTRIES, version="2.0", encoding="UTF-8", compression=1,
|
||||
fmt="Html", title="HTML Fixture", stylesheet="1\n<b>\n</b>\n2\n<i>\n</i>\n",
|
||||
description="HTML fixture with links, duplicates and a stylesheet")
|
||||
sizes["v2_utf16_encrypted2.mdx"] = write_mdict(
|
||||
out("v2_utf16_encrypted2.mdx"), TEXT_ENTRIES, version="2.0", encoding="UTF-16", compression=2,
|
||||
encrypted=2, fmt="Text", title="UTF-16 Fixture")
|
||||
sizes["v1_utf8_stored.mdx"] = write_mdict(
|
||||
out("v1_utf8_stored.mdx"), TEXT_ENTRIES, version="1.2", encoding="UTF-8", compression=0,
|
||||
fmt="Text", title="V1 Fixture")
|
||||
sizes["v2_utf8_lzo_html.mdd"] = write_mdict(
|
||||
out("v2_utf8_lzo_html.mdd"), MDD_ENTRIES, version="2.0", compression=2, kind="mdd",
|
||||
title="HTML Fixture Media")
|
||||
# Malformed inputs. Each must fail with a specific message.
|
||||
sizes["bad_truncated.mdx"] = write_mdict(
|
||||
out("bad_truncated.mdx"), TEXT_ENTRIES, fmt="Text", corrupt="truncate")
|
||||
sizes["bad_adler.mdx"] = write_mdict(
|
||||
out("bad_adler.mdx"), TEXT_ENTRIES, fmt="Text", corrupt="record_adler")
|
||||
sizes["bad_huge_block.mdx"] = write_mdict(
|
||||
out("bad_huge_block.mdx"), TEXT_ENTRIES, fmt="Text", corrupt="huge_block")
|
||||
sizes["bad_encrypted1.mdx"] = write_mdict(
|
||||
out("bad_encrypted1.mdx"), TEXT_ENTRIES, fmt="Text", encrypted=1)
|
||||
sizes["bad_gbk.mdx"] = write_mdict(
|
||||
out("bad_gbk.mdx"), TEXT_ENTRIES, fmt="Text", encoding="GBK")
|
||||
sizes["bad_v3.mdx"] = write_mdict(
|
||||
out("bad_v3.mdx"), TEXT_ENTRIES, fmt="Text", version="3.0")
|
||||
for name, size in sizes.items():
|
||||
print("%-28s %6d bytes" % (name, size))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
+128
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Writes small_dict.zip, the Yomitan dictionary used by import_equivalence_test.
|
||||
|
||||
The archive is deterministic (fixed timestamps, fixed entry order, fixed
|
||||
compression) so the golden hashes in golden.sha256 stay valid when it is
|
||||
regenerated. Run from any directory:
|
||||
|
||||
python3 tests/fixtures/yomitan/gen_fixture.py
|
||||
|
||||
Every bank kind the importer reads is present, plus a stylesheet and media so
|
||||
the equivalence test covers blobs.bin, hash.table, bloom.filter, media.bin,
|
||||
media.idx and dict.zstd.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import zipfile
|
||||
import zlib
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
OUT = os.path.join(HERE, "small_dict.zip")
|
||||
STAMP = (2020, 1, 1, 0, 0, 0)
|
||||
|
||||
# 1x1 PNG, opaque white.
|
||||
PNG = bytes.fromhex(
|
||||
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489"
|
||||
"0000000d49444154789c63f8ffff3f0005fe02fea72d5a5e0000000049454e44ae426082"
|
||||
)
|
||||
|
||||
|
||||
def dumps(value):
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def structured(text, extra=None):
|
||||
content = [{"tag": "span", "style": {"fontWeight": "bold"}, "content": text}]
|
||||
if extra:
|
||||
content.append({"tag": "div", "data": {"kind": "note"}, "content": extra})
|
||||
return {"type": "structured-content", "content": {"tag": "div", "content": content}}
|
||||
|
||||
|
||||
def term_bank_1():
|
||||
terms = []
|
||||
# Enough long glossaries to let the zstd trainer produce a dictionary.
|
||||
for i in range(48):
|
||||
gloss = "語釈 {}: これはテスト用の見出し語の説明文です。".format(i) + "同じ語尾を繰り返します。" * 3
|
||||
terms.append(["見出し{}".format(i), "みだし{}".format(i), "n", "", 100 - i, [gloss], i + 1, "P"])
|
||||
terms.append(["食べる", "たべる", "v1", "v1", 50, ["to eat", structured("食べる", "Ichidan verb")], 1000, ""])
|
||||
terms.append(["食べる", "たべる", "v1", "v1", 40, ["to eat"], 1000, ""]) # duplicate glossary text
|
||||
terms.append(["猫", "ねこ", "n", "", 10, [{"type": "image", "path": "img/neko.png", "width": 1, "height": 1}], 1001, ""])
|
||||
terms.append(["日本", "にほん", None, "", 0, ["Japan"], 1002, "P"])
|
||||
terms.append(["日本", "にっぽん", "n", "", 0, ["Japan"], 1002, ""])
|
||||
terms.append(["同形", "", "n", "", 0, ["reading omitted"], 1003, ""])
|
||||
return terms
|
||||
|
||||
|
||||
def term_bank_2():
|
||||
return [
|
||||
["走る", "はしる", "v5", "v5", 5, ["to run"], 2000, ""],
|
||||
["走る", "はしる", "v5", "v5", 5, ["to run"], 2000, ""], # exact duplicate entry
|
||||
["\"quoted\" [brackets] \\backslash", "quoted", "", "", 0, ["escapes \"\\ 【】"], 2001, ""],
|
||||
]
|
||||
|
||||
|
||||
def term_meta_bank_1():
|
||||
return [
|
||||
["食べる", "freq", 12],
|
||||
["食べる", "freq", {"reading": "たべる", "frequency": {"value": 12, "displayValue": "12㋕"}}],
|
||||
["猫", "freq", {"value": 3, "displayValue": "3"}],
|
||||
["食べる", "pitch", {"reading": "たべる", "pitches": [{"position": 2}, {"position": 0, "nasal": [1], "devoice": []}]}],
|
||||
["日本", "ipa", {"reading": "にほん", "transcriptions": [{"ipa": "ɲihoɰ̃"}]}],
|
||||
]
|
||||
|
||||
|
||||
def kanji_bank_1():
|
||||
return [
|
||||
["食", "ショク ジキ", "く.う た.べる", "jouyou", ["eat", "food"], {"grade": "2", "strokes": "9"}],
|
||||
["猫", "ビョウ", "ねこ", "jouyou", ["cat"], {"grade": "8"}],
|
||||
]
|
||||
|
||||
|
||||
def kanji_meta_bank_1():
|
||||
return [["食", "freq", 7], ["猫", "freq", {"value": 9, "displayValue": "9"}]]
|
||||
|
||||
|
||||
def tag_bank_1():
|
||||
return [["n", "partOfSpeech", -3, "noun", 0], ["v1", "partOfSpeech", -3, "Ichidan verb", 0], ["P", "popular", -10, "common", 10]]
|
||||
|
||||
|
||||
def main():
|
||||
entries = [
|
||||
("index.json", dumps({
|
||||
"title": "Hoshidicts Fixture",
|
||||
"revision": "fixture-1",
|
||||
"format": 3,
|
||||
"sequenced": True,
|
||||
"author": "hoshidicts tests",
|
||||
"description": "Deterministic fixture for the import equivalence test.",
|
||||
"sourceLanguage": "ja",
|
||||
"targetLanguage": "en",
|
||||
"frequencyMode": "rank-based",
|
||||
}).encode()),
|
||||
("styles.css", b".mdict-yomitan-content { color: #333; }\n"),
|
||||
("term_bank_1.json", dumps(term_bank_1()).encode()),
|
||||
("term_bank_2.json", dumps(term_bank_2()).encode()),
|
||||
("term_meta_bank_1.json", dumps(term_meta_bank_1()).encode()),
|
||||
("kanji_bank_1.json", dumps(kanji_bank_1()).encode()),
|
||||
("kanji_meta_bank_1.json", dumps(kanji_meta_bank_1()).encode()),
|
||||
("tag_bank_1.json", dumps(tag_bank_1()).encode()),
|
||||
("img/", b""),
|
||||
("img/neko.png", PNG),
|
||||
("audio/neko.txt", b"stored, not deflated"),
|
||||
]
|
||||
with zipfile.ZipFile(OUT, "w") as zf:
|
||||
for name, data in entries:
|
||||
info = zipfile.ZipInfo(name, date_time=STAMP)
|
||||
info.create_system = 3
|
||||
if name.endswith("/"):
|
||||
info.external_attr = 0o40755 << 16
|
||||
zf.writestr(info, b"")
|
||||
continue
|
||||
info.external_attr = 0o644 << 16
|
||||
info.compress_type = zipfile.ZIP_STORED if name.startswith("audio/") else zipfile.ZIP_DEFLATED
|
||||
zf.writestr(info, data, compresslevel=9)
|
||||
print(OUT, os.path.getsize(OUT), "bytes")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,11 @@
|
||||
# SHA-256 of every file the pre-DictionarySource importer (bee-san/hoshidicts main @ fa833f9) wrote for small_dict.zip.
|
||||
# index.json is hashed with "importDate":<n> replaced by "importDate":0. Regenerate only when the output format changes on purpose.
|
||||
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 .hoshidicts_6
|
||||
552b6e672c3b8e64d12e7eb43182c23234eaaea506bbd1c19b7d447e523840ad blobs.bin
|
||||
4bb3bf0755e28ae676f98c575174a62054651cc0004173bb03bf0d8b5a875c5c bloom.filter
|
||||
91de707c8eb26bff94240513c03a41b7f9995c67f5392a621fb3151aae429ef3 dict.zstd
|
||||
517a21ea1d33fa05bdc9e4655c3c8d171ded15002f297843cb17f1e9db3f045a hash.table
|
||||
b27413b159ab02a83b2f820087db41ffaf81ec2f7fb97c20493b41a1fcba3f5f media.bin
|
||||
56879507ea4d0e870990d88b906e754eaa34f47b4e634842a6f4499290f3fd2c media.idx
|
||||
91b1a6c60e52df5da7db5a1ff10a2d7d2b7d25a282b2142206f45e3cc63bb376 index.json
|
||||
4f6e1b0bf6ed1f02d3eb24617e9693284ea0607f30e92cab693e0583675bb9ca scan.idx
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,229 @@
|
||||
// Unit tests for mdict::convert_html and its helpers. Expectations follow
|
||||
// manabitan's mdx-converter.js behaviour; the JSON text is compared exactly
|
||||
// because the emitter is deterministic, and each glossary is also parsed with
|
||||
// glaze to prove it is well-formed JSON.
|
||||
#include <glaze/glaze.hpp>
|
||||
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mdict/html_to_structured.hpp"
|
||||
|
||||
namespace {
|
||||
int failures = 0;
|
||||
|
||||
void check(bool ok, const std::string& what) {
|
||||
if (!ok) {
|
||||
std::printf("FAIL %s\n", what.c_str());
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
|
||||
void check_eq(const std::string& actual, const std::string& expected, const std::string& what) {
|
||||
if (actual != expected) {
|
||||
std::printf("FAIL %s\n expected: %s\n actual: %s\n", what.c_str(), expected.c_str(), actual.c_str());
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
|
||||
const std::string root_open =
|
||||
R"({"type":"structured-content","content":{"tag":"div","data":{"tag":"div","class":"mdict-yomitan-content"},"content":[)";
|
||||
const std::string root_close = "]}}";
|
||||
|
||||
// Converts and returns only the root content array body.
|
||||
std::string body(const std::string& html, const mdict::ConvertOptions& options = {}) {
|
||||
const mdict::ConvertResult result = mdict::convert_html(html, options);
|
||||
glz::generic parsed;
|
||||
if (auto error = glz::read_json(parsed, result.glossary_json)) {
|
||||
check(false, "glossary is not valid JSON: " + result.glossary_json);
|
||||
}
|
||||
check(result.glossary_json.starts_with(root_open) && result.glossary_json.ends_with(root_close),
|
||||
"root wrapper for " + html);
|
||||
return result.glossary_json.substr(root_open.size(),
|
||||
result.glossary_json.size() - root_open.size() - root_close.size());
|
||||
}
|
||||
|
||||
void test_basic_formatting() {
|
||||
check_eq(body("<b>bold</b> plain <i>it</i>"),
|
||||
R"({"tag":"span","style":{"fontWeight":"bold"},"content":["bold"]}," plain ",)"
|
||||
R"({"tag":"span","style":{"fontStyle":"italic"},"content":["it"]})",
|
||||
"b/i become styled spans");
|
||||
check_eq(body("<h1>Head</h1><p>para</p>"),
|
||||
R"({"tag":"div","style":{"fontWeight":"bold","fontSize":"2em"},"content":["Head"]},)"
|
||||
R"({"tag":"div","content":["para"]})",
|
||||
"h1/p become divs");
|
||||
check_eq(body("<abbr>a</abbr>b<u>c</u>"),
|
||||
R"("ab",{"tag":"span","style":{"textDecorationLine":"underline"},"content":["c"]})",
|
||||
"unsupported element unwrapped and adjacent text merged");
|
||||
check_eq(body("x<br>y"), R"("x",{"tag":"br"},"y")", "br has no content");
|
||||
check_eq(body("& <b> "), "\"& <b> \xc2\xa0\"", "entities decoded");
|
||||
check_eq(body(""), "", "empty definition");
|
||||
check_eq(body("<!-- c --><script>alert(1)</script><noscript>n</noscript>t"), R"("t")", "script and comments dropped");
|
||||
}
|
||||
|
||||
void test_data_and_attributes() {
|
||||
check_eq(body(R"(<span class=" a b " id=" x " lang="ja" title="T">s</span>)"),
|
||||
R"({"tag":"span","data":{"tag":"span","class":"a b","id":"x"},"lang":"ja","title":"T","content":["s"]})",
|
||||
"class collapsed, id trimmed, lang and title kept");
|
||||
check_eq(body(R"(<td colspan="2" rowspan="3">c</td>)"), R"("c")", "stray td is dropped, its text kept");
|
||||
check_eq(body(R"(<table><tr><td colspan="2" rowspan="3">c</td><th colspan="x">h</th></tr></table>)"),
|
||||
R"({"tag":"table","content":[{"tag":"tbody","content":[{"tag":"tr","content":[)"
|
||||
R"({"tag":"td","colSpan":2,"rowSpan":3,"content":["c"]},{"tag":"th","content":["h"]}]}]}]})",
|
||||
"table with spans; non-numeric span ignored; tbody inserted by the parser");
|
||||
check_eq(body(R"(<details open><summary>s</summary>d</details>)"),
|
||||
R"({"tag":"details","open":true,"content":[{"tag":"summary","content":["s"]},"d"]})", "details open");
|
||||
check_eq(body("<ruby>漢<rp>(</rp><rt>かん</rt><rp>)</rp></ruby>"),
|
||||
R"({"tag":"ruby","content":["漢",{"tag":"rp","content":["("]},{"tag":"rt","content":["かん"]},)"
|
||||
R"~({"tag":"rp","content":[")"]}]})~",
|
||||
"ruby");
|
||||
}
|
||||
|
||||
void test_styles() {
|
||||
check_eq(body(R"(<span style="color: red; text-decoration: underline line-through; font-size:12px; bogus:1">s</span>)"),
|
||||
R"({"tag":"span","style":{"color":"red","textDecorationLine":["underline","line-through"],"fontSize":"12px"},)"
|
||||
R"("content":["s"]})",
|
||||
"inline style mapping and text-decoration splitting");
|
||||
check_eq(body(R"(<b style="font-weight: normal">s</b>)"),
|
||||
R"({"tag":"span","style":{"fontWeight":"normal"},"content":["s"]})", "inline style overrides default");
|
||||
check_eq(body(R"(<font color="red" size="3" face="Arial">f</font>)"),
|
||||
R"({"tag":"span","style":{"color":"red","fontSize":"3","fontFamily":"Arial"},"content":["f"]})", "font");
|
||||
check_eq(body(R"(<table style="color:red"><tr><td>c</td></tr></table>)"),
|
||||
R"({"tag":"table","content":[{"tag":"tbody","content":[{"tag":"tr","content":[{"tag":"td","content":["c"]}]}]}]})",
|
||||
"style dropped on tags whose schema has none");
|
||||
mdict::ConvertResult result = mdict::convert_html(R"(<style>.a{color:red}</style><style> </style>x<style>b{}</style>)", {});
|
||||
check(result.inline_stylesheets.size() == 2, "two non-empty style blocks collected");
|
||||
if (result.inline_stylesheets.size() == 2) {
|
||||
check_eq(result.inline_stylesheets[0].first, "inline/1.css", "inline stylesheet name");
|
||||
check_eq(result.inline_stylesheets[0].second, ".a{color:red}", "inline stylesheet text");
|
||||
check_eq(result.inline_stylesheets[1].first, "inline/2.css", "second inline stylesheet name");
|
||||
}
|
||||
result = mdict::convert_html(R"(<span style="background: url('img/bg.png') no-repeat">s</span>)", {});
|
||||
check(result.asset_references == std::vector<std::string>{"img/bg.png"}, "url() in inline style referenced");
|
||||
check(result.glossary_json.find(R"("background":"url(\"mdict-media/img/bg.png\") no-repeat")") != std::string::npos,
|
||||
"url() in inline style rewritten");
|
||||
}
|
||||
|
||||
void test_links() {
|
||||
check_eq(body(R"(<a href="entry://食べる">e</a>)"),
|
||||
R"({"tag":"a","href":"?query=%E9%A3%9F%E3%81%B9%E3%82%8B","content":["e"]})", "entry:// link");
|
||||
check_eq(body(R"(<a href="bword://a b">e</a>)"), R"({"tag":"a","href":"?query=a%20b","content":["e"]})", "bword://");
|
||||
check_eq(body(R"(<a href="x:term">e</a>)"), R"({"tag":"a","href":"?query=term","content":["e"]})", "x: link");
|
||||
check_eq(body(R"(<a href="https://example.com/a?b=1">e</a>)"),
|
||||
R"({"tag":"a","href":"https://example.com/a?b=1","content":["e"]})", "https kept");
|
||||
check_eq(body(R"~(<a href="javascript:alert(1)">e</a><a href="#top">f</a><a href="vbscript:x">g</a>)~"),
|
||||
R"({"tag":"a","href":"#","content":["e"]},{"tag":"a","href":"#","content":["f"]},)"
|
||||
R"({"tag":"a","href":"#","content":["g"]})",
|
||||
"script and fragment links neutralised");
|
||||
check_eq(body(R"(<a href="\img\x y.png">e</a>)"),
|
||||
R"({"tag":"a","href":"media:mdict-media/img/x%20y.png","content":["e"]})", "relative path -> media:");
|
||||
check_eq(body(R"(<a href="sound://a.spx">e</a>)"), R"({"tag":"a","href":"#","content":["e"]})",
|
||||
"sound:// is # with audio disabled");
|
||||
mdict::ConvertOptions audio;
|
||||
audio.enable_audio = true;
|
||||
check_eq(body(R"(<a href="sound://a.spx">e</a>)", audio),
|
||||
R"({"tag":"a","href":"media:mdict-media/a.spx","content":["e"]})", "sound:// with audio enabled");
|
||||
mdict::ConvertResult result = mdict::convert_html(R"(<a href="sound://a.spx">e</a><a href="img/p.png">f</a>)", {});
|
||||
check(result.asset_references == std::vector<std::string>{"a.spx", "img/p.png"}, "link asset references collected");
|
||||
check_eq(body(R"(<audio src="snd/a.mp3"></audio><video src="v.mp4">cap</video>)"),
|
||||
R"({"tag":"a","href":"media:mdict-media/snd/a.mp3","content":["audio"]},)"
|
||||
R"({"tag":"a","href":"media:mdict-media/v.mp4","content":["cap"]})",
|
||||
"audio/video become links");
|
||||
}
|
||||
|
||||
void test_images() {
|
||||
check_eq(body(R"(<img src="img/pic.png" width="10" height="20" alt="pic" title="t" class="c">)"),
|
||||
R"({"tag":"img","path":"mdict-media/img/pic.png","data":{"tag":"img","class":"c"},"width":10,"height":20,)"
|
||||
R"("title":"t","alt":"pic"})",
|
||||
"img attributes");
|
||||
check_eq(body(R"(<img src="javascript:x"><img>)"), "", "img without a usable source dropped");
|
||||
mdict::ConvertResult result =
|
||||
mdict::convert_html(R"(<img src="data:image/png;base64,iVBORw0KGgo="><a href="data:text/plain,hi%20there">t</a>)", {});
|
||||
check(result.embedded_assets.size() == 2, "two embedded assets");
|
||||
if (result.embedded_assets.size() == 2) {
|
||||
const auto& png = result.embedded_assets[0];
|
||||
check(png.path.starts_with("mdict-media/embedded/image/") && png.path.ends_with(".png"), "png asset path " + png.path);
|
||||
check(png.data == std::vector<uint8_t>{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}, "base64 decoded");
|
||||
const auto& txt = result.embedded_assets[1];
|
||||
check(txt.path.starts_with("mdict-media/embedded/text/") && txt.path.ends_with(".bin"), "text asset path " + txt.path);
|
||||
check(std::string(txt.data.begin(), txt.data.end()) == "hi there", "percent-decoded payload");
|
||||
check(result.glossary_json.find("\"path\":\"" + png.path + "\"") != std::string::npos, "img path points at asset");
|
||||
check(result.glossary_json.find("\"href\":\"media:" + txt.path + "\"") != std::string::npos, "link points at asset");
|
||||
}
|
||||
check_eq(body(R"(<img src="data:image/png;base64,@@@">)"), "", "malformed data URL dropped");
|
||||
// Identical data yields one asset with the same name whichever entry saw it first.
|
||||
mdict::ConvertResult twice = mdict::convert_html(
|
||||
R"(<img src="data:image/png;base64,iVBORw0KGgo="><img src="data:image/png;base64,iVBORw0KGgo=">)", {});
|
||||
check(twice.embedded_assets.size() == 1, "identical embedded assets deduplicated");
|
||||
}
|
||||
|
||||
void test_depth_limit() {
|
||||
std::string html;
|
||||
for (int i = 0; i < 40; ++i) {
|
||||
html += "<div>";
|
||||
}
|
||||
html += "deep";
|
||||
for (int i = 0; i < 40; ++i) {
|
||||
html += "</div>";
|
||||
}
|
||||
mdict::ConvertOptions options;
|
||||
options.max_depth = 20;
|
||||
const std::string content = body(html, options);
|
||||
size_t divs = 0;
|
||||
for (size_t pos = content.find("{\"tag\":\"div\""); pos != std::string::npos;
|
||||
pos = content.find("{\"tag\":\"div\"", pos + 1)) {
|
||||
divs++;
|
||||
}
|
||||
check(divs == 19, "nesting flattened to the depth limit (19 nested divs under the root), got " + std::to_string(divs));
|
||||
check(content.find("deep") != std::string::npos, "deep text kept");
|
||||
}
|
||||
|
||||
void test_apply_stylesheet() {
|
||||
const std::string sheet = "1\n<b>\n</b>\n2\n<i>\n</i>\n";
|
||||
check_eq(mdict::apply_stylesheet("`1`to eat`2` (ichidan)", sheet), "<b>to eat</b><i> (ichidan)</i>",
|
||||
"backtick styles expanded");
|
||||
check_eq(mdict::apply_stylesheet("plain `1`x\n`9`kept", sheet), "plain <b>x</b>\r\nkept",
|
||||
"line-ending segment, unknown style keeps text");
|
||||
check_eq(mdict::apply_stylesheet("no markers", sheet), "no markers", "no markers");
|
||||
check_eq(mdict::apply_stylesheet("`1`x", ""), "`1`x", "no stylesheet");
|
||||
check_eq(mdict::apply_stylesheet("a ` b `1`c", sheet), "a ` b <b>c</b>", "stray backtick is text");
|
||||
}
|
||||
|
||||
void test_paths_and_css() {
|
||||
check_eq(mdict::normalize_asset_path("\\Images\\BG.PNG?x=1#frag"), "Images/BG.PNG", "backslashes, query, fragment");
|
||||
check_eq(mdict::normalize_asset_path("/images/space%20name.png"), "images/space name.png", "percent decoded");
|
||||
check_eq(mdict::normalize_asset_path("images/bad%ZZname.png"), "images/bad%ZZname.png", "malformed escape kept");
|
||||
check_eq(mdict::normalize_asset_path("../../etc/passwd"), "etc/passwd", "traversal collapsed");
|
||||
check_eq(mdict::normalize_asset_path("a/./b/../c"), "a/c", "dot segments");
|
||||
check_eq(mdict::normalize_asset_path("entry://x"), "", "scheme is not an asset");
|
||||
check_eq(mdict::normalize_asset_path("//host/x"), "", "protocol-relative is not an asset");
|
||||
check_eq(mdict::normalize_asset_path("file:///abs/x.png"), "abs/x.png", "file scheme stripped");
|
||||
check_eq(mdict::normalize_asset_path("../images/bg.png", "styles/extra.css"), "images/bg.png",
|
||||
"relative to stylesheet");
|
||||
check_eq(mdict::normalize_asset_path("./bg.png", "top.css"), "bg.png", "relative to top-level stylesheet");
|
||||
|
||||
std::vector<std::string> refs;
|
||||
check_eq(mdict::rewrite_css_asset_urls(R"(a{background:url("../images/bg.png") ;b:URL( 'x.png' );c:url(data:x,y)})",
|
||||
"mdict-media/", "styles/extra.css", refs),
|
||||
R"(a{background:url("mdict-media/images/bg.png") ;b:url("mdict-media/x.png");c:url(data:x,y)})",
|
||||
"css url rewriting (only ./ and ../ resolve against the stylesheet path)");
|
||||
check(refs == std::vector<std::string>{"images/bg.png", "x.png"}, "css references collected");
|
||||
check_eq(mdict::rewrite_css_asset_urls("url(", "mdict-media/", "", refs), "url(", "unterminated url() kept");
|
||||
check_eq(mdict::encode_uri_component("a b/漢-_.!~*'()"), "a%20b%2F%E6%BC%A2-_.!~*'()", "encodeURIComponent");
|
||||
}
|
||||
}
|
||||
|
||||
int main() {
|
||||
test_basic_formatting();
|
||||
test_data_and_attributes();
|
||||
test_styles();
|
||||
test_links();
|
||||
test_images();
|
||||
test_depth_limit();
|
||||
test_apply_stylesheet();
|
||||
test_paths_and_css();
|
||||
if (failures == 0) {
|
||||
std::printf("ok\n");
|
||||
}
|
||||
return failures == 0 ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
// Regression guard for the importer's output format.
|
||||
//
|
||||
// Imports tests/fixtures/yomitan/small_dict.zip into a fresh directory and
|
||||
// checks the SHA-256 of every output file against tests/fixtures/yomitan/
|
||||
// golden.sha256, which was produced by the importer before the DictionarySource
|
||||
// seam existed. index.json is compared with its importDate removed. The import
|
||||
// is run twice, normal and low_ram, and both must match the golden.
|
||||
//
|
||||
// import_equivalence_test <small_dict.zip> <golden.sha256>
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <random>
|
||||
#include <regex>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "hoshidicts/importer.hpp"
|
||||
|
||||
namespace {
|
||||
// FIPS 180-4 SHA-256, enough for a test.
|
||||
class Sha256 {
|
||||
public:
|
||||
void update(const uint8_t* data, size_t len) {
|
||||
total_ += len;
|
||||
while (len > 0) {
|
||||
const size_t take = std::min(len, buffer_.size() - buffered_);
|
||||
std::memcpy(buffer_.data() + buffered_, data, take);
|
||||
buffered_ += take;
|
||||
data += take;
|
||||
len -= take;
|
||||
if (buffered_ == buffer_.size()) {
|
||||
block(buffer_.data());
|
||||
buffered_ = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::string hex() {
|
||||
const uint64_t bits = total_ * 8;
|
||||
const uint8_t one = 0x80;
|
||||
update(&one, 1);
|
||||
const uint8_t zero = 0;
|
||||
while (buffered_ != 56) {
|
||||
update(&zero, 1);
|
||||
}
|
||||
std::array<uint8_t, 8> length{};
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
length[static_cast<size_t>(i)] = static_cast<uint8_t>(bits >> (56 - 8 * i));
|
||||
}
|
||||
update(length.data(), length.size());
|
||||
std::string out;
|
||||
static const char digits[] = "0123456789abcdef";
|
||||
for (uint32_t word : state_) {
|
||||
for (int shift = 28; shift >= 0; shift -= 4) {
|
||||
out += digits[(word >> shift) & 0xf];
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private:
|
||||
static uint32_t rotr(uint32_t x, int n) { return (x >> n) | (x << (32 - n)); }
|
||||
|
||||
void block(const uint8_t* p) {
|
||||
static constexpr std::array<uint32_t, 64> k = {
|
||||
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
|
||||
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
|
||||
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
||||
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
|
||||
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
|
||||
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
||||
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
|
||||
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};
|
||||
std::array<uint32_t, 64> w{};
|
||||
for (size_t i = 0; i < 16; ++i) {
|
||||
w[i] = (uint32_t{p[4 * i]} << 24) | (uint32_t{p[4 * i + 1]} << 16) | (uint32_t{p[4 * i + 2]} << 8) |
|
||||
uint32_t{p[4 * i + 3]};
|
||||
}
|
||||
for (size_t i = 16; i < 64; ++i) {
|
||||
const uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3);
|
||||
const uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10);
|
||||
w[i] = w[i - 16] + s0 + w[i - 7] + s1;
|
||||
}
|
||||
uint32_t a = state_[0], b = state_[1], c = state_[2], d = state_[3];
|
||||
uint32_t e = state_[4], f = state_[5], g = state_[6], h = state_[7];
|
||||
for (size_t i = 0; i < 64; ++i) {
|
||||
const uint32_t s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25);
|
||||
const uint32_t ch = (e & f) ^ (~e & g);
|
||||
const uint32_t t1 = h + s1 + ch + k[i] + w[i];
|
||||
const uint32_t s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22);
|
||||
const uint32_t maj = (a & b) ^ (a & c) ^ (b & c);
|
||||
const uint32_t t2 = s0 + maj;
|
||||
h = g;
|
||||
g = f;
|
||||
f = e;
|
||||
e = d + t1;
|
||||
d = c;
|
||||
c = b;
|
||||
b = a;
|
||||
a = t1 + t2;
|
||||
}
|
||||
state_[0] += a;
|
||||
state_[1] += b;
|
||||
state_[2] += c;
|
||||
state_[3] += d;
|
||||
state_[4] += e;
|
||||
state_[5] += f;
|
||||
state_[6] += g;
|
||||
state_[7] += h;
|
||||
}
|
||||
|
||||
std::array<uint32_t, 8> state_ = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
|
||||
0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19};
|
||||
std::array<uint8_t, 64> buffer_{};
|
||||
size_t buffered_ = 0;
|
||||
uint64_t total_ = 0;
|
||||
};
|
||||
|
||||
std::string read_file(const std::filesystem::path& path) {
|
||||
std::ifstream in(path, std::ios::binary);
|
||||
std::ostringstream out;
|
||||
out << in.rdbuf();
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string sha256_hex(std::string data) {
|
||||
Sha256 sha;
|
||||
sha.update(reinterpret_cast<const uint8_t*>(data.data()), data.size());
|
||||
return sha.hex();
|
||||
}
|
||||
|
||||
// The import date is the only thing that legitimately differs between runs.
|
||||
std::string strip_import_date(std::string json) {
|
||||
static const std::regex import_date(R"("importDate":\d+)");
|
||||
return std::regex_replace(json, import_date, R"("importDate":0)");
|
||||
}
|
||||
|
||||
std::map<std::string, std::string> load_golden(const std::filesystem::path& path) {
|
||||
std::map<std::string, std::string> golden;
|
||||
std::ifstream in(path);
|
||||
std::string line;
|
||||
while (std::getline(in, line)) {
|
||||
if (line.empty() || line[0] == '#') {
|
||||
continue;
|
||||
}
|
||||
const size_t split = line.find_first_of(" \t");
|
||||
if (split == std::string::npos) {
|
||||
continue;
|
||||
}
|
||||
const std::string hash = line.substr(0, split);
|
||||
const size_t name_start = line.find_first_not_of(" \t*", split);
|
||||
golden[line.substr(name_start)] = hash;
|
||||
}
|
||||
return golden;
|
||||
}
|
||||
|
||||
std::filesystem::path fresh_temp_dir(const char* tag) {
|
||||
std::random_device rd;
|
||||
const auto dir = std::filesystem::temp_directory_path() /
|
||||
("hoshidicts-eq-" + std::string(tag) + "-" + std::to_string(rd()));
|
||||
std::filesystem::create_directories(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
int check_import(const std::string& fixture, const std::map<std::string, std::string>& golden, bool low_ram) {
|
||||
const auto out_dir = fresh_temp_dir(low_ram ? "lowram" : "normal");
|
||||
int failures = 0;
|
||||
const ImportResult result = dictionary_importer::import(fixture, out_dir.string(), low_ram);
|
||||
if (!result.success) {
|
||||
std::printf("FAIL low_ram=%d import failed: %s\n", low_ram, result.error.c_str());
|
||||
std::filesystem::remove_all(out_dir);
|
||||
return 1;
|
||||
}
|
||||
const auto dict_dir = out_dir / result.title;
|
||||
|
||||
std::map<std::string, std::string> actual;
|
||||
for (const auto& entry : std::filesystem::directory_iterator(dict_dir)) {
|
||||
std::string data = read_file(entry.path());
|
||||
const std::string name = entry.path().filename().string();
|
||||
if (name == "index.json") {
|
||||
data = strip_import_date(std::move(data));
|
||||
}
|
||||
actual[name] = sha256_hex(std::move(data));
|
||||
}
|
||||
|
||||
for (const auto& [name, hash] : golden) {
|
||||
auto it = actual.find(name);
|
||||
if (it == actual.end()) {
|
||||
std::printf("FAIL low_ram=%d missing output %s\n", low_ram, name.c_str());
|
||||
failures++;
|
||||
} else if (it->second != hash) {
|
||||
std::printf("FAIL low_ram=%d %s\n expected %s\n actual %s\n", low_ram, name.c_str(), hash.c_str(),
|
||||
it->second.c_str());
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
for (const auto& [name, hash] : actual) {
|
||||
if (!golden.contains(name)) {
|
||||
std::printf("FAIL low_ram=%d unexpected output %s (%s)\n", low_ram, name.c_str(), hash.c_str());
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
|
||||
std::filesystem::remove_all(out_dir);
|
||||
if (failures == 0) {
|
||||
std::printf("ok low_ram=%d %zu files match\n", low_ram, golden.size());
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc < 3) {
|
||||
std::printf("usage: %s <small_dict.zip> <golden.sha256>\n", argv[0]);
|
||||
return 2;
|
||||
}
|
||||
// Known-answer check so a broken hash cannot make everything "match".
|
||||
if (sha256_hex("abc") != "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") {
|
||||
std::printf("FAIL sha256 self test\n");
|
||||
return 1;
|
||||
}
|
||||
const auto golden = load_golden(argv[2]);
|
||||
if (golden.empty()) {
|
||||
std::printf("FAIL no golden hashes in %s\n", argv[2]);
|
||||
return 1;
|
||||
}
|
||||
int failures = 0;
|
||||
failures += check_import(argv[1], golden, false);
|
||||
failures += check_import(argv[1], golden, true);
|
||||
return failures == 0 ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// Differential test: skip_json_container must stop exactly where glaze's raw_json_view
|
||||
// skip stops (or fail exactly when it fails) on generated arrays with nested
|
||||
// values, strings full of quotes, brackets and backslash runs straddling the
|
||||
// 64-byte blocks, multibyte text, and truncated input.
|
||||
#include "json/json_skip.hpp"
|
||||
#include <cstdio>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
static std::string gen(std::mt19937& rng, int depth) {
|
||||
std::uniform_int_distribution<int> kind(0, 9);
|
||||
std::string s = "[";
|
||||
int n = std::uniform_int_distribution<int>(0, 6)(rng);
|
||||
for (int i = 0; i < n; ++i) {
|
||||
if (i) s += ",";
|
||||
int k = kind(rng);
|
||||
if (k < 4 || depth > 4) {
|
||||
// string with random content including quotes, brackets, backslash runs, multibyte
|
||||
s += '"';
|
||||
int len = std::uniform_int_distribution<int>(0, 150)(rng);
|
||||
for (int j = 0; j < len; ++j) {
|
||||
int c = std::uniform_int_distribution<int>(0, 12)(rng);
|
||||
switch (c) {
|
||||
case 0: s += "\\\""; break;
|
||||
case 1: s += "\\\\"; break;
|
||||
case 2: { int run = std::uniform_int_distribution<int>(1, 9)(rng); for (int r = 0; r < run; ++r) s += "\\\\"; if (rng() & 1) s += "\\\""; break; }
|
||||
case 3: s += "["; break;
|
||||
case 4: s += "]"; break;
|
||||
case 5: s += "\\u30c6"; break;
|
||||
case 6: s += "テスト"; break;
|
||||
case 7: s += "\\n"; break;
|
||||
default: s += char('a' + (rng() % 26)); break;
|
||||
}
|
||||
}
|
||||
s += '"';
|
||||
} else if (k < 6) {
|
||||
s += std::to_string(int(rng() % 1000));
|
||||
} else if (k < 8) {
|
||||
s += gen(rng, depth + 1);
|
||||
} else {
|
||||
s += "{\"k\":" + gen(rng, depth + 1) + ",\"x\":\"]]][[\\\"\"}";
|
||||
}
|
||||
}
|
||||
s += "]";
|
||||
return s;
|
||||
}
|
||||
|
||||
int main() {
|
||||
std::mt19937 rng(12345);
|
||||
long cases = 0, mismatches = 0;
|
||||
for (int iter = 0; iter < 200000; ++iter) {
|
||||
std::string doc = gen(rng, 0);
|
||||
if (iter % 3 == 1) {
|
||||
// Object at the top level: wrap the array's items as values.
|
||||
doc = "{\"a\":" + doc + ",\"b\":{\"c\":\"}}]\",\"d\":[" + gen(rng, 3) + "]},\"e\":" + gen(rng, 2) + "}";
|
||||
}
|
||||
// pad with a tail so the value is followed by more input, like in a bank
|
||||
std::string buf = doc + ",\"tail\",[1,2]]";
|
||||
// sometimes truncate to test the unexpected-end path
|
||||
bool truncated = (iter % 7 == 0);
|
||||
if (truncated) buf = doc.substr(0, std::uniform_int_distribution<size_t>(0, doc.size() - 1)(rng));
|
||||
const char* begin = buf.data();
|
||||
const char* end = buf.data() + buf.size();
|
||||
const char* mine = hoshidicts::skip_json_container(begin, end);
|
||||
glz::raw_json_view ref;
|
||||
glz::context ctx{};
|
||||
const char* it = begin;
|
||||
glz::from<glz::JSON, glz::raw_json_view>::op<glz::opts{}>(ref, ctx, it, end);
|
||||
bool ref_ok = !bool(ctx.error);
|
||||
const char* ref_end = ref_ok ? it : nullptr;
|
||||
++cases;
|
||||
if ((mine == nullptr) != (ref_end == nullptr) || (mine && mine != ref_end)) {
|
||||
++mismatches;
|
||||
if (mismatches <= 5) {
|
||||
std::printf("MISMATCH truncated=%d mine=%ld ref=%ld doc=%.*s\n", truncated, mine ? long(mine - begin) : -1L,
|
||||
ref_end ? long(ref_end - begin) : -1L, int(std::min<size_t>(buf.size(), 300)), buf.data());
|
||||
}
|
||||
}
|
||||
}
|
||||
std::printf("cases=%ld mismatches=%ld\n", cases, mismatches);
|
||||
return mismatches ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
// Long-key scan index (src/scan_index.hpp): a dictionary with keys longer than
|
||||
// the scan length must still have them found when the input begins like one,
|
||||
// an inflected form of such a key must be found through deinflection, and a
|
||||
// scan shorter than eight code points must not extend at all.
|
||||
#include "hoshidicts/deinflector.hpp"
|
||||
#include "hoshidicts/importer.hpp"
|
||||
#include "hoshidicts/lookup.hpp"
|
||||
#include "hoshidicts/query.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
uint32_t crc32(const std::string& data) {
|
||||
uint32_t crc = 0xFFFFFFFFu;
|
||||
for (unsigned char c : data) {
|
||||
crc ^= c;
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
crc = (crc >> 1) ^ (0xEDB88320u & (0u - (crc & 1u)));
|
||||
}
|
||||
}
|
||||
return ~crc;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void put(std::string& out, T value) {
|
||||
char buf[sizeof(T)];
|
||||
std::memcpy(buf, &value, sizeof(T));
|
||||
out.append(buf, sizeof(T));
|
||||
}
|
||||
|
||||
// A stored (method 0) ZIP: local headers, then the central directory, then EOCD.
|
||||
std::string build_zip(const std::vector<std::pair<std::string, std::string>>& files) {
|
||||
std::string out;
|
||||
std::string central;
|
||||
for (const auto& [name, data] : files) {
|
||||
const uint32_t offset = static_cast<uint32_t>(out.size());
|
||||
const uint32_t crc = crc32(data);
|
||||
const uint32_t size = static_cast<uint32_t>(data.size());
|
||||
put<uint32_t>(out, 0x04034b50);
|
||||
put<uint16_t>(out, 20);
|
||||
put<uint16_t>(out, 0);
|
||||
put<uint16_t>(out, 0);
|
||||
put<uint16_t>(out, 0);
|
||||
put<uint16_t>(out, 0);
|
||||
put<uint32_t>(out, crc);
|
||||
put<uint32_t>(out, size);
|
||||
put<uint32_t>(out, size);
|
||||
put<uint16_t>(out, static_cast<uint16_t>(name.size()));
|
||||
put<uint16_t>(out, 0);
|
||||
out += name;
|
||||
out += data;
|
||||
|
||||
put<uint32_t>(central, 0x02014b50);
|
||||
put<uint16_t>(central, 20);
|
||||
put<uint16_t>(central, 20);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint32_t>(central, crc);
|
||||
put<uint32_t>(central, size);
|
||||
put<uint32_t>(central, size);
|
||||
put<uint16_t>(central, static_cast<uint16_t>(name.size()));
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint32_t>(central, 0);
|
||||
put<uint32_t>(central, offset);
|
||||
central += name;
|
||||
}
|
||||
const uint32_t central_offset = static_cast<uint32_t>(out.size());
|
||||
out += central;
|
||||
put<uint32_t>(out, 0x06054b50);
|
||||
put<uint16_t>(out, 0);
|
||||
put<uint16_t>(out, 0);
|
||||
put<uint16_t>(out, static_cast<uint16_t>(files.size()));
|
||||
put<uint16_t>(out, static_cast<uint16_t>(files.size()));
|
||||
put<uint32_t>(out, static_cast<uint32_t>(central.size()));
|
||||
put<uint32_t>(out, central_offset);
|
||||
put<uint16_t>(out, 0);
|
||||
return out;
|
||||
}
|
||||
|
||||
int failures = 0;
|
||||
|
||||
void check(bool ok, const std::string& what) {
|
||||
if (!ok) {
|
||||
++failures;
|
||||
std::fprintf(stderr, "FAIL: %s\n", what.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
bool has_expression(const std::vector<LookupResult>& results, std::string_view expression) {
|
||||
for (const auto& r : results) {
|
||||
if (r.term.expression == expression) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
size_t cp_len(std::string_view s) {
|
||||
size_t n = 0;
|
||||
for (unsigned char c : s) n += (c & 0xC0) != 0x80;
|
||||
return n;
|
||||
}
|
||||
|
||||
std::string term(std::string_view expression, std::string_view reading, std::string_view rules = "") {
|
||||
std::string t = "[\"";
|
||||
t += expression;
|
||||
t += "\",\"";
|
||||
t += reading;
|
||||
t += "\",\"\",\"";
|
||||
t += rules;
|
||||
t += "\",1,[\"gloss\"],0,\"\"]";
|
||||
return t;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
const std::filesystem::path root =
|
||||
std::filesystem::temp_directory_path() / ("hoshidicts-long-key-test-" + std::to_string(std::random_device{}()));
|
||||
std::filesystem::remove_all(root);
|
||||
std::filesystem::create_directories(root);
|
||||
|
||||
// A 27-code-point proverb whose reading is longer still; a 17-code-point
|
||||
// phrase ending in a verb (rules v1 so 〜られなかった deinflects); a
|
||||
// two-character key sharing the proverb's first characters but not its first
|
||||
// eight; and an ordinary verb.
|
||||
const std::string proverb = "身体髪膚これを父母に受くあえて毀傷せざるは孝の始めなり";
|
||||
const std::string proverb_reading = "しんたいはっぷこれをふぼにうくあえてきしょうせざるはこうのはじめなり";
|
||||
const std::string phrase = "自分の思うところをはっきりと述べる";
|
||||
const std::string bank = "[" + term(proverb, proverb_reading) + "," +
|
||||
term(phrase, "じぶんのおもうところをはっきりとのべる", "v1") + "," + term("身体", "しんたい") +
|
||||
"," + term("食べる", "たべる", "v1") + "]";
|
||||
const std::string index = R"({"title":"long-key-test","format":3,"revision":"1"})";
|
||||
const std::string zip = build_zip({{"index.json", index}, {"term_bank_1.json", bank}});
|
||||
const std::filesystem::path zip_path = root / "long-key-test.zip";
|
||||
{
|
||||
std::ofstream f(zip_path, std::ios::binary);
|
||||
f.write(zip.data(), static_cast<std::streamsize>(zip.size()));
|
||||
}
|
||||
|
||||
const std::filesystem::path out_dir = root / "out";
|
||||
std::filesystem::create_directories(out_dir);
|
||||
const auto result = dictionary_importer::import(zip_path.string(), out_dir.string());
|
||||
check(result.success, "import succeeded: " + result.error);
|
||||
if (!result.success) {
|
||||
return 1;
|
||||
}
|
||||
const std::filesystem::path dict_dir = out_dir / result.summary.title;
|
||||
check(std::filesystem::is_regular_file(dict_dir / "scan.idx"), "importer wrote scan.idx");
|
||||
|
||||
DictionaryQuery query;
|
||||
check(query.add_term_dict(dict_dir.string()), "add_term_dict");
|
||||
check(query.max_long_key_length() == cp_len(proverb_reading),
|
||||
"max_long_key_length is the reading's length " + std::to_string(cp_len(proverb_reading)) + ", got " +
|
||||
std::to_string(query.max_long_key_length()));
|
||||
check(query.long_key_length(proverb) == cp_len(proverb), "long_key_length(proverb) is its length");
|
||||
check(query.long_key_length("身体髪膚これを父") == cp_len(proverb), "eight-code-point prefix resolves the proverb");
|
||||
check(query.long_key_length(proverb_reading) == cp_len(proverb_reading), "the reading is indexed too");
|
||||
check(query.long_key_length("身体") == 0, "shorter than eight code points resolves nothing");
|
||||
check(query.long_key_length("食べるのが好きです") == 0, "a prefix of no long key resolves nothing");
|
||||
|
||||
Deinflector deinflector;
|
||||
Lookup lookup(query, deinflector);
|
||||
const std::string tail = "と昔から言われている。";
|
||||
|
||||
// Scan 16 alone cannot see a 26-code-point key; the index extends it.
|
||||
{
|
||||
const auto results = lookup.lookup(proverb + tail, 16, 16);
|
||||
check(has_expression(results, proverb), "proverb found with scan 16 through the long-key index");
|
||||
check(has_expression(results, "身体"), "shorter matches are still reported");
|
||||
bool matched_whole = false;
|
||||
for (const auto& r : results) {
|
||||
if (r.term.expression == proverb) {
|
||||
matched_whole = r.matched == proverb;
|
||||
}
|
||||
}
|
||||
check(matched_whole, "the proverb's matched text is the whole proverb");
|
||||
}
|
||||
|
||||
// An inflected form of the 17-code-point verb phrase: surface 22 code points.
|
||||
{
|
||||
const std::string inflected = "自分の思うところをはっきりと述べられなかった";
|
||||
const auto results = lookup.lookup(inflected + tail, 16, 16);
|
||||
check(has_expression(results, phrase), "inflected long phrase found through deinflection");
|
||||
}
|
||||
|
||||
// Not a long-key prefix: the ordinary scan result only.
|
||||
{
|
||||
const auto results = lookup.lookup("食べられなかった" + proverb, 16, 16);
|
||||
check(has_expression(results, "食べる"), "ordinary verb still found");
|
||||
check(!has_expression(results, proverb), "no extension for a prefix that is not a long key");
|
||||
}
|
||||
|
||||
// Scan shorter than the eight-code-point prefix never extends.
|
||||
{
|
||||
const auto results = lookup.lookup(proverb + tail, 16, 4);
|
||||
check(has_expression(results, "身体"), "scan 4 finds the two-character key");
|
||||
check(!has_expression(results, proverb), "scan 4 does not extend to the proverb");
|
||||
}
|
||||
|
||||
// Scan 8 is the shortest that can extend.
|
||||
{
|
||||
const auto results = lookup.lookup(proverb + tail, 16, 8);
|
||||
check(has_expression(results, proverb), "scan 8 extends to the proverb");
|
||||
}
|
||||
|
||||
// The text itself bounds the extension.
|
||||
{
|
||||
const auto results = lookup.lookup(proverb.substr(0, proverb.find("せざる")), 16, 16);
|
||||
check(!has_expression(results, proverb), "a truncated proverb is not matched");
|
||||
check(has_expression(results, "身体"), "truncated input still yields the short key");
|
||||
}
|
||||
|
||||
// Per-dictionary lookup consults only that dictionary's index.
|
||||
{
|
||||
const auto results = lookup.lookup_dictionary(proverb + tail, dict_dir.string(), 16, 16);
|
||||
check(has_expression(results, proverb), "lookup_dictionary extends through its own index");
|
||||
const std::string other = dict_dir.string() + "-missing";
|
||||
check(query.long_key_length(proverb, &other) == 0, "an unknown dictionary path resolves nothing");
|
||||
}
|
||||
|
||||
std::filesystem::remove_all(root);
|
||||
if (failures) {
|
||||
std::fprintf(stderr, "%d failure(s)\n", failures);
|
||||
return 1;
|
||||
}
|
||||
std::puts("long-key scan: ok");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
// Unit tests for mdict::Reader against the fixtures written by
|
||||
// tests/fixtures/mdict/gen_fixtures.py.
|
||||
//
|
||||
// mdict_reader_test <tests/fixtures/mdict>
|
||||
#include <cstdio>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iterator>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "mdict/mdict_reader.hpp"
|
||||
#include "mdict/ripemd128.hpp"
|
||||
|
||||
namespace {
|
||||
int failures = 0;
|
||||
std::filesystem::path fixtures;
|
||||
|
||||
void check(bool ok, const std::string& what) {
|
||||
if (!ok) {
|
||||
std::printf("FAIL %s\n", what.c_str());
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void check_eq(const T& actual, const T& expected, const std::string& what) {
|
||||
if (actual != expected) {
|
||||
std::printf("FAIL %s\n", what.c_str());
|
||||
if constexpr (std::is_convertible_v<T, std::string>) {
|
||||
std::printf(" expected: %s\n actual: %s\n", std::string(expected).c_str(), std::string(actual).c_str());
|
||||
} else {
|
||||
std::printf(" expected: %s\n actual: %s\n", std::to_string(expected).c_str(),
|
||||
std::to_string(actual).c_str());
|
||||
}
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
|
||||
std::string hex(const std::array<uint8_t, 16>& digest) {
|
||||
std::string out;
|
||||
for (uint8_t b : digest) {
|
||||
char buf[3];
|
||||
std::snprintf(buf, sizeof buf, "%02x", b);
|
||||
out += buf;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Reads every record of the file through the block-cursor API.
|
||||
std::vector<std::pair<std::string, std::string>> all_records(const mdict::Reader& reader, bool text) {
|
||||
const std::vector<mdict::KeyEntry> keys = reader.read_all_keys();
|
||||
std::vector<std::pair<std::string, std::string>> out;
|
||||
size_t block_index = static_cast<size_t>(-1);
|
||||
std::vector<uint8_t> block;
|
||||
for (size_t i = 0; i < keys.size(); ++i) {
|
||||
const uint64_t next = i + 1 < keys.size() ? keys[i + 1].record_offset : reader.record_space_size();
|
||||
const size_t wanted = reader.record_block_for(keys[i].record_offset);
|
||||
if (wanted != block_index) {
|
||||
block = reader.read_record_block(wanted);
|
||||
block_index = wanted;
|
||||
}
|
||||
const std::string_view record = reader.record_in_block(block, block_index, keys[i].record_offset, next);
|
||||
out.emplace_back(keys[i].key, text ? reader.record_text(record) : std::string(record));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
void expect_text_fixture(const mdict::Reader& reader, const std::string& name) {
|
||||
check_eq<uint64_t>(reader.key_count(), 4, name + ": key count");
|
||||
const auto records = all_records(reader, true);
|
||||
check_eq<size_t>(records.size(), 4, name + ": record count");
|
||||
if (records.size() != 4) {
|
||||
return;
|
||||
}
|
||||
check_eq<std::string>(records[0].first, "alpha", name + ": key 0");
|
||||
check_eq<std::string>(records[0].second, "first definition", name + ": record 0");
|
||||
check_eq<std::string>(records[1].first, "beta", name + ": key 1");
|
||||
check_eq<std::string>(records[1].second, "second\ndefinition with newline", name + ": record 1");
|
||||
check_eq<std::string>(records[2].second, "third", name + ": record 2");
|
||||
check_eq<std::string>(records[3].first, "日本語", name + ": unicode key");
|
||||
check_eq<std::string>(records[3].second, "Japanese text \"quoted\"", name + ": record 3");
|
||||
}
|
||||
|
||||
void test_ripemd128() {
|
||||
check_eq<std::string>(hex(mdict::ripemd128(nullptr, 0)), "cdf26213a150dc3ecb610f18f6b38b46", "ripemd128 empty");
|
||||
const std::string abc = "abc";
|
||||
check_eq<std::string>(hex(mdict::ripemd128(reinterpret_cast<const uint8_t*>(abc.data()), abc.size())),
|
||||
"c14a12199c66e4ba84636b0f69144c77", "ripemd128 abc");
|
||||
const std::string md = "message digest";
|
||||
check_eq<std::string>(hex(mdict::ripemd128(reinterpret_cast<const uint8_t*>(md.data()), md.size())),
|
||||
"9e327b3d6e523062afc1132d7df9d1b8", "ripemd128 message digest");
|
||||
const std::string eighty = "12345678901234567890123456789012345678901234567890123456789012345678901234567890";
|
||||
check_eq<std::string>(hex(mdict::ripemd128(reinterpret_cast<const uint8_t*>(eighty.data()), eighty.size())),
|
||||
"3f45ef194732c2dbb2c4a2c769795fa3", "ripemd128 two blocks");
|
||||
}
|
||||
|
||||
void test_v2_text() {
|
||||
mdict::Reader reader;
|
||||
reader.open(fixtures / "v2_utf8_zlib_text.mdx");
|
||||
const auto& h = reader.header();
|
||||
check(h.kind == mdict::Kind::Mdx, "v2 text: kind");
|
||||
check_eq<std::string>(h.engine_version, "2.0", "v2 text: engine version");
|
||||
check(h.encoding == mdict::Encoding::Utf8, "v2 text: encoding");
|
||||
check_eq<std::string>(h.format, "Text", "v2 text: format");
|
||||
check_eq<std::string>(h.title, "Text Fixture", "v2 text: title");
|
||||
check_eq<std::string>(h.description, "A <b>text</b> fixture & entities", "v2 text: description unescaped");
|
||||
check_eq<int>(h.encrypted, 0, "v2 text: encrypted");
|
||||
check(h.compact, "v2 text: compact");
|
||||
check_eq<size_t>(reader.key_blocks().size(), 2, "v2 text: key block count (3 per block)");
|
||||
check_eq<std::string>(reader.key_blocks()[0].first_key, "alpha", "v2 text: first key of block 0");
|
||||
check_eq<std::string>(reader.key_blocks()[0].last_key, "gamma", "v2 text: last key of block 0");
|
||||
expect_text_fixture(reader, "v2 text");
|
||||
}
|
||||
|
||||
void test_v2_lzo_html() {
|
||||
mdict::Reader reader;
|
||||
reader.open(fixtures / "v2_utf8_lzo_html.mdx");
|
||||
check_eq<std::string>(reader.header().format, "Html", "v2 html: format");
|
||||
check_eq<std::string>(reader.header().stylesheet, "1\n<b>\n</b>\n2\n<i>\n</i>\n", "v2 html: stylesheet");
|
||||
check_eq<uint64_t>(reader.key_count(), 9, "v2 html: key count");
|
||||
check_eq<size_t>(reader.key_blocks().size(), 3, "v2 html: key blocks");
|
||||
check(reader.record_blocks().size() >= 2, "v2 html: several record blocks");
|
||||
const auto records = all_records(reader, true);
|
||||
check_eq<size_t>(records.size(), 9, "v2 html: record count");
|
||||
if (records.size() != 9) {
|
||||
return;
|
||||
}
|
||||
check_eq<std::string>(records[1].first, "alias", "v2 html: alias key");
|
||||
check_eq<std::string>(records[1].second, "@@@LINK=@@@LINK_target\r\n", "v2 html: link record");
|
||||
check_eq<std::string>(records[2].first, "dup", "v2 html: duplicate key 1");
|
||||
check_eq<std::string>(records[3].first, "dup", "v2 html: duplicate key 2");
|
||||
check_eq<std::string>(records[3].second, "<p class=\"a\">second dup</p>", "v2 html: duplicate record kept apart");
|
||||
check_eq<std::string>(records[6].second,
|
||||
"<table><tr><td><ruby>漢<rt>かん</rt></ruby></td></tr></table><img src=\"img/pic.png\">",
|
||||
"v2 html: LZO record");
|
||||
check_eq<std::string>(records[7].first, "食べる", "v2 html: unicode key");
|
||||
check_eq<std::string>(records[7].second, "`1`to eat`2` (ichidan)", "v2 html: backtick styles untouched");
|
||||
|
||||
// Reading one key block at a time gives the same keys as read_all_keys.
|
||||
std::vector<mdict::KeyEntry> block1;
|
||||
reader.read_key_block(1, block1);
|
||||
check_eq<size_t>(block1.size(), 3, "v2 html: block 1 has 3 keys");
|
||||
if (block1.size() == 3) {
|
||||
check_eq<std::string>(block1[0].key, "dup", "v2 html: block 1 starts at second dup");
|
||||
}
|
||||
}
|
||||
|
||||
void test_v2_utf16_encrypted() {
|
||||
mdict::Reader reader;
|
||||
reader.open(fixtures / "v2_utf16_encrypted2.mdx");
|
||||
check(reader.header().encoding == mdict::Encoding::Utf16le, "utf16: encoding");
|
||||
check_eq<int>(reader.header().encrypted, 2, "utf16: encrypted flag");
|
||||
check_eq<std::string>(reader.key_blocks()[1].first_key, "日本語", "utf16: index key decoded");
|
||||
expect_text_fixture(reader, "utf16 encrypted");
|
||||
}
|
||||
|
||||
void test_v1() {
|
||||
mdict::Reader reader;
|
||||
reader.open(fixtures / "v1_utf8_stored.mdx");
|
||||
check_eq<std::string>(reader.header().engine_version, "1.2", "v1: engine version");
|
||||
check_eq<std::string>(reader.header().title, "V1 Fixture", "v1: title");
|
||||
expect_text_fixture(reader, "v1");
|
||||
}
|
||||
|
||||
void test_mdd() {
|
||||
mdict::Reader reader;
|
||||
reader.open(fixtures / "v2_utf8_lzo_html.mdd");
|
||||
check(reader.header().kind == mdict::Kind::Mdd, "mdd: kind");
|
||||
check(reader.header().encoding == mdict::Encoding::Utf16le, "mdd: keys are UTF-16");
|
||||
const auto records = all_records(reader, false);
|
||||
check_eq<size_t>(records.size(), 5, "mdd: record count");
|
||||
if (records.size() != 4) {
|
||||
return;
|
||||
}
|
||||
check_eq<std::string>(records[0].first, "\\a.spx", "mdd: key 0");
|
||||
check_eq<std::string>(records[0].second, "not really speex", "mdd: raw record");
|
||||
check_eq<std::string>(records[1].first, "\\img\\pic.png", "mdd: key 1");
|
||||
check_eq<size_t>(records[1].second.size(), 69, "mdd: png size");
|
||||
check(records[1].second.starts_with("\x89PNG"), "mdd: png bytes");
|
||||
check_eq<std::string>(records[3].first, "\\..\\evil.png", "mdd: traversal key is delivered verbatim");
|
||||
}
|
||||
|
||||
void expect_open_fails(const char* file, const std::string& needle) {
|
||||
mdict::Reader reader;
|
||||
try {
|
||||
reader.open(fixtures / file);
|
||||
// Opening only reads the indexes; a bad record block shows up when read.
|
||||
for (size_t i = 0; i < reader.record_blocks().size(); ++i) {
|
||||
reader.read_record_block(i);
|
||||
}
|
||||
check(false, std::string(file) + ": expected an error containing \"" + needle + "\"");
|
||||
} catch (const mdict::Error& e) {
|
||||
const std::string message = e.what();
|
||||
check(message.find(needle) != std::string::npos,
|
||||
std::string(file) + ": error \"" + message + "\" does not mention \"" + needle + "\"");
|
||||
}
|
||||
}
|
||||
|
||||
void test_malformed() {
|
||||
expect_open_fails("bad_truncated.mdx", "truncated file");
|
||||
expect_open_fails("bad_adler.mdx", "Adler-32");
|
||||
expect_open_fails("bad_huge_block.mdx", "impossible size");
|
||||
expect_open_fails("bad_encrypted1.mdx", "registration-protected");
|
||||
expect_open_fails("bad_gbk.mdx", "unsupported MDX encoding: GBK");
|
||||
expect_open_fails("bad_v3.mdx", "unsupported MDX engine version 3.0");
|
||||
expect_open_fails("does_not_exist.mdx", "could not open");
|
||||
}
|
||||
|
||||
void test_sniff() {
|
||||
auto head = [](const std::filesystem::path& path) {
|
||||
std::ifstream in(path, std::ios::binary);
|
||||
std::vector<uint8_t> bytes(64);
|
||||
in.read(reinterpret_cast<char*>(bytes.data()), static_cast<std::streamsize>(bytes.size()));
|
||||
bytes.resize(static_cast<size_t>(in.gcount()));
|
||||
return bytes;
|
||||
};
|
||||
auto mdx = head(fixtures / "v2_utf8_zlib_text.mdx");
|
||||
check(mdict::looks_like_mdict(mdx.data(), mdx.size()), "sniff: mdx");
|
||||
auto mdd = head(fixtures / "v2_utf8_lzo_html.mdd");
|
||||
check(mdict::looks_like_mdict(mdd.data(), mdd.size()), "sniff: mdd");
|
||||
auto zip = head(fixtures / ".." / "yomitan" / "small_dict.zip");
|
||||
check(!mdict::looks_like_mdict(zip.data(), zip.size()), "sniff: zip is not mdict");
|
||||
const uint8_t junk[] = {0, 0, 0, 0};
|
||||
check(!mdict::looks_like_mdict(junk, sizeof junk), "sniff: zeros");
|
||||
}
|
||||
|
||||
void test_utf16() {
|
||||
const uint8_t text[] = {0x3d, 0xd8, 0x00, 0xde, 0x41, 0x00, 0x00, 0xdc}; // U+1F600, 'A', lone low surrogate
|
||||
check_eq<std::string>(mdict::utf16le_to_utf8(text, sizeof text), "\xf0\x9f\x98\x80" "A" "\xef\xbf\xbd",
|
||||
"utf16: surrogate pair, ascii, lone surrogate");
|
||||
}
|
||||
|
||||
// Corrupt the fixtures at random positions; every outcome must be either a
|
||||
// clean read or an mdict::Error, never a crash or another exception type.
|
||||
void test_mutations() {
|
||||
std::mt19937 rng(20260921);
|
||||
const std::filesystem::path scratch =
|
||||
std::filesystem::temp_directory_path() / ("hoshidicts-mdict-mutation-" + std::to_string(rng()));
|
||||
std::filesystem::create_directories(scratch);
|
||||
const char* sources[] = {"v2_utf8_zlib_text.mdx", "v2_utf8_lzo_html.mdx", "v2_utf16_encrypted2.mdx",
|
||||
"v1_utf8_stored.mdx", "v2_utf8_lzo_html.mdd"};
|
||||
size_t clean = 0;
|
||||
size_t rejected = 0;
|
||||
for (const char* source : sources) {
|
||||
std::ifstream in(fixtures / source, std::ios::binary);
|
||||
std::vector<char> original((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
|
||||
for (int round = 0; round < 400; ++round) {
|
||||
std::vector<char> mutated = original;
|
||||
const int flips = 1 + static_cast<int>(rng() % 4);
|
||||
for (int f = 0; f < flips; ++f) {
|
||||
mutated[rng() % mutated.size()] = static_cast<char>(rng());
|
||||
}
|
||||
if (round % 50 == 49) {
|
||||
mutated.resize(rng() % mutated.size());
|
||||
}
|
||||
const auto path = scratch / "mutant.mdx";
|
||||
{
|
||||
std::ofstream out(path, std::ios::binary | std::ios::trunc);
|
||||
out.write(mutated.data(), static_cast<std::streamsize>(mutated.size()));
|
||||
}
|
||||
try {
|
||||
mdict::Reader reader;
|
||||
reader.open(path);
|
||||
all_records(reader, reader.header().kind == mdict::Kind::Mdx);
|
||||
clean++;
|
||||
} catch (const mdict::Error&) {
|
||||
rejected++;
|
||||
} catch (const std::exception& e) {
|
||||
check(false, std::string("mutation of ") + source + " escaped as " + e.what());
|
||||
}
|
||||
}
|
||||
}
|
||||
std::filesystem::remove_all(scratch);
|
||||
std::printf("mutations: %zu read cleanly, %zu rejected\n", clean, rejected);
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc < 2) {
|
||||
std::printf("usage: %s <fixture dir>\n", argv[0]);
|
||||
return 2;
|
||||
}
|
||||
fixtures = argv[1];
|
||||
test_ripemd128();
|
||||
test_utf16();
|
||||
test_v2_text();
|
||||
test_v2_lzo_html();
|
||||
test_v2_utf16_encrypted();
|
||||
test_v1();
|
||||
test_mdd();
|
||||
test_malformed();
|
||||
test_sniff();
|
||||
test_mutations();
|
||||
if (failures == 0) {
|
||||
std::printf("ok\n");
|
||||
}
|
||||
return failures == 0 ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
// End-to-end MDX import: dictionary_importer::import on the fixtures in
|
||||
// tests/fixtures/mdict, then DictionaryQuery lookups against the result.
|
||||
//
|
||||
// mdict_test <tests/fixtures/mdict>
|
||||
#include <cstdio>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iterator>
|
||||
#include <map>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "hoshidicts/importer.hpp"
|
||||
#include "hoshidicts/query.hpp"
|
||||
|
||||
namespace {
|
||||
int failures = 0;
|
||||
std::filesystem::path fixtures;
|
||||
|
||||
void check(bool ok, const std::string& what) {
|
||||
if (!ok) {
|
||||
std::printf("FAIL %s\n", what.c_str());
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
|
||||
void check_contains(const std::string& haystack, const std::string& needle, const std::string& what) {
|
||||
if (haystack.find(needle) == std::string::npos) {
|
||||
std::printf("FAIL %s\n \"%s\" not in: %s\n", what.c_str(), needle.c_str(), haystack.c_str());
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
|
||||
std::filesystem::path fresh_dir(const char* tag) {
|
||||
std::random_device rd;
|
||||
const auto dir = std::filesystem::temp_directory_path() / ("hoshidicts-mdict-" + std::string(tag) + "-" +
|
||||
std::to_string(rd()));
|
||||
std::filesystem::create_directories(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
// Copies the fixture (and its sibling MDD when present) into `dir` so MDD
|
||||
// discovery runs on real neighbours and imports it there.
|
||||
ImportResult import_fixture(const std::filesystem::path& dir, const char* name, bool low_ram) {
|
||||
std::filesystem::copy_file(fixtures / name, dir / name, std::filesystem::copy_options::overwrite_existing);
|
||||
std::filesystem::path mdd = fixtures / name;
|
||||
mdd.replace_extension(".mdd");
|
||||
if (std::filesystem::exists(mdd)) {
|
||||
std::filesystem::copy_file(mdd, dir / mdd.filename(), std::filesystem::copy_options::overwrite_existing);
|
||||
}
|
||||
return dictionary_importer::import((dir / name).string(), dir.string(), low_ram);
|
||||
}
|
||||
|
||||
std::string glossary_of(const std::vector<TermResult>& results, size_t term = 0, size_t glossary = 0) {
|
||||
if (results.size() <= term || results[term].glossaries.size() <= glossary) {
|
||||
return {};
|
||||
}
|
||||
return results[term].glossaries[glossary].glossary;
|
||||
}
|
||||
|
||||
std::string read_file(const std::filesystem::path& path) {
|
||||
std::ifstream in(path, std::ios::binary);
|
||||
return std::string(std::istreambuf_iterator<char>(in), {});
|
||||
}
|
||||
|
||||
void test_html_import() {
|
||||
const auto dir = fresh_dir("html");
|
||||
const ImportResult result = import_fixture(dir, "v2_utf8_lzo_html.mdx", false);
|
||||
check(result.success, "html: import succeeded: " + result.error);
|
||||
if (!result.success) {
|
||||
return;
|
||||
}
|
||||
check(result.title == "HTML Fixture", "html: title from header, got " + result.title);
|
||||
// 7 non-redirect entries + 1 resolved alias; the alias to a missing target is dropped.
|
||||
check(result.summary.counts.terms.total == 8, "html: 8 term rows, got " +
|
||||
std::to_string(result.summary.counts.terms.total));
|
||||
check(result.summary.sequenced, "html: sequenced");
|
||||
check(result.summary.revision == "mdx import", "html: revision");
|
||||
check(result.summary.description == "HTML fixture with links, duplicates and a stylesheet", "html: description");
|
||||
// style.css, utf16.css, a.spx (sound://), img/pic.png; ../evil.png is rejected.
|
||||
check(result.summary.counts.media.total == 4,
|
||||
"html: 4 media files, got " + std::to_string(result.summary.counts.media.total));
|
||||
|
||||
const std::string dict = (dir / result.title).string();
|
||||
DictionaryQuery query;
|
||||
check(query.add_term_dict(dict), "html: dictionary loads");
|
||||
|
||||
check_contains(glossary_of(query.query("entry")), R"("href":"?query=alias")", "html: entry:// link");
|
||||
check_contains(glossary_of(query.query("entry")), R"({"tag":"a","href":"#","content":["snd"]})",
|
||||
"html: sound:// is # with audio off");
|
||||
check_contains(glossary_of(query.query("alias")), "target of a link", "html: alias resolves to its target");
|
||||
check(query.query("missing-alias").empty(), "html: alias to a missing target is dropped");
|
||||
check(query.query("@@@LINK_target").size() == 1, "html: target itself is still a headword");
|
||||
|
||||
const auto dup = query.query("dup");
|
||||
check(dup.size() == 1 && dup[0].glossaries.size() == 2, "html: duplicate headwords keep both entries");
|
||||
check_contains(glossary_of(dup, 0, 0), "first dup", "html: first duplicate");
|
||||
check_contains(glossary_of(dup, 0, 1), "second dup", "html: second duplicate");
|
||||
|
||||
check_contains(glossary_of(query.query("食べる")),
|
||||
R"({"tag":"span","style":{"fontWeight":"bold"},"content":["to eat"]})",
|
||||
"html: StyleSheet backticks expanded and converted");
|
||||
check_contains(glossary_of(query.query("ruby")), R"({"tag":"img","path":"mdict-media/img/pic.png"})",
|
||||
"html: image path under mdict-media/");
|
||||
check_contains(glossary_of(query.query("見出し")), R"("style":{"color":"red","fontSize":"12px"})",
|
||||
"html: inline style on a Unicode headword");
|
||||
|
||||
const auto styles = query.get_styles();
|
||||
check(styles.size() == 1, "html: one stylesheet");
|
||||
if (styles.size() == 1) {
|
||||
check_contains(styles[0].styles, "/* Source: style.css */\n.mdx-red { color: red; }", "html: MDD css");
|
||||
check_contains(styles[0].styles, "/* Source: utf16.css */\n.u16::before { content: \"\xe2\x86\x92\"; }",
|
||||
"html: BOM-less UTF-16 css decoded");
|
||||
check_contains(styles[0].styles, "/* Source: entry/inline/1.css */\n.inline-x { color: blue; }",
|
||||
"html: inline <style> collected");
|
||||
}
|
||||
|
||||
const auto png = query.get_media_file(result.title, "mdict-media/img/pic.png");
|
||||
check(png.size() == 69 && png.size() > 4 && png[1] == 'P' && png[2] == 'N' && png[3] == 'G',
|
||||
"html: PNG from the MDD, got " + std::to_string(png.size()) + " bytes");
|
||||
check(!query.get_media_file(result.title, "mdict-media/a.spx").empty(), "html: sound asset extracted");
|
||||
check(query.get_media_file(result.title, "mdict-media/evil.png").empty(), "html: traversal key not imported");
|
||||
check(query.get_media_file(result.title, "mdict-media/../evil.png").empty(), "html: traversal path not imported");
|
||||
|
||||
std::filesystem::remove_all(dir);
|
||||
}
|
||||
|
||||
void expect_text_dictionary(const char* name) {
|
||||
const auto dir = fresh_dir("text");
|
||||
const ImportResult result = import_fixture(dir, name, false);
|
||||
check(result.success, std::string(name) + ": import succeeded: " + result.error);
|
||||
if (!result.success) {
|
||||
return;
|
||||
}
|
||||
check(result.summary.counts.terms.total == 4, std::string(name) + ": 4 terms");
|
||||
DictionaryQuery query;
|
||||
query.add_term_dict((dir / result.title).string());
|
||||
check(glossary_of(query.query("alpha")) == R"(["first definition"])",
|
||||
std::string(name) + ": Format=Text glossary is a plain string, got " + glossary_of(query.query("alpha")));
|
||||
check(glossary_of(query.query("日本語")) == R"(["Japanese text \"quoted\""])",
|
||||
std::string(name) + ": Unicode key and escaped text");
|
||||
check(glossary_of(query.query("beta")) == R"(["second\ndefinition with newline"])",
|
||||
std::string(name) + ": newline kept");
|
||||
std::filesystem::remove_all(dir);
|
||||
}
|
||||
|
||||
void expect_import_fails(const char* name, const std::string& needle) {
|
||||
const auto dir = fresh_dir("bad");
|
||||
const ImportResult result = import_fixture(dir, name, false);
|
||||
check(!result.success, std::string(name) + ": import must fail");
|
||||
check(result.error.find(needle) != std::string::npos,
|
||||
std::string(name) + ": error \"" + result.error + "\" does not mention \"" + needle + "\"");
|
||||
size_t leftovers = 0;
|
||||
for (const auto& entry : std::filesystem::directory_iterator(dir)) {
|
||||
if (entry.is_directory()) {
|
||||
leftovers++;
|
||||
}
|
||||
}
|
||||
check(leftovers == 0, std::string(name) + ": no output directory left behind");
|
||||
std::filesystem::remove_all(dir);
|
||||
}
|
||||
|
||||
void test_malformed() {
|
||||
expect_import_fails("bad_truncated.mdx", "truncated file");
|
||||
expect_import_fails("bad_adler.mdx", "Adler-32");
|
||||
expect_import_fails("bad_huge_block.mdx", "impossible size");
|
||||
expect_import_fails("bad_encrypted1.mdx", "registration-protected");
|
||||
expect_import_fails("bad_gbk.mdx", "unsupported MDX encoding: GBK");
|
||||
expect_import_fails("bad_v3.mdx", "unsupported MDX engine version 3.0");
|
||||
expect_import_fails("v2_utf8_lzo_html.mdd", "MDD resource file");
|
||||
}
|
||||
|
||||
// Sibling MDDs: Dict.MDD is found for Dict.mdx regardless of case, and a
|
||||
// numbered Dict.1.mdd is found when there is no plain Dict.mdd.
|
||||
void test_mdd_discovery() {
|
||||
for (const char* mdd_name : {"Sib.MDD", "Sib.1.mdd"}) {
|
||||
const auto dir = fresh_dir("sib");
|
||||
std::filesystem::copy_file(fixtures / "v2_utf8_lzo_html.mdx", dir / "Sib.mdx");
|
||||
std::filesystem::copy_file(fixtures / "v2_utf8_lzo_html.mdd", dir / mdd_name);
|
||||
const ImportResult result = dictionary_importer::import((dir / "Sib.mdx").string(), dir.string(), false);
|
||||
check(result.success, std::string(mdd_name) + ": import succeeded: " + result.error);
|
||||
if (result.success) {
|
||||
DictionaryQuery query;
|
||||
query.add_term_dict((dir / result.title).string());
|
||||
check(query.get_media_file(result.title, "mdict-media/img/pic.png").size() == 69,
|
||||
std::string(mdd_name) + ": MDD discovered");
|
||||
}
|
||||
std::filesystem::remove_all(dir);
|
||||
}
|
||||
// No MDD at all is fine: the glossaries still import, just without media.
|
||||
const auto dir = fresh_dir("nomdd");
|
||||
std::filesystem::copy_file(fixtures / "v2_utf8_lzo_html.mdx", dir / "Alone.mdx");
|
||||
const ImportResult result = dictionary_importer::import((dir / "Alone.mdx").string(), dir.string(), false);
|
||||
check(result.success && result.summary.counts.media.total == 0, "no mdd: import succeeds without media");
|
||||
std::filesystem::remove_all(dir);
|
||||
}
|
||||
|
||||
// low_ram must not change a single output byte.
|
||||
void test_low_ram_identical() {
|
||||
const auto normal = fresh_dir("normal");
|
||||
const auto low = fresh_dir("lowram");
|
||||
const ImportResult a = import_fixture(normal, "v2_utf8_lzo_html.mdx", false);
|
||||
const ImportResult b = import_fixture(low, "v2_utf8_lzo_html.mdx", true);
|
||||
check(a.success && b.success, "low_ram: both imports succeed");
|
||||
if (a.success && b.success) {
|
||||
for (const auto& entry : std::filesystem::directory_iterator(normal / a.title)) {
|
||||
const std::string name = entry.path().filename().string();
|
||||
if (name == "index.json") {
|
||||
continue; // importDate
|
||||
}
|
||||
check(read_file(entry.path()) == read_file(low / b.title / name), "low_ram: " + name + " identical");
|
||||
}
|
||||
}
|
||||
std::filesystem::remove_all(normal);
|
||||
std::filesystem::remove_all(low);
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc < 2) {
|
||||
std::printf("usage: %s <fixture dir>\n", argv[0]);
|
||||
return 2;
|
||||
}
|
||||
fixtures = argv[1];
|
||||
test_html_import();
|
||||
expect_text_dictionary("v2_utf8_zlib_text.mdx");
|
||||
expect_text_dictionary("v2_utf16_encrypted2.mdx");
|
||||
expect_text_dictionary("v1_utf8_stored.mdx");
|
||||
test_malformed();
|
||||
test_mdd_discovery();
|
||||
test_low_ram_identical();
|
||||
if (failures == 0) {
|
||||
std::printf("ok\n");
|
||||
}
|
||||
return failures == 0 ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
// Term scores are JSON numbers in the Yomitan schema. Import a tiny dictionary
|
||||
// whose scores are fractional, negative, signed zero, spelt with exponents, and
|
||||
// beyond int32 in both directions, then look every term up and require the
|
||||
// exact double back. Also requires that two terms whose scores differ only in
|
||||
// the fraction sort in score order, which is what int32 storage broke.
|
||||
#include "hoshidicts/deinflector.hpp"
|
||||
#include "hoshidicts/importer.hpp"
|
||||
#include "hoshidicts/lookup.hpp"
|
||||
#include "hoshidicts/query.hpp"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
uint32_t crc32(const std::string& data) {
|
||||
uint32_t crc = 0xFFFFFFFFu;
|
||||
for (unsigned char c : data) {
|
||||
crc ^= c;
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
crc = (crc >> 1) ^ (0xEDB88320u & (0u - (crc & 1u)));
|
||||
}
|
||||
}
|
||||
return ~crc;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void put(std::string& out, T value) {
|
||||
char buf[sizeof(T)];
|
||||
std::memcpy(buf, &value, sizeof(T));
|
||||
out.append(buf, sizeof(T));
|
||||
}
|
||||
|
||||
// A stored (method 0) ZIP: local headers, then the central directory, then EOCD.
|
||||
std::string build_zip(const std::vector<std::pair<std::string, std::string>>& files) {
|
||||
std::string out;
|
||||
std::string central;
|
||||
for (const auto& [name, data] : files) {
|
||||
const uint32_t offset = static_cast<uint32_t>(out.size());
|
||||
const uint32_t crc = crc32(data);
|
||||
const uint32_t size = static_cast<uint32_t>(data.size());
|
||||
put<uint32_t>(out, 0x04034b50);
|
||||
put<uint16_t>(out, 20);
|
||||
put<uint16_t>(out, 0);
|
||||
put<uint16_t>(out, 0); // stored
|
||||
put<uint16_t>(out, 0);
|
||||
put<uint16_t>(out, 0);
|
||||
put<uint32_t>(out, crc);
|
||||
put<uint32_t>(out, size);
|
||||
put<uint32_t>(out, size);
|
||||
put<uint16_t>(out, static_cast<uint16_t>(name.size()));
|
||||
put<uint16_t>(out, 0);
|
||||
out += name;
|
||||
out += data;
|
||||
|
||||
put<uint32_t>(central, 0x02014b50);
|
||||
put<uint16_t>(central, 20);
|
||||
put<uint16_t>(central, 20);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint32_t>(central, crc);
|
||||
put<uint32_t>(central, size);
|
||||
put<uint32_t>(central, size);
|
||||
put<uint16_t>(central, static_cast<uint16_t>(name.size()));
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint16_t>(central, 0);
|
||||
put<uint32_t>(central, 0);
|
||||
put<uint32_t>(central, offset);
|
||||
central += name;
|
||||
}
|
||||
const uint32_t central_offset = static_cast<uint32_t>(out.size());
|
||||
out += central;
|
||||
put<uint32_t>(out, 0x06054b50);
|
||||
put<uint16_t>(out, 0);
|
||||
put<uint16_t>(out, 0);
|
||||
put<uint16_t>(out, static_cast<uint16_t>(files.size()));
|
||||
put<uint16_t>(out, static_cast<uint16_t>(files.size()));
|
||||
put<uint32_t>(out, static_cast<uint32_t>(central.size()));
|
||||
put<uint32_t>(out, central_offset);
|
||||
put<uint16_t>(out, 0);
|
||||
return out;
|
||||
}
|
||||
|
||||
struct Case {
|
||||
const char* expression;
|
||||
const char* json_score; // as it appears in the bank
|
||||
double expected;
|
||||
};
|
||||
|
||||
// Each expression is unique so a lookup returns exactly one candidate.
|
||||
const Case kCases[] = {
|
||||
{"甲", "1.5", 1.5},
|
||||
{"乙", "-2.25", -2.25},
|
||||
{"丙", "0.1", 0.1},
|
||||
{"丁", "1099511627776.5", 1099511627776.5},
|
||||
{"戊", "-0", -0.0},
|
||||
{"己", "2147483648", 2147483648.0},
|
||||
{"庚", "-2147483649", -2147483649.0},
|
||||
{"辛", "1e0", 1.0},
|
||||
{"壬", "1.25e2", 125.0},
|
||||
{"癸", "5e-1", 0.5},
|
||||
{"子", "9007199254740993", 9007199254740992.0}, // beyond 2^53: rounds like JS Number
|
||||
};
|
||||
|
||||
int failures = 0;
|
||||
|
||||
void check(bool ok, const std::string& what) {
|
||||
if (!ok) {
|
||||
++failures;
|
||||
std::fprintf(stderr, "FAIL: %s\n", what.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
const std::filesystem::path root =
|
||||
std::filesystem::temp_directory_path() /
|
||||
("hoshidicts-score-test-" + std::to_string(std::random_device{}()));
|
||||
std::filesystem::remove_all(root);
|
||||
std::filesystem::create_directories(root);
|
||||
|
||||
std::string bank = "[";
|
||||
for (size_t i = 0; i < std::size(kCases); ++i) {
|
||||
if (i) bank += ",";
|
||||
bank += "[\"";
|
||||
bank += kCases[i].expression;
|
||||
bank += "\",\"よみ\",\"\",\"\",";
|
||||
bank += kCases[i].json_score;
|
||||
bank += ",[\"gloss\"],0,\"\"]";
|
||||
}
|
||||
// Two same-reading terms whose scores differ only by a fraction: 2.75 must
|
||||
// rank above 2.25 once fractions survive; both truncated to 2 before.
|
||||
bank += ",[\"高\",\"おなじ\",\"\",\"\",2.25,[\"low\"],0,\"\"]";
|
||||
bank += ",[\"高\",\"おなじ\",\"\",\"\",2.75,[\"high\"],0,\"\"]";
|
||||
bank += "]";
|
||||
|
||||
const std::string index = R"({"title":"score-test","format":3,"revision":"1"})";
|
||||
const std::string zip = build_zip({{"index.json", index}, {"term_bank_1.json", bank}});
|
||||
const std::filesystem::path zip_path = root / "score-test.zip";
|
||||
{
|
||||
std::ofstream f(zip_path, std::ios::binary);
|
||||
f.write(zip.data(), static_cast<std::streamsize>(zip.size()));
|
||||
}
|
||||
|
||||
const std::filesystem::path out_dir = root / "out";
|
||||
std::filesystem::create_directories(out_dir);
|
||||
const auto result = dictionary_importer::import(zip_path.string(), out_dir.string());
|
||||
check(result.success, "import succeeded: " + result.error);
|
||||
if (!result.success) {
|
||||
return 1;
|
||||
}
|
||||
const std::filesystem::path dict_dir = out_dir / result.summary.title;
|
||||
check(std::filesystem::is_regular_file(dict_dir / ".hoshidicts_5") ||
|
||||
std::filesystem::is_regular_file(dict_dir / ".hoshidicts_6"),
|
||||
"importer wrote a double-score marker");
|
||||
|
||||
DictionaryQuery query;
|
||||
check(query.add_term_dict(dict_dir.string()), "add_dict");
|
||||
|
||||
for (const auto& c : kCases) {
|
||||
const auto terms = query.query(c.expression);
|
||||
check(terms.size() == 1, std::string("one result for ") + c.expression);
|
||||
if (terms.empty()) continue;
|
||||
const double got = terms[0].score;
|
||||
const bool same = got == c.expected && std::signbit(got) == std::signbit(c.expected);
|
||||
check(same, std::string("score for ") + c.expression + " json=" + c.json_score + " expected=" +
|
||||
std::to_string(c.expected) + " got=" + std::to_string(got));
|
||||
}
|
||||
|
||||
// The merged term keeps the maximum score of its glossaries, which is now
|
||||
// the fractional 2.75 rather than 2.
|
||||
{
|
||||
const auto terms = query.query("高");
|
||||
check(terms.size() == 1, "one merged result for 高");
|
||||
if (!terms.empty()) {
|
||||
check(terms[0].score == 2.75, "merged score is max(2.25, 2.75)=2.75, got " + std::to_string(terms[0].score));
|
||||
check(terms[0].glossaries.size() == 2, "merged term keeps both glossaries");
|
||||
}
|
||||
}
|
||||
|
||||
// The lookup path (deinflection + ranking) surfaces the same double.
|
||||
{
|
||||
Deinflector deinflector;
|
||||
Lookup lookup(query, deinflector);
|
||||
const auto results = lookup.lookup("丁");
|
||||
check(!results.empty(), "lookup finds 丁");
|
||||
if (!results.empty()) {
|
||||
check(results[0].term.score == 1099511627776.5,
|
||||
"lookup score for 丁 got " + std::to_string(results[0].term.score));
|
||||
}
|
||||
}
|
||||
|
||||
std::filesystem::remove_all(root);
|
||||
if (failures) {
|
||||
std::fprintf(stderr, "%d failure(s)\n", failures);
|
||||
return 1;
|
||||
}
|
||||
std::puts("score round-trip: ok");
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user