mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-09-22 17:16:19 -07:00
feat(dictionary): add Hachidori backend support
- Add backend selection, setup gating, Anki integration, and external host support - Add launcher flags, documentation, packaging, and focused tests - Open on-demand overlay modals on the first attempt
This commit is contained in:
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
+338
@@ -0,0 +1,338 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Writes the MDX/MDD fixtures used by mdict_reader_test and mdict_test.
|
||||
|
||||
A small MDict writer (format per
|
||||
https://github.com/zhansliu/writemdict/blob/master/fileformat.md) rather than
|
||||
a dependency on writemdict, which is not on PyPI. Supports engine versions
|
||||
1.2 and 2.0, UTF-8 and UTF-16 text, compression 0 (stored), 1 (LZO1X, written
|
||||
as a single literal run, which is a valid stream) and 2 (zlib), and the
|
||||
Encrypted=2 key-index cipher. Every fixture is a few KB and deterministic.
|
||||
|
||||
python3 tests/fixtures/mdict/gen_fixtures.py
|
||||
"""
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- RIPEMD-128
|
||||
def _rol(x, n):
|
||||
return ((x << n) | (x >> (32 - n))) & 0xFFFFFFFF
|
||||
|
||||
|
||||
_RL = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
7, 4, 13, 1, 10, 6, 15, 3, 12, 0, 9, 5, 2, 14, 11, 8,
|
||||
3, 10, 14, 4, 9, 15, 8, 1, 2, 7, 0, 6, 13, 11, 5, 12,
|
||||
1, 9, 11, 10, 0, 8, 12, 4, 13, 3, 7, 15, 14, 5, 6, 2]
|
||||
_RR = [5, 14, 7, 0, 9, 2, 11, 4, 13, 6, 15, 8, 1, 10, 3, 12,
|
||||
6, 11, 3, 7, 0, 13, 5, 10, 14, 15, 8, 12, 4, 9, 1, 2,
|
||||
15, 5, 1, 3, 7, 14, 6, 9, 11, 8, 12, 2, 10, 0, 4, 13,
|
||||
8, 6, 4, 1, 3, 11, 15, 0, 5, 12, 2, 13, 9, 7, 10, 14]
|
||||
_SL = [11, 14, 15, 12, 5, 8, 7, 9, 11, 13, 14, 15, 6, 7, 9, 8,
|
||||
7, 6, 8, 13, 11, 9, 7, 15, 7, 12, 15, 9, 11, 7, 13, 12,
|
||||
11, 13, 6, 7, 14, 9, 13, 15, 14, 8, 13, 6, 5, 12, 7, 5,
|
||||
11, 12, 14, 15, 14, 15, 9, 8, 9, 14, 5, 6, 8, 6, 5, 12]
|
||||
_SR = [8, 9, 9, 11, 13, 15, 15, 5, 7, 7, 8, 11, 14, 14, 12, 6,
|
||||
9, 13, 15, 7, 12, 8, 9, 11, 7, 7, 12, 7, 6, 15, 13, 11,
|
||||
9, 7, 15, 11, 8, 6, 6, 14, 12, 13, 5, 14, 13, 13, 7, 5,
|
||||
15, 5, 8, 11, 14, 14, 6, 14, 6, 9, 12, 9, 12, 5, 15, 8]
|
||||
_KL = [0x00000000, 0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC]
|
||||
_KR = [0x50A28BE6, 0x5C4DD124, 0x6D703EF3, 0x00000000]
|
||||
|
||||
|
||||
def _f(r, x, y, z):
|
||||
if r == 0:
|
||||
return x ^ y ^ z
|
||||
if r == 1:
|
||||
return (x & y) | (~x & z)
|
||||
if r == 2:
|
||||
return (x | ~y) ^ z
|
||||
return (x & z) | (y & ~z)
|
||||
|
||||
|
||||
def ripemd128(data):
|
||||
h = [0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476]
|
||||
msg = bytearray(data) + b"\x80"
|
||||
while len(msg) % 64 != 56:
|
||||
msg += b"\x00"
|
||||
msg += struct.pack("<Q", len(data) * 8)
|
||||
for off in range(0, len(msg), 64):
|
||||
x = list(struct.unpack("<16I", msg[off:off + 64]))
|
||||
al, bl, cl, dl = h
|
||||
ar, br, cr, dr = h
|
||||
for j in range(64):
|
||||
r = j // 16
|
||||
t = _rol((al + _f(r, bl, cl, dl) + x[_RL[j]] + _KL[r]) & 0xFFFFFFFF, _SL[j])
|
||||
al, dl, cl, bl = dl, cl, bl, t
|
||||
t = _rol((ar + _f(3 - r, br, cr, dr) + x[_RR[j]] + _KR[r]) & 0xFFFFFFFF, _SR[j])
|
||||
ar, dr, cr, br = dr, cr, br, t
|
||||
t = (h[1] + cl + dr) & 0xFFFFFFFF
|
||||
h[1] = (h[2] + dl + ar) & 0xFFFFFFFF
|
||||
h[2] = (h[3] + al + br) & 0xFFFFFFFF
|
||||
h[3] = (h[0] + bl + cr) & 0xFFFFFFFF
|
||||
h[0] = t
|
||||
return struct.pack("<4I", *h)
|
||||
|
||||
|
||||
assert ripemd128(b"abc").hex() == "c14a12199c66e4ba84636b0f69144c77"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ helpers
|
||||
def lzo_literal_stream(data):
|
||||
"""A valid LZO1X stream that stores `data` as one literal run.
|
||||
|
||||
First-byte shortcut: 18..255 copies (byte - 17) literals. Longer runs use
|
||||
the regular long-literal instruction 0x00 with zero-byte length extension.
|
||||
0x11 0x00 0x00 is the end-of-stream marker (M4 with distance 16384).
|
||||
"""
|
||||
n = len(data)
|
||||
end = b"\x11\x00\x00"
|
||||
if n == 0:
|
||||
return end
|
||||
if n <= 238:
|
||||
return bytes([17 + n]) + data + end
|
||||
rest = n - 18
|
||||
zeros = (rest - 1) // 255
|
||||
last = rest - 255 * zeros
|
||||
return b"\x00" + b"\x00" * zeros + bytes([last]) + data + end
|
||||
|
||||
|
||||
def frame(payload, compression):
|
||||
"""MDict block framing: LE u32 compression, BE adler32 of payload, packed payload."""
|
||||
if compression == 0:
|
||||
packed = payload
|
||||
elif compression == 1:
|
||||
packed = lzo_literal_stream(payload)
|
||||
elif compression == 2:
|
||||
packed = zlib.compress(payload, 9)
|
||||
else:
|
||||
raise ValueError(compression)
|
||||
return struct.pack("<I", compression) + struct.pack(">I", zlib.adler32(payload) & 0xFFFFFFFF) + packed
|
||||
|
||||
|
||||
def encrypt_key_index(framed):
|
||||
key = ripemd128(framed[4:8] + struct.pack("<L", 0x3695))
|
||||
body = bytearray(framed[8:])
|
||||
previous = 0x36
|
||||
# Inverse of readmdict's _fast_decrypt: plaintext byte p -> stored byte b with
|
||||
# decrypt(b) = swap(b) ^ prev ^ i ^ key[i], prev = previous stored byte.
|
||||
out = bytearray()
|
||||
for i, p in enumerate(body):
|
||||
t = p ^ previous ^ (i & 0xFF) ^ key[i % 16]
|
||||
b = ((t >> 4) | (t << 4)) & 0xFF
|
||||
out.append(b)
|
||||
previous = b
|
||||
return framed[:8] + bytes(out)
|
||||
|
||||
|
||||
def xml_escape(s):
|
||||
return (s.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- writer
|
||||
def write_mdict(path, entries, *, version="2.0", encoding="UTF-8", compression=2, encrypted=0,
|
||||
kind="mdx", fmt="Html", title="Fixture", description="", stylesheet="",
|
||||
block_entries=3, record_block_bytes=200, extra_attrs=None, corrupt=None):
|
||||
"""entries: list of (key, value) with value str for mdx, bytes for mdd.
|
||||
|
||||
corrupt: None or one of 'truncate', 'record_adler', 'huge_block' to produce a
|
||||
malformed file for the negative tests.
|
||||
"""
|
||||
v2 = float(version) >= 2.0
|
||||
num = ">Q" if v2 else ">I"
|
||||
width = 8 if v2 else 4
|
||||
text_enc = "utf-16-le" if (kind == "mdd" or encoding.upper().startswith("UTF-16")) else "utf-8"
|
||||
term = b"\x00\x00" if text_enc == "utf-16-le" else b"\x00"
|
||||
|
||||
# Records, in key order (MDict stores keys sorted; we keep caller order,
|
||||
# which the fixtures keep sorted where the reader cares).
|
||||
records = []
|
||||
for _, value in entries:
|
||||
if kind == "mdd":
|
||||
records.append(value)
|
||||
else:
|
||||
records.append(value.encode(text_enc) + term)
|
||||
|
||||
# Record blocks.
|
||||
record_blocks = []
|
||||
cur = b""
|
||||
for rec in records:
|
||||
if cur and len(cur) + len(rec) > record_block_bytes:
|
||||
record_blocks.append(cur)
|
||||
cur = b""
|
||||
cur += rec
|
||||
if cur or not record_blocks:
|
||||
record_blocks.append(cur)
|
||||
|
||||
# Key blocks with record offsets into the concatenated record space.
|
||||
offsets = []
|
||||
off = 0
|
||||
for rec in records:
|
||||
offsets.append(off)
|
||||
off += len(rec)
|
||||
keys = []
|
||||
for (key, _), o in zip(entries, offsets):
|
||||
keys.append((key, o))
|
||||
key_blocks = [keys[i:i + block_entries] for i in range(0, len(keys), block_entries)]
|
||||
|
||||
key_block_bytes = []
|
||||
for block in key_blocks:
|
||||
raw = b"".join(struct.pack(num, o) + k.encode(text_enc) + term for k, o in block)
|
||||
key_block_bytes.append((raw, frame(raw, compression)))
|
||||
|
||||
# Key-block index.
|
||||
size_fmt = ">H" if v2 else ">B"
|
||||
index = b""
|
||||
for block, (raw, packed) in zip(key_blocks, key_block_bytes):
|
||||
first = block[0][0].encode(text_enc)
|
||||
last = block[-1][0].encode(text_enc)
|
||||
unit = 2 if text_enc == "utf-16-le" else 1
|
||||
index += struct.pack(num, len(block))
|
||||
index += struct.pack(size_fmt, len(first) // unit) + first + (term if v2 else b"")
|
||||
index += struct.pack(size_fmt, len(last) // unit) + last + (term if v2 else b"")
|
||||
index += struct.pack(num, len(packed)) + struct.pack(num, len(raw))
|
||||
if v2:
|
||||
index_packed = frame(index, 2)
|
||||
if encrypted & 2:
|
||||
index_packed = encrypt_key_index(index_packed)
|
||||
else:
|
||||
index_packed = index
|
||||
|
||||
key_blocks_packed = b"".join(p for _, p in key_block_bytes)
|
||||
if v2:
|
||||
key_header = struct.pack(num, len(key_blocks)) + struct.pack(num, len(keys)) + struct.pack(num, len(index))
|
||||
key_header += struct.pack(num, len(index_packed)) + struct.pack(num, len(key_blocks_packed))
|
||||
key_header += struct.pack(">I", zlib.adler32(key_header) & 0xFFFFFFFF)
|
||||
else:
|
||||
key_header = struct.pack(num, len(key_blocks)) + struct.pack(num, len(keys))
|
||||
key_header += struct.pack(num, len(index_packed)) + struct.pack(num, len(key_blocks_packed))
|
||||
|
||||
# Record section.
|
||||
record_packed = []
|
||||
record_index = b""
|
||||
for i, blk in enumerate(record_blocks):
|
||||
framed = frame(blk, compression)
|
||||
if corrupt == "record_adler" and i == 0:
|
||||
framed = framed[:4] + struct.pack(">I", (struct.unpack(">I", framed[4:8])[0] ^ 1)) + framed[8:]
|
||||
record_packed.append(framed)
|
||||
declared_unpacked = len(blk)
|
||||
if corrupt == "huge_block" and i == 0:
|
||||
declared_unpacked = 1 << 40
|
||||
record_index += struct.pack(num, len(framed)) + struct.pack(num, declared_unpacked)
|
||||
record_blocks_packed = b"".join(record_packed)
|
||||
record_header = struct.pack(num, len(record_blocks)) + struct.pack(num, len(keys))
|
||||
record_header += struct.pack(num, len(record_index)) + struct.pack(num, len(record_blocks_packed))
|
||||
|
||||
# Header.
|
||||
attrs = {
|
||||
"GeneratedByEngineVersion": version,
|
||||
"RequiredEngineVersion": version,
|
||||
"Format": fmt,
|
||||
"KeyCaseSensitive": "No",
|
||||
"StripKey": "Yes",
|
||||
"Encrypted": str(encrypted),
|
||||
"RegisterBy": "EMail",
|
||||
"Description": description,
|
||||
"Title": title,
|
||||
"Encoding": "UTF-16" if encoding.upper().startswith("UTF-16") else encoding,
|
||||
"CreationDate": "2020-1-1",
|
||||
"Compact": "Yes",
|
||||
"Compat": "Yes",
|
||||
"Left2Right": "Yes",
|
||||
"DataSourceFormat": "107",
|
||||
"StyleSheet": stylesheet,
|
||||
}
|
||||
if kind == "mdd":
|
||||
attrs["Encoding"] = ""
|
||||
if extra_attrs:
|
||||
attrs.update(extra_attrs)
|
||||
root = "Library_Data" if kind == "mdd" else "Dictionary"
|
||||
header_text = "<%s %s/>\r\n" % (root, " ".join('%s="%s"' % (k, xml_escape(v)) for k, v in attrs.items()))
|
||||
header_bytes = header_text.encode("utf-16-le") + b"\x00\x00"
|
||||
header = struct.pack(">I", len(header_bytes)) + header_bytes
|
||||
header += struct.pack("<I", zlib.adler32(header_bytes) & 0xFFFFFFFF)
|
||||
|
||||
data = header + key_header + index_packed + key_blocks_packed + record_header + record_index + record_blocks_packed
|
||||
if corrupt == "truncate":
|
||||
data = data[:-(len(record_blocks_packed) // 2 + 1)]
|
||||
with open(path, "wb") as f:
|
||||
f.write(data)
|
||||
return len(data)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- fixtures
|
||||
HTML_ENTRIES = [
|
||||
("@@@LINK_target", "<div>target of a link</div>"),
|
||||
("alias", "@@@LINK=@@@LINK_target\r\n"),
|
||||
("dup", '<p class="a">first dup</p>'),
|
||||
("dup", '<p class="a">second dup</p>'),
|
||||
("entry", '<b>bold</b> <i>italic</i> <a href="entry://alias">alias</a> <a href="sound://a.spx">snd</a>'
|
||||
'<img src="../evil.png"><style>.inline-x { color: blue; }</style>'),
|
||||
("missing-alias", "@@@LINK=nowhere"),
|
||||
("ruby", '<table><tr><td><ruby>漢<rt>かん</rt></ruby></td></tr></table><img src="img/pic.png">'),
|
||||
("食べる", "`1`to eat`2` (ichidan)"),
|
||||
("見出し", "<span style=\"color:red;font-size:12px\">見出し語</span>"),
|
||||
]
|
||||
|
||||
TEXT_ENTRIES = [
|
||||
("alpha", "first definition"),
|
||||
("beta", "second\ndefinition with newline"),
|
||||
("gamma", "third"),
|
||||
("日本語", "Japanese text \"quoted\""),
|
||||
]
|
||||
|
||||
PNG = bytes.fromhex(
|
||||
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489"
|
||||
"0000000d49444154789c63f8ffff3f0005fe02fea72d5a5e0000000049454e44ae426082"
|
||||
)
|
||||
|
||||
MDD_ENTRIES = [
|
||||
("\\a.spx", b"not really speex"),
|
||||
("\\img\\pic.png", PNG),
|
||||
("\\style.css", ".mdx-red { color: red; }\n".encode("utf-8")),
|
||||
("\\..\\evil.png", b"traversal"),
|
||||
# UTF-16LE without a BOM and with non-ASCII text, as some MDD authors save it.
|
||||
("\\utf16.css", ".u16::before { content: \"\u2192\"; }\n".encode("utf-16-le")),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
out = lambda name: os.path.join(HERE, name) # noqa: E731
|
||||
sizes = {}
|
||||
sizes["v2_utf8_zlib_text.mdx"] = write_mdict(
|
||||
out("v2_utf8_zlib_text.mdx"), TEXT_ENTRIES, version="2.0", encoding="UTF-8", compression=2,
|
||||
fmt="Text", title="Text Fixture", description="A <b>text</b> fixture & entities")
|
||||
sizes["v2_utf8_lzo_html.mdx"] = write_mdict(
|
||||
out("v2_utf8_lzo_html.mdx"), HTML_ENTRIES, version="2.0", encoding="UTF-8", compression=1,
|
||||
fmt="Html", title="HTML Fixture", stylesheet="1\n<b>\n</b>\n2\n<i>\n</i>\n",
|
||||
description="HTML fixture with links, duplicates and a stylesheet")
|
||||
sizes["v2_utf16_encrypted2.mdx"] = write_mdict(
|
||||
out("v2_utf16_encrypted2.mdx"), TEXT_ENTRIES, version="2.0", encoding="UTF-16", compression=2,
|
||||
encrypted=2, fmt="Text", title="UTF-16 Fixture")
|
||||
sizes["v1_utf8_stored.mdx"] = write_mdict(
|
||||
out("v1_utf8_stored.mdx"), TEXT_ENTRIES, version="1.2", encoding="UTF-8", compression=0,
|
||||
fmt="Text", title="V1 Fixture")
|
||||
sizes["v2_utf8_lzo_html.mdd"] = write_mdict(
|
||||
out("v2_utf8_lzo_html.mdd"), MDD_ENTRIES, version="2.0", compression=2, kind="mdd",
|
||||
title="HTML Fixture Media")
|
||||
# Malformed inputs. Each must fail with a specific message.
|
||||
sizes["bad_truncated.mdx"] = write_mdict(
|
||||
out("bad_truncated.mdx"), TEXT_ENTRIES, fmt="Text", corrupt="truncate")
|
||||
sizes["bad_adler.mdx"] = write_mdict(
|
||||
out("bad_adler.mdx"), TEXT_ENTRIES, fmt="Text", corrupt="record_adler")
|
||||
sizes["bad_huge_block.mdx"] = write_mdict(
|
||||
out("bad_huge_block.mdx"), TEXT_ENTRIES, fmt="Text", corrupt="huge_block")
|
||||
sizes["bad_encrypted1.mdx"] = write_mdict(
|
||||
out("bad_encrypted1.mdx"), TEXT_ENTRIES, fmt="Text", encrypted=1)
|
||||
sizes["bad_gbk.mdx"] = write_mdict(
|
||||
out("bad_gbk.mdx"), TEXT_ENTRIES, fmt="Text", encoding="GBK")
|
||||
sizes["bad_v3.mdx"] = write_mdict(
|
||||
out("bad_v3.mdx"), TEXT_ENTRIES, fmt="Text", version="3.0")
|
||||
for name, size in sizes.items():
|
||||
print("%-28s %6d bytes" % (name, size))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
+128
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Writes small_dict.zip, the Yomitan dictionary used by import_equivalence_test.
|
||||
|
||||
The archive is deterministic (fixed timestamps, fixed entry order, fixed
|
||||
compression) so the golden hashes in golden.sha256 stay valid when it is
|
||||
regenerated. Run from any directory:
|
||||
|
||||
python3 tests/fixtures/yomitan/gen_fixture.py
|
||||
|
||||
Every bank kind the importer reads is present, plus a stylesheet and media so
|
||||
the equivalence test covers blobs.bin, hash.table, bloom.filter, media.bin,
|
||||
media.idx and dict.zstd.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import zipfile
|
||||
import zlib
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
OUT = os.path.join(HERE, "small_dict.zip")
|
||||
STAMP = (2020, 1, 1, 0, 0, 0)
|
||||
|
||||
# 1x1 PNG, opaque white.
|
||||
PNG = bytes.fromhex(
|
||||
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489"
|
||||
"0000000d49444154789c63f8ffff3f0005fe02fea72d5a5e0000000049454e44ae426082"
|
||||
)
|
||||
|
||||
|
||||
def dumps(value):
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def structured(text, extra=None):
|
||||
content = [{"tag": "span", "style": {"fontWeight": "bold"}, "content": text}]
|
||||
if extra:
|
||||
content.append({"tag": "div", "data": {"kind": "note"}, "content": extra})
|
||||
return {"type": "structured-content", "content": {"tag": "div", "content": content}}
|
||||
|
||||
|
||||
def term_bank_1():
|
||||
terms = []
|
||||
# Enough long glossaries to let the zstd trainer produce a dictionary.
|
||||
for i in range(48):
|
||||
gloss = "語釈 {}: これはテスト用の見出し語の説明文です。".format(i) + "同じ語尾を繰り返します。" * 3
|
||||
terms.append(["見出し{}".format(i), "みだし{}".format(i), "n", "", 100 - i, [gloss], i + 1, "P"])
|
||||
terms.append(["食べる", "たべる", "v1", "v1", 50, ["to eat", structured("食べる", "Ichidan verb")], 1000, ""])
|
||||
terms.append(["食べる", "たべる", "v1", "v1", 40, ["to eat"], 1000, ""]) # duplicate glossary text
|
||||
terms.append(["猫", "ねこ", "n", "", 10, [{"type": "image", "path": "img/neko.png", "width": 1, "height": 1}], 1001, ""])
|
||||
terms.append(["日本", "にほん", None, "", 0, ["Japan"], 1002, "P"])
|
||||
terms.append(["日本", "にっぽん", "n", "", 0, ["Japan"], 1002, ""])
|
||||
terms.append(["同形", "", "n", "", 0, ["reading omitted"], 1003, ""])
|
||||
return terms
|
||||
|
||||
|
||||
def term_bank_2():
|
||||
return [
|
||||
["走る", "はしる", "v5", "v5", 5, ["to run"], 2000, ""],
|
||||
["走る", "はしる", "v5", "v5", 5, ["to run"], 2000, ""], # exact duplicate entry
|
||||
["\"quoted\" [brackets] \\backslash", "quoted", "", "", 0, ["escapes \"\\ 【】"], 2001, ""],
|
||||
]
|
||||
|
||||
|
||||
def term_meta_bank_1():
|
||||
return [
|
||||
["食べる", "freq", 12],
|
||||
["食べる", "freq", {"reading": "たべる", "frequency": {"value": 12, "displayValue": "12㋕"}}],
|
||||
["猫", "freq", {"value": 3, "displayValue": "3"}],
|
||||
["食べる", "pitch", {"reading": "たべる", "pitches": [{"position": 2}, {"position": 0, "nasal": [1], "devoice": []}]}],
|
||||
["日本", "ipa", {"reading": "にほん", "transcriptions": [{"ipa": "ɲihoɰ̃"}]}],
|
||||
]
|
||||
|
||||
|
||||
def kanji_bank_1():
|
||||
return [
|
||||
["食", "ショク ジキ", "く.う た.べる", "jouyou", ["eat", "food"], {"grade": "2", "strokes": "9"}],
|
||||
["猫", "ビョウ", "ねこ", "jouyou", ["cat"], {"grade": "8"}],
|
||||
]
|
||||
|
||||
|
||||
def kanji_meta_bank_1():
|
||||
return [["食", "freq", 7], ["猫", "freq", {"value": 9, "displayValue": "9"}]]
|
||||
|
||||
|
||||
def tag_bank_1():
|
||||
return [["n", "partOfSpeech", -3, "noun", 0], ["v1", "partOfSpeech", -3, "Ichidan verb", 0], ["P", "popular", -10, "common", 10]]
|
||||
|
||||
|
||||
def main():
|
||||
entries = [
|
||||
("index.json", dumps({
|
||||
"title": "Hoshidicts Fixture",
|
||||
"revision": "fixture-1",
|
||||
"format": 3,
|
||||
"sequenced": True,
|
||||
"author": "hoshidicts tests",
|
||||
"description": "Deterministic fixture for the import equivalence test.",
|
||||
"sourceLanguage": "ja",
|
||||
"targetLanguage": "en",
|
||||
"frequencyMode": "rank-based",
|
||||
}).encode()),
|
||||
("styles.css", b".mdict-yomitan-content { color: #333; }\n"),
|
||||
("term_bank_1.json", dumps(term_bank_1()).encode()),
|
||||
("term_bank_2.json", dumps(term_bank_2()).encode()),
|
||||
("term_meta_bank_1.json", dumps(term_meta_bank_1()).encode()),
|
||||
("kanji_bank_1.json", dumps(kanji_bank_1()).encode()),
|
||||
("kanji_meta_bank_1.json", dumps(kanji_meta_bank_1()).encode()),
|
||||
("tag_bank_1.json", dumps(tag_bank_1()).encode()),
|
||||
("img/", b""),
|
||||
("img/neko.png", PNG),
|
||||
("audio/neko.txt", b"stored, not deflated"),
|
||||
]
|
||||
with zipfile.ZipFile(OUT, "w") as zf:
|
||||
for name, data in entries:
|
||||
info = zipfile.ZipInfo(name, date_time=STAMP)
|
||||
info.create_system = 3
|
||||
if name.endswith("/"):
|
||||
info.external_attr = 0o40755 << 16
|
||||
zf.writestr(info, b"")
|
||||
continue
|
||||
info.external_attr = 0o644 << 16
|
||||
info.compress_type = zipfile.ZIP_STORED if name.startswith("audio/") else zipfile.ZIP_DEFLATED
|
||||
zf.writestr(info, data, compresslevel=9)
|
||||
print(OUT, os.path.getsize(OUT), "bytes")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,11 @@
|
||||
# SHA-256 of every file the pre-DictionarySource importer (bee-san/hoshidicts main @ fa833f9) wrote for small_dict.zip.
|
||||
# index.json is hashed with "importDate":<n> replaced by "importDate":0. Regenerate only when the output format changes on purpose.
|
||||
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 .hoshidicts_6
|
||||
552b6e672c3b8e64d12e7eb43182c23234eaaea506bbd1c19b7d447e523840ad blobs.bin
|
||||
4bb3bf0755e28ae676f98c575174a62054651cc0004173bb03bf0d8b5a875c5c bloom.filter
|
||||
91de707c8eb26bff94240513c03a41b7f9995c67f5392a621fb3151aae429ef3 dict.zstd
|
||||
517a21ea1d33fa05bdc9e4655c3c8d171ded15002f297843cb17f1e9db3f045a hash.table
|
||||
b27413b159ab02a83b2f820087db41ffaf81ec2f7fb97c20493b41a1fcba3f5f media.bin
|
||||
56879507ea4d0e870990d88b906e754eaa34f47b4e634842a6f4499290f3fd2c media.idx
|
||||
91b1a6c60e52df5da7db5a1ff10a2d7d2b7d25a282b2142206f45e3cc63bb376 index.json
|
||||
4f6e1b0bf6ed1f02d3eb24617e9693284ea0607f30e92cab693e0583675bb9ca scan.idx
|
||||
BIN
Binary file not shown.
Reference in New Issue
Block a user